From ba13e86a70c331275d40ed8f84c3f264845afc6e Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 11 Mar 2025 13:30:06 +0100 Subject: [PATCH] [Streams] Replay loghub data with synthtrace (#212120) Download, parse and replay loghub data with Synthtrace, for use in the Streams project. In summary: - adds a `@kbn/sample-log-parser` package which parses Loghub sample data, creates valid parsers for extracting and replacing timestamps, using the LLM - add a `sample_logs` scenario which uses the parsed data sets to replay Loghub data continuously as if it were live data - refactor some parts of Synthtrace (follow-up work captured in https://github.com/elastic/kibana/issues/212179) ## Synthtrace changes - Replace custom Logger object with Kibana-standard ToolingLog - Report progress and estimated time to completion for long-running jobs - Simplify scenarioOpts (allow comma-separated key-value pairs instead of just JSON) - Simplify client initialization - When using workers, only bootstrap once (in the main thread) - Allow workers to gracefully shutdown - Downgrade some logging levels for less noise --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + package.json | 1 + .../kbn-journeys/services/synthtrace.ts | 42 +- .../shared/kbn-apm-synthtrace-client/index.ts | 2 +- .../src/lib/interval.ts | 15 + .../src/lib/timerange.ts | 15 +- .../src/lib/timerange_progress_reporter.ts | 82 ++ .../kbn-apm-synthtrace-client/tsconfig.json | 1 + .../shared/kbn-apm-synthtrace/index.ts | 7 +- .../src/cli/run_synthtrace.ts | 56 +- .../kbn-apm-synthtrace/src/cli/scenario.ts | 36 +- .../src/cli/utils/bootstrap.ts | 96 +- .../src/cli/utils/get_apm_es_client.ts | 41 - .../src/cli/utils/get_clients.ts | 95 ++ .../cli/utils/get_entites_kibana_client.ts | 20 - .../src/cli/utils/get_entities_es_client.ts | 37 - .../src/cli/utils/get_infra_es_client.ts | 37 - .../src/cli/utils/get_kibana_client.ts | 5 +- .../src/cli/utils/get_logs_es_client.ts | 36 - .../src/cli/utils/get_otel_es_client.ts | 36 - .../src/cli/utils/get_service_urls.test.ts | 2 +- .../src/cli/utils/get_service_urls.ts | 6 +- .../src/cli/utils/get_synthetics_es_client.ts | 36 - .../src/cli/utils/index_historical_data.ts | 80 ++ .../src/cli/utils/logger_proxy.ts | 33 +- .../src/cli/utils/parse_run_cli_flags.ts | 20 +- .../cli/utils/start_historical_data_upload.ts | 104 ++- .../src/cli/utils/start_live_data_upload.ts | 158 +--- .../src/cli/utils/stream_manager.ts | 192 ++++ .../src/cli/utils/synthtrace_worker.ts | 155 +--- .../client/apm_synthtrace_es_client/index.ts | 2 +- .../client/apm_synthtrace_kibana_client.ts | 118 +-- .../entities_synthtrace_kibana_client.ts | 34 +- .../src/lib/shared/base_client.ts | 31 +- .../src/lib/shared/base_kibana_client.ts | 57 ++ .../src/lib/shared/client_headers.ts | 6 + .../lib/streams/streams_synthtrace_client.ts | 108 +++ .../src/lib/utils/create_logger.ts | 71 +- .../src/lib/utils/log_perf.ts | 9 +- .../src/lib/utils/with_client.ts | 13 +- .../src/scenarios/sample_logs.ts | 41 + .../src/scenarios/traces_logs_entities.ts | 3 +- .../src/test/es_client_indexer.test.ts | 13 +- .../test/scenarios/01_simple_trace.test.ts | 4 +- .../scenarios/02_transaction_metrics.test.ts | 4 +- .../03_span_destination_metrics.test.ts | 4 +- .../scenarios/04_breakdown_metrics.test.ts | 7 +- .../shared/kbn-apm-synthtrace/tsconfig.json | 3 + tsconfig.base.json | 2 + .../shared/kbn-sample-parser/README.md | 12 + .../shared/kbn-sample-parser/cli/index.ts | 103 +++ .../client/create_loghub_generator.test.ts | 99 +++ .../client/create_loghub_generator.ts | 85 ++ .../shared/kbn-sample-parser/client/index.ts | 76 ++ .../kbn-sample-parser/client/parse_dataset.ts | 52 ++ .../shared/kbn-sample-parser/client/types.ts | 22 + .../shared/kbn-sample-parser/index.ts | 10 + .../shared/kbn-sample-parser/jest.config.js | 12 + .../shared/kbn-sample-parser/kibana.jsonc | 8 + .../shared/kbn-sample-parser/package.json | 6 + .../parsers/Android/parser.ts | 25 + .../parsers/Android/queries.json | 365 ++++++++ .../parsers/Apache/parser.ts | 27 + .../parsers/Apache/queries.json | 118 +++ .../kbn-sample-parser/parsers/BGL/parser.ts | 26 + .../parsers/BGL/queries.json | 821 ++++++++++++++++++ .../kbn-sample-parser/parsers/HDFS/parser.ts | 31 + .../parsers/HDFS/queries.json | 232 +++++ .../kbn-sample-parser/parsers/HPC/parser.ts | 21 + .../parsers/HPC/queries.json | 330 +++++++ .../parsers/Hadoop/parser.ts | 24 + .../parsers/Hadoop/queries.json | 308 +++++++ .../parsers/HealthApp/invalid_parser.ts | 26 + .../parsers/HealthApp/parser.ts | 23 + .../parsers/HealthApp/queries.json | 226 +++++ .../kbn-sample-parser/parsers/Linux/parser.ts | 26 + .../parsers/Linux/queries.json | 369 ++++++++ .../kbn-sample-parser/parsers/Mac/parser.ts | 28 + .../parsers/Mac/queries.json | 457 ++++++++++ .../parsers/OpenSSH/parser.ts | 27 + .../parsers/OpenSSH/queries.json | 517 +++++++++++ .../parsers/OpenStack/parser.ts | 24 + .../parsers/OpenStack/queries.json | 207 +++++ .../parsers/Proxifier/parser.ts | 27 + .../parsers/Proxifier/queries.json | 153 ++++ .../kbn-sample-parser/parsers/Spark/parser.ts | 24 + .../parsers/Spark/queries.json | 280 ++++++ .../parsers/Thunderbird/parser.ts | 26 + .../parsers/Thunderbird/queries.json | 360 ++++++++ .../parsers/Windows/parser.ts | 41 + .../parsers/Windows/queries.json | 268 ++++++ .../parsers/Zookeeper/parser.ts | 25 + .../parsers/Zookeeper/queries.json | 201 +++++ .../shared/kbn-sample-parser/src/constants.ts | 13 + .../src/create_openai_client.ts | 34 + .../src/ensure_loghub_repo.ts | 46 + .../src/ensure_valid_parser.ts | 176 ++++ .../src/ensure_valid_queries.ts | 201 +++++ .../kbn-sample-parser/src/get_parser.ts | 16 + .../kbn-sample-parser/src/get_queries.ts | 17 + .../src/read_loghub_system_files.ts | 75 ++ .../shared/kbn-sample-parser/src/types.ts | 11 + .../shared/kbn-sample-parser/src/utils.ts | 30 + .../kbn-sample-parser/src/validate_parser.ts | 49 ++ .../kbn-sample-parser/src/validate_queries.ts | 149 ++++ .../shared/kbn-sample-parser/tsconfig.json | 21 + x-pack/scripts/sample_log_parser.js | 9 + .../scripts/evaluation/setup_synthtrace.ts | 28 +- yarn.lock | 4 + 109 files changed, 8096 insertions(+), 908 deletions(-) create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange_progress_reporter.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_clients.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts delete mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/index_historical_data.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_kibana_client.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/lib/streams/streams_synthtrace_client.ts create mode 100644 src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/sample_logs.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/README.md create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/cli/index.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.test.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/client/index.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/client/parse_dataset.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/client/types.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/index.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/jest.config.js create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/kibana.jsonc create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/package.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/invalid_parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/queries.json create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/constants.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/create_openai_client.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_loghub_repo.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_queries.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/get_parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/get_queries.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/read_loghub_system_files.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/types.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/utils.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/validate_parser.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/src/validate_queries.ts create mode 100644 x-pack/platform/packages/shared/kbn-sample-parser/tsconfig.json create mode 100755 x-pack/scripts/sample_log_parser.js diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7bb868068f9b..f79ebb09185d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -825,6 +825,7 @@ x-pack/platform/packages/shared/kbn-event-stacktrace @elastic/obs-ux-infra_servi x-pack/platform/packages/shared/kbn-inference-endpoint-ui-common @elastic/response-ops @elastic/appex-ai-infra @elastic/obs-ai-assistant @elastic/security-generative-ai x-pack/platform/packages/shared/kbn-key-value-metadata-table @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team x-pack/platform/packages/shared/kbn-langchain @elastic/security-generative-ai +x-pack/platform/packages/shared/kbn-sample-parser @elastic/streams-program-team x-pack/platform/packages/shared/kbn-slo-schema @elastic/obs-ux-management-team x-pack/platform/packages/shared/kbn-streams-schema @elastic/streams-program-team x-pack/platform/packages/shared/logs-overview @elastic/obs-ux-logs-team diff --git a/package.json b/package.json index 5a87328b7d61..5f9b2675cc35 100644 --- a/package.json +++ b/package.json @@ -1493,6 +1493,7 @@ "@kbn/repo-source-classifier": "link:packages/kbn-repo-source-classifier", "@kbn/repo-source-classifier-cli": "link:packages/kbn-repo-source-classifier-cli", "@kbn/reporting-mocks-server": "link:src/platform/packages/private/kbn-reporting/mocks_server", + "@kbn/sample-log-parser": "link:x-pack/platform/packages/shared/kbn-sample-parser", "@kbn/scout": "link:src/platform/packages/shared/kbn-scout", "@kbn/scout-info": "link:src/platform/packages/private/kbn-scout-info", "@kbn/scout-oblt": "link:x-pack/solutions/observability/packages/kbn-scout-oblt", diff --git a/src/platform/packages/private/kbn-journeys/services/synthtrace.ts b/src/platform/packages/private/kbn-journeys/services/synthtrace.ts index 00e787498f55..f146af112e9e 100644 --- a/src/platform/packages/private/kbn-journeys/services/synthtrace.ts +++ b/src/platform/packages/private/kbn-journeys/services/synthtrace.ts @@ -15,7 +15,7 @@ import { } from '@kbn/apm-synthtrace'; import { ToolingLog } from '@kbn/tooling-log'; import Url from 'url'; -import { Logger } from '@kbn/apm-synthtrace/src/lib/utils/create_logger'; +import { type Logger, extendToolingLog } from '@kbn/apm-synthtrace'; import { Auth, Es } from '.'; import { KibanaUrl } from './kibana_url'; @@ -39,45 +39,9 @@ export async function getSynthtraceClient( } } -// Adapting ToolingLog instance to Logger interface -class LoggerAdapter implements Logger { - private log: ToolingLog; - private joiner = ', '; - - constructor(log: ToolingLog) { - this.log = log; - } - - debug(...args: any[]): void { - this.log.debug(args.join(this.joiner)); - } - - info(...args: any[]): void { - this.log.info(args.join(this.joiner)); - } - - warn(...args: any[]): void { - this.log.warning(args.join(this.joiner)); - } - - error(arg: string | Error): void { - this.log.error(arg); - } - - perf(name: string, cb: () => T): T { - const startTime = Date.now(); - const result = cb(); - const duration = Date.now() - startTime; - const durationInSeconds = duration / 1000; - const formattedTime = durationInSeconds.toFixed(3) + 's'; - this.log.info(`${name} took ${formattedTime}.`); - return result; - } -} - async function initInfraSynthtraceClient(options: SynthtraceClientOptions) { const { log, es, auth, kbnUrl } = options; - const logger: Logger = new LoggerAdapter(log); + const logger: Logger = extendToolingLog(log); const synthKbnClient = new InfraSynthtraceKibanaClient({ logger, @@ -99,7 +63,7 @@ async function initInfraSynthtraceClient(options: SynthtraceClientOptions) { async function initApmSynthtraceClient(options: SynthtraceClientOptions) { const { log, es, auth, kbnUrl } = options; - const logger: Logger = new LoggerAdapter(log); + const logger: Logger = extendToolingLog(log); const kibanaUrl = new URL(kbnUrl.get()); const kibanaUrlWithAuth = Url.format({ protocol: kibanaUrl.protocol, diff --git a/src/platform/packages/shared/kbn-apm-synthtrace-client/index.ts b/src/platform/packages/shared/kbn-apm-synthtrace-client/index.ts index ff343ab78ab4..8d25ab5a999c 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace-client/index.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace-client/index.ts @@ -27,7 +27,7 @@ export { Entity } from './src/lib/entity'; export { infra, type InfraDocument } from './src/lib/infra'; export { parseInterval } from './src/lib/interval'; export { monitoring, type MonitoringDocument } from './src/lib/monitoring'; -export type { Serializable } from './src/lib/serializable'; +export { Serializable } from './src/lib/serializable'; export { timerange } from './src/lib/timerange'; export type { Timerange } from './src/lib/timerange'; export { dedot } from './src/lib/utils/dedot'; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/interval.ts b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/interval.ts index 5a5ed3ab5fdb..9d494f200923 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/interval.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/interval.ts @@ -7,11 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { ToolingLog } from '@kbn/tooling-log'; import { castArray } from 'lodash'; import moment, { unitOfTime } from 'moment'; import { SynthtraceGenerator } from '../types'; import { Fields } from './entity'; import { Serializable } from './serializable'; +import { TimerangeProgressReporter } from './timerange_progress_reporter'; export function parseInterval(interval: string): { intervalAmount: number; @@ -32,6 +34,7 @@ interface IntervalOptions { to: Date; interval: string; rate?: number; + log?: ToolingLog; } interface StepDetails { @@ -86,10 +89,22 @@ export class Interval { }; let index = 0; + const calculateEvery = 10; + + const reporter = this.options.log + ? new TimerangeProgressReporter({ + log: this.options.log, + reportEvery: 5000, + total: timestamps.length, + }) + : undefined; for (const timestamp of timestamps) { const events = castArray(map(timestamp, index, stepDetails)); index++; + if (index % calculateEvery === 0) { + reporter?.next(index); + } for (const event of events) { yield event; } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange.ts b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange.ts index 1c6f12414a14..f07ea8d4599f 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange.ts @@ -9,15 +9,20 @@ import datemath from '@kbn/datemath'; import type { Moment } from 'moment'; +import { ToolingLog } from '@kbn/tooling-log'; import { GaussianEvents } from './gaussian_events'; import { Interval } from './interval'; import { PoissonEvents } from './poisson_events'; export class Timerange { - constructor(public readonly from: Date, public readonly to: Date) {} + constructor( + public readonly from: Date, + public readonly to: Date, + private readonly log?: ToolingLog + ) {} interval(interval: string) { - return new Interval({ from: this.from, to: this.to, interval }); + return new Interval({ from: this.from, to: this.to, interval, log: this.log }); } ratePerMinute(rate: number) { @@ -39,7 +44,7 @@ export class Timerange { return Array.from({ length: segmentCount }, (_, i) => { const from = new Date(this.from.getTime() + i * segmentDuration); const to = new Date(from.getTime() + segmentDuration); - return new Timerange(from, to); + return new Timerange(from, to, this.log); }); } @@ -65,7 +70,7 @@ function getDateFrom(date: DateLike, now: Date): Date { return date.toDate(); } -export function timerange(from: DateLike, to: DateLike) { +export function timerange(from: DateLike, to: DateLike, log?: ToolingLog) { const now = new Date(); - return new Timerange(getDateFrom(from, now), getDateFrom(to, now)); + return new Timerange(getDateFrom(from, now), getDateFrom(to, now), log); } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange_progress_reporter.ts b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange_progress_reporter.ts new file mode 100644 index 000000000000..f0f025c89ad8 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange_progress_reporter.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { first, last } from 'lodash'; +import moment from 'moment'; + +interface TimerangeProgressOptions { + log: ToolingLog; + total: number; + reportEvery: number; +} + +export class TimerangeProgressReporter { + private readonly startOfRun: number = performance.now(); + + private measurements: Array<{ + measuredAt: number; + index: number; + }> = [ + { + measuredAt: this.startOfRun, + index: 0, + }, + ]; + + private lastReported: number = this.startOfRun; + + constructor(private readonly options: TimerangeProgressOptions) {} + + next(index: number) { + const now = performance.now(); + + this.measurements.unshift({ + index, + measuredAt: now, + }); + + this.measurements.length = Math.min(10, this.measurements.length); + + const timeSinceLastReported = now - this.lastReported; + + if (timeSinceLastReported >= this.options.reportEvery) { + this.report(now); + } + } + + private report(now: number) { + this.lastReported = now; + + const firstMeasurement = first(this.measurements)!; + const lastMeasurement = last(this.measurements)!; + + const totalDurationFormatted = moment.duration(now - this.startOfRun).humanize(); + + const indicesLeft = this.options.total - lastMeasurement.index; + + const measuredIndicesProcessed = lastMeasurement.index - firstMeasurement.index; + + const measuredDuration = lastMeasurement.measuredAt - firstMeasurement.measuredAt; + + const indicesPerMs = measuredIndicesProcessed / measuredDuration; + + const timeLeft = indicesLeft / indicesPerMs; + + const timeLeftFormatted = moment.duration(timeLeft).humanize(true); + + const totalProgress = lastMeasurement.index / this.options.total; + + this.options.log.info( + `progress=${(totalProgress * 100).toPrecision( + 3 + )}%, duration=${totalDurationFormatted}, eta=${timeLeftFormatted}` + ); + } +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace-client/tsconfig.json b/src/platform/packages/shared/kbn-apm-synthtrace-client/tsconfig.json index 2360dd2e0f9f..08b76000d921 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace-client/tsconfig.json +++ b/src/platform/packages/shared/kbn-apm-synthtrace-client/tsconfig.json @@ -15,5 +15,6 @@ "kbn_references": [ "@kbn/datemath", "@kbn/safer-lodash-set", + "@kbn/tooling-log", ] } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/index.ts b/src/platform/packages/shared/kbn-apm-synthtrace/index.ts index 8cb2ca6eaa8f..c69adf79b583 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/index.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/index.ts @@ -7,7 +7,12 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { createLogger, LogLevel } from './src/lib/utils/create_logger'; +export { + createLogger, + LogLevel, + type Logger, + extendToolingLog, +} from './src/lib/utils/create_logger'; export { ApmSynthtraceEsClient } from './src/lib/apm/client/apm_synthtrace_es_client'; export { ApmSynthtraceKibanaClient } from './src/lib/apm/client/apm_synthtrace_kibana_client'; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/run_synthtrace.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/run_synthtrace.ts index f4646de82d19..e827caaad41c 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/run_synthtrace.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/run_synthtrace.ts @@ -67,15 +67,55 @@ function options(y: Argv) { number: true, default: 1, }) + .option('debug', { + describe: 'Use a debug log level', + boolean: true, + }) + .option('verbose', { + describe: 'Use a verbose log level', + boolean: true, + }) .option('logLevel', { describe: 'Log level', - choices: ['trace', 'debug', 'info', 'error'], + choices: ['verbose', 'debug', 'info', 'error'], default: 'info', }) .option('scenarioOpts', { describe: 'Options specific to the scenario', - coerce: (arg) => { - return arg as Record | undefined; + type: 'string', + coerce: (arg: string): Record => { + if (!arg) { + return {}; + } + + let scenarioOptions: Record = {}; + + try { + scenarioOptions = JSON.parse(arg); + } catch (error) { + scenarioOptions = Object.fromEntries( + arg.split(',').map((kv) => { + const [key, value] = kv + .trim() + .split('=') + .map((part) => part.trim()); + if (value === 'true') { + return [key, true]; + } + if (value === 'false') { + return [key, false]; + } + + if (!isNaN(Number(value))) { + return [key, Number(value)]; + } + + return [key, value]; + }) + ); + } + + return scenarioOptions; }, }) .option('assume-package-version', { @@ -95,20 +135,18 @@ function options(y: Argv) { async function run(argv: RunCliFlags) { const runOptions = parseRunCliFlags(argv); - const toMs = datemath.parse(String(argv.to ?? 'now'))!.valueOf(); - const to = new Date(toMs); + const to = datemath.parse(String(argv.to ?? 'now'))!.valueOf(); const defaultTimeRange = '1m'; - const fromMs = argv.from + const from = argv.from ? datemath.parse(String(argv.from))!.valueOf() - : toMs - intervalToMs(defaultTimeRange); - const from = new Date(fromMs); + : to - intervalToMs(defaultTimeRange); const live = argv.live; if (live) { - await startLiveDataUpload({ runOptions, start: from }); + await startLiveDataUpload({ runOptions, from, to }); } else { await startHistoricalDataUpload({ runOptions, from, to }); } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/scenario.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/scenario.ts index 6b25f7873514..b78c13d55d77 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/scenario.ts @@ -7,40 +7,22 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { Timerange } from '@kbn/apm-synthtrace-client'; -import { - ApmSynthtraceEsClient, - InfraSynthtraceEsClient, - LogsSynthtraceEsClient, - SyntheticsSynthtraceEsClient, - OtelSynthtraceEsClient, - EntitiesSynthtraceEsClient, -} from '../..'; +import { Fields, Timerange } from '@kbn/apm-synthtrace-client'; import { Logger } from '../lib/utils/create_logger'; import { ScenarioReturnType } from '../lib/utils/with_client'; +import { SynthtraceClients } from './utils/get_clients'; import { RunOptions } from './utils/parse_run_cli_flags'; -import { EntitiesSynthtraceKibanaClient } from '../lib/entities/entities_synthtrace_kibana_client'; -interface EsClients { - apmEsClient: ApmSynthtraceEsClient; - logsEsClient: LogsSynthtraceEsClient; - infraEsClient: InfraSynthtraceEsClient; - syntheticsEsClient: SyntheticsSynthtraceEsClient; - otelEsClient: OtelSynthtraceEsClient; - entitiesEsClient: EntitiesSynthtraceEsClient; -} +export type ScenarioInitOptions = RunOptions & { logger: Logger; from: number; to: number }; +export type ScenarioPhaseOptions = SynthtraceClients; -interface KibanaClients { - entitiesKibanaClient: EntitiesSynthtraceKibanaClient; -} - -type Generate = (options: { +type Generate = (options: { range: Timerange; - clients: EsClients; + clients: SynthtraceClients; }) => ScenarioReturnType | Array>; -export type Scenario = (options: RunOptions & { logger: Logger }) => Promise<{ - bootstrap?: (options: EsClients & KibanaClients) => Promise; +export type Scenario = (options: ScenarioInitOptions) => Promise<{ + bootstrap?: (options: ScenarioPhaseOptions) => Promise; generate: Generate; - teardown?: (options: EsClients & KibanaClients) => Promise; + teardown?: (options: ScenarioPhaseOptions) => Promise; }>; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts index a305e4354c14..50b517c3c8a3 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts @@ -7,23 +7,20 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { Client, HttpConnection } from '@elastic/elasticsearch'; import { createLogger } from '../../lib/utils/create_logger'; -import { getApmEsClient } from './get_apm_es_client'; -import { getLogsEsClient } from './get_logs_es_client'; -import { getInfraEsClient } from './get_infra_es_client'; +import { getClients } from './get_clients'; import { getKibanaClient } from './get_kibana_client'; import { getServiceUrls } from './get_service_urls'; import { RunOptions } from './parse_run_cli_flags'; -import { getSyntheticsEsClient } from './get_synthetics_es_client'; -import { getOtelSynthtraceEsClient } from './get_otel_es_client'; -import { getEntitiesEsClient } from './get_entities_es_client'; -import { getEntitiesKibanaClient } from './get_entites_kibana_client'; +import { getEsClientTlsSettings } from './ssl'; -export async function bootstrap(runOptions: RunOptions) { +export async function bootstrap({ + skipClientBootstrap, + ...runOptions +}: RunOptions & { skipClientBootstrap?: boolean }) { const logger = createLogger(runOptions.logLevel); - let version = runOptions['assume-package-version']; - const { kibanaUrl, esUrl } = await getServiceUrls({ ...runOptions, logger }); const kibanaClient = getKibanaClient({ @@ -31,76 +28,37 @@ export async function bootstrap(runOptions: RunOptions) { logger, }); - if (!version) { - version = await kibanaClient.fetchLatestApmPackageVersion(); - await kibanaClient.installApmPackage(version); - } else if (version === 'latest') { - version = await kibanaClient.fetchLatestApmPackageVersion(); - } - - logger.info(`Using package version: ${version}`); - - const apmEsClient = getApmEsClient({ - target: esUrl, - logger, - concurrency: runOptions.concurrency, - version, + const client = new Client({ + node: esUrl, + tls: getEsClientTlsSettings(esUrl), + Connection: HttpConnection, + requestTimeout: 30_000, }); - const logsEsClient = getLogsEsClient({ - target: esUrl, + const clients = await getClients({ logger, - concurrency: runOptions.concurrency, - }); - - const infraEsClient = getInfraEsClient({ - target: esUrl, - logger, - concurrency: runOptions.concurrency, - }); - - const entitiesEsClient = getEntitiesEsClient({ - target: esUrl, - logger, - concurrency: runOptions.concurrency, - }); - - const entitiesKibanaClient = getEntitiesKibanaClient({ - target: kibanaUrl, - logger, - }); - - const syntheticsEsClient = getSyntheticsEsClient({ - target: esUrl, - logger, - concurrency: runOptions.concurrency, - }); - const otelEsClient = getOtelSynthtraceEsClient({ - target: esUrl, - logger, - concurrency: runOptions.concurrency, + packageVersion: runOptions['assume-package-version'], + options: { + client, + logger, + concurrency: runOptions.concurrency, + kibana: kibanaClient, + }, + skipBootstrap: skipClientBootstrap, }); if (runOptions.clean) { - await apmEsClient.clean(); - await logsEsClient.clean(); - await infraEsClient.clean(); - await entitiesEsClient.clean(); - await syntheticsEsClient.clean(); - await otelEsClient.clean(); + for (const synthtraceClient of Object.values(clients)) { + if ('clean' in synthtraceClient) { + await synthtraceClient.clean(); + } + } } return { + clients, logger, - apmEsClient, - logsEsClient, - infraEsClient, - entitiesEsClient, - syntheticsEsClient, - otelEsClient, - version, kibanaUrl, esUrl, - entitiesKibanaClient, }; } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts deleted file mode 100644 index 90833654a81e..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { ApmSynthtraceEsClient } from '../../..'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; - -export function getApmEsClient({ - target, - logger, - version, - concurrency, -}: Pick & { - version: string; - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - const apmEsClient = new ApmSynthtraceEsClient({ - client, - logger, - version, - concurrency, - }); - - return apmEsClient; -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_clients.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_clients.ts new file mode 100644 index 000000000000..75d736b78cf4 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_clients.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Required } from 'utility-types'; +import { Client } from '@elastic/elasticsearch'; +import { ApmSynthtraceEsClient } from '../../lib/apm/client/apm_synthtrace_es_client'; +import { ApmSynthtraceKibanaClient } from '../../lib/apm/client/apm_synthtrace_kibana_client'; +import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client'; +import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client'; +import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client'; +import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client'; +import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client'; +import { SynthtraceEsClientOptions } from '../../lib/shared/base_client'; +import { StreamsSynthtraceClient } from '../../lib/streams/streams_synthtrace_client'; +import { SyntheticsSynthtraceEsClient } from '../../lib/synthetics/synthetics_synthtrace_es_client'; +import { Logger } from '../../lib/utils/create_logger'; + +export interface SynthtraceClients { + apmEsClient: ApmSynthtraceEsClient; + entitiesEsClient: EntitiesSynthtraceEsClient; + infraEsClient: InfraSynthtraceEsClient; + logsEsClient: LogsSynthtraceEsClient; + otelEsClient: OtelSynthtraceEsClient; + streamsClient: StreamsSynthtraceClient; + syntheticsEsClient: SyntheticsSynthtraceEsClient; + entitiesKibanaClient: EntitiesSynthtraceKibanaClient; + esClient: Client; +} + +export async function getClients({ + logger, + options, + packageVersion, + skipBootstrap, +}: { + logger: Logger; + options: Required, 'kibana'>; + packageVersion?: string; + skipBootstrap?: boolean; +}): Promise { + const apmKibanaClient = new ApmSynthtraceKibanaClient({ + logger, + kibanaClient: options.kibana, + }); + + let version = packageVersion; + + if (!version) { + version = await apmKibanaClient.fetchLatestApmPackageVersion(); + if (!skipBootstrap) { + await apmKibanaClient.installApmPackage(version); + } + } else if (version === 'latest') { + version = await apmKibanaClient.fetchLatestApmPackageVersion(); + } + + logger.debug(`Using package version: ${version}`); + + const apmEsClient = new ApmSynthtraceEsClient({ + ...options, + version, + }); + + const logsEsClient = new LogsSynthtraceEsClient(options); + const infraEsClient = new InfraSynthtraceEsClient(options); + const entitiesEsClient = new EntitiesSynthtraceEsClient(options); + + const entitiesKibanaClient = new EntitiesSynthtraceKibanaClient({ + ...options, + kibanaClient: options.kibana, + }); + + const syntheticsEsClient = new SyntheticsSynthtraceEsClient(options); + const otelEsClient = new OtelSynthtraceEsClient(options); + + const streamsClient = new StreamsSynthtraceClient(options); + + return { + apmEsClient, + entitiesEsClient, + infraEsClient, + logsEsClient, + otelEsClient, + streamsClient, + syntheticsEsClient, + entitiesKibanaClient, + esClient: options.client, + }; +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts deleted file mode 100644 index eb06e42da3aa..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client'; -import { Logger } from '../../lib/utils/create_logger'; - -export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) { - const kibanaClient = new EntitiesSynthtraceKibanaClient({ - logger, - target, - }); - - return kibanaClient; -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts deleted file mode 100644 index e9ced638b571..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; - -export function getEntitiesEsClient({ - target, - logger, - concurrency, -}: Pick & { - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - return new EntitiesSynthtraceEsClient({ - client, - logger, - concurrency, - refreshAfterIndex: true, - }); -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts deleted file mode 100644 index 00ebedbfb179..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; - -export function getInfraEsClient({ - target, - logger, - concurrency, -}: Pick & { - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - return new InfraSynthtraceEsClient({ - client, - logger, - concurrency, - refreshAfterIndex: true, - }); -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_kibana_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_kibana_client.ts index f9612bb5494e..46947855dad8 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_kibana_client.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_kibana_client.ts @@ -7,12 +7,11 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { ApmSynthtraceKibanaClient } from '../../lib/apm/client/apm_synthtrace_kibana_client'; +import { KibanaClient } from '../../lib/shared/base_kibana_client'; import { Logger } from '../../lib/utils/create_logger'; export function getKibanaClient({ target, logger }: { target: string; logger: Logger }) { - const kibanaClient = new ApmSynthtraceKibanaClient({ - logger, + const kibanaClient = new KibanaClient({ target, }); diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts deleted file mode 100644 index 75603c94eb0b..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; - -export function getLogsEsClient({ - target, - logger, - concurrency, -}: Pick & { - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - return new LogsSynthtraceEsClient({ - client, - logger, - concurrency, - }); -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts deleted file mode 100644 index 236ad2812ef4..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; -import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client'; - -export function getOtelSynthtraceEsClient({ - target, - logger, - concurrency, -}: Pick & { - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - return new OtelSynthtraceEsClient({ - client, - logger, - concurrency, - }); -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.test.ts index 795005ce5b34..2842ee5aa2d0 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.test.ts @@ -178,7 +178,7 @@ describe('getServiceUrls', () => { const target = 'https://elastic_serverless:changeme@127.0.0.1:9200'; const kibana = 'https://elastic_serverless:changeme@localhost:5601'; - const warnSpy = jest.spyOn(logger, 'warn'); + const warnSpy = jest.spyOn(logger, 'warning'); mockFetchWithAllowedSegments([target, kibana]); await expectServiceUrls(target, kibana, { esUrl: 'https://elastic_serverless:changeme@127.0.0.1:9200', diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts index e4cfb14062a5..a43e5bee9d43 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts @@ -98,7 +98,7 @@ async function getKibanaUrl({ ); } - logger.info(`Discovered kibana running at: ${stripAuthIfCi(discoveredKibanaUrlWithAuth)}`); + logger.debug(`Discovered kibana running at: ${stripAuthIfCi(discoveredKibanaUrlWithAuth)}`); return discoveredKibanaUrlWithAuth.replace(/\/$/, ''); } catch (error) { @@ -183,7 +183,7 @@ function logCertificateWarningsIfNeeded(parsedTarget: Url, parsedKibanaUrl: Url, (parsedTarget.protocol === 'https:' || parsedKibanaUrl.protocol === 'https:') && (parsedTarget.hostname === '127.0.0.1' || parsedKibanaUrl.hostname === '127.0.0.1') ) { - logger.warn( + logger.warning( `WARNING: Self-signed certificate may not work with hostname: '127.0.0.1'. Consider using 'localhost' instead.` ); } @@ -193,7 +193,7 @@ export async function getServiceUrls({ logger, target, kibana }: RunOptions & { if (!target) { if (!kibana) { kibana = 'http://localhost:5601'; - logger.info(`No target provided, defaulting Kibana to ${kibana}`); + logger.debug(`No target provided, defaulting Kibana to ${kibana}`); } target = await discoverTargetFromKibanaUrl(kibana); } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts deleted file mode 100644 index bb78980f9ad2..000000000000 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { Client, HttpConnection } from '@elastic/elasticsearch'; -import { Logger } from '../../lib/utils/create_logger'; -import { RunOptions } from './parse_run_cli_flags'; -import { getEsClientTlsSettings } from './ssl'; -import { SyntheticsSynthtraceEsClient } from '../../lib/synthetics/synthetics_synthtrace_es_client'; - -export function getSyntheticsEsClient({ - target, - logger, - concurrency, -}: Pick & { - target: string; - logger: Logger; -}) { - const client = new Client({ - node: target, - tls: getEsClientTlsSettings(target), - Connection: HttpConnection, - requestTimeout: 30_000, - }); - - return new SyntheticsSynthtraceEsClient({ - client, - logger, - concurrency, - }); -} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/index_historical_data.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/index_historical_data.ts new file mode 100644 index 000000000000..efd4147f335c --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/index_historical_data.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { castArray } from 'lodash'; +import { memoryUsage } from 'process'; +import { timerange } from '@kbn/apm-synthtrace-client'; +import { Logger } from '../../lib/utils/create_logger'; +import { SynthtraceClients } from './get_clients'; +import { getScenario } from './get_scenario'; +import { WorkerData } from './synthtrace_worker'; +import { StreamManager } from './stream_manager'; + +export async function indexHistoricalData({ + bucketFrom, + bucketTo, + runOptions, + workerId, + logger, + clients, + from, + to, + streamManager, +}: WorkerData & { logger: Logger; clients: SynthtraceClients; streamManager: StreamManager }) { + const file = runOptions.file; + + const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger })); + + logger.info( + `Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()} (pid: ${ + process.pid + })` + ); + + const { generate } = await scenario({ ...runOptions, logger, from, to }); + + logger.debug('Generating scenario'); + + const generatorsAndClients = castArray( + logger.perf('generate_scenario', () => + generate({ + range: timerange(bucketFrom, bucketTo, logger), + clients, + }) + ) + ); + + logger.debug('Indexing scenario'); + + function mb(value: number): string { + return Math.round(value / 1024 ** 2).toString() + 'mb'; + } + + let cpuUsage = process.cpuUsage(); + + const intervalId = setInterval(async () => { + cpuUsage = process.cpuUsage(cpuUsage); + const mem = memoryUsage(); + logger.debug( + `cpu time: (user: ${Math.round(cpuUsage.user / 1000)}mss, sys: ${Math.round( + cpuUsage.system / 1000 + )}ms), memory: ${mb(mem.heapUsed)}/${mb(mem.heapTotal)}` + ); + }, 5000); + + await logger.perf('index_scenario', async () => { + await Promise.all( + generatorsAndClients.map(async ({ client, generator }) => { + await streamManager.index(client, generator); + }) + ).finally(() => { + clearInterval(intervalId); + }); + }); +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/logger_proxy.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/logger_proxy.ts index f51cc48237af..fd785374b02f 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/logger_proxy.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/logger_proxy.ts @@ -7,38 +7,17 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import util from 'util'; import { parentPort, isMainThread, workerData } from 'worker_threads'; import { createLogger, Logger, LogLevel } from '../../lib/utils/create_logger'; -import { logPerf } from '../../lib/utils/log_perf'; import { WorkerData } from './synthtrace_worker'; const { workerId } = isMainThread ? { workerId: -1 } : (workerData as WorkerData); - -function getLogMethod(log: LogLevel) { - return (...args: any) => { - parentPort?.postMessage({ - log, - args: [`[${workerId}]`].concat( - args.map((arg: any) => - typeof arg === 'string' || typeof arg === 'number' - ? arg - : util.inspect(arg, { depth: 10 }) - ) - ), - }); - }; -} - // logging proxy to main thread, ensures we see real time logging export const loggerProxy: Logger = isMainThread - ? createLogger(LogLevel.trace) - : { - perf: (name: string, cb: () => T): T => { - return logPerf(loggerProxy, LogLevel.trace, name, cb); + ? createLogger(LogLevel.verbose) + : createLogger(LogLevel.verbose, { + write: (msg) => { + parentPort?.postMessage([msg.type, [`[${workerId}]`, msg.args[0]].join(' ')]); + return true; }, - debug: getLogMethod(LogLevel.debug), - info: getLogMethod(LogLevel.info), - warn: getLogMethod(LogLevel.warn), - error: getLogMethod(LogLevel.error), - }; + }); diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/parse_run_cli_flags.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/parse_run_cli_flags.ts index 2b08144da613..4a66ca90c6ad 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/parse_run_cli_flags.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/parse_run_cli_flags.ts @@ -29,26 +29,25 @@ function getParsedFile(flags: RunCliFlags) { path.resolve(__dirname, '../../scenarios', `${parsedFile}.js`), ].find((p) => existsSync(p)); - if (filepath) { - // eslint-disable-next-line no-console - console.log(`Loading scenario from ${filepath}`); - return filepath; + if (!filepath) { + throw new Error(`Could not find scenario file: "${parsedFile}"`); } - throw new Error(`Could not find scenario file: "${parsedFile}"`); + return filepath; } export function parseRunCliFlags(flags: RunCliFlags) { - const { logLevel, target } = flags; + const { logLevel, target, debug, verbose } = flags; if (target?.includes('.kb.')) { throw new Error(`Target URL seems to be a Kibana URL, please provide Elasticsearch URL`); } const parsedFile = getParsedFile(flags); - let parsedLogLevel = LogLevel.info; + let parsedLogLevel = verbose ? LogLevel.verbose : debug ? LogLevel.debug : LogLevel.info; + switch (logLevel) { - case 'trace': - parsedLogLevel = LogLevel.trace; + case 'verbose': + parsedLogLevel = LogLevel.verbose; break; case 'info': @@ -67,13 +66,11 @@ export function parseRunCliFlags(flags: RunCliFlags) { parsedLogLevel = LogLevel.error; break; } - return { ...pick( flags, 'target', 'workers', - 'scenarioOpts', 'kibana', 'concurrency', 'versionOverride', @@ -81,6 +78,7 @@ export function parseRunCliFlags(flags: RunCliFlags) { 'assume-package-version', 'liveBucketSize' ), + scenarioOpts: flags.scenarioOpts as unknown as Record, logLevel: parsedLogLevel, file: parsedFile, }; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts index e7b3b96b5fa4..aabd357eed6b 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { range } from 'lodash'; +import { once, range } from 'lodash'; import moment from 'moment'; import { cpus } from 'os'; import Path from 'path'; @@ -16,6 +16,9 @@ import { LogLevel } from '../../..'; import { bootstrap } from './bootstrap'; import { RunOptions } from './parse_run_cli_flags'; import { WorkerData } from './synthtrace_worker'; +import { getScenario } from './get_scenario'; +import { StreamManager } from './stream_manager'; +import { indexHistoricalData } from './index_historical_data'; export async function startHistoricalDataUpload({ runOptions, @@ -23,23 +26,45 @@ export async function startHistoricalDataUpload({ to, }: { runOptions: RunOptions; - from: Date; - to: Date; + from: number; + to: number; }) { - const { logger, esUrl, version, kibanaUrl } = await bootstrap(runOptions); + const { logger, clients } = await bootstrap(runOptions); + + const file = runOptions.file; + + const scenario = await logger.perf('get_scenario', async () => { + const fn = await getScenario({ file, logger }); + return fn({ + ...runOptions, + logger, + from, + to, + }); + }); + + const teardown = once(async () => { + if (scenario.teardown) { + await scenario.teardown(clients); + } + }); + + const streamManager = new StreamManager(logger, teardown); + + if (scenario.bootstrap) { + await scenario.bootstrap(clients); + } const cores = cpus().length; let workers = Math.min(runOptions.workers ?? 10, cores - 1); - const rangeEnd = to; - - const diff = moment(from).diff(rangeEnd); + const diff = moment(from).diff(to); const d = moment.duration(Math.abs(diff), 'ms'); - // make sure ranges cover at least 100k documents - const minIntervalSpan = moment.duration(60, 'm'); + // make sure ranges cover at least 1m + const minIntervalSpan = moment.duration(1, 'm'); const minNumberOfRanges = d.asMilliseconds() / minIntervalSpan.asMilliseconds(); if (minNumberOfRanges < workers) { @@ -52,15 +77,12 @@ export async function startHistoricalDataUpload({ logger.info(`updating maxWorkers to ${workers} to ensure each worker does enough work`); } - logger.info(`Generating data from ${from.toISOString()} to ${rangeEnd.toISOString()}`); - - interface WorkerMessages { - log: LogLevel; - args: any[]; - } + logger.info( + `Generating data from ${new Date(from).toISOString()} to ${new Date(to).toISOString()}` + ); function rangeStep(interval: number) { - if (from > rangeEnd) return moment(from).subtract(interval, 'ms').toDate(); + if (from > to) return moment(from).subtract(interval, 'ms').toDate(); return moment(from).add(interval, 'ms').toDate(); } @@ -91,32 +113,34 @@ export async function startHistoricalDataUpload({ bucketFrom, bucketTo, workerId: workerIndex.toString(), - esUrl, - version, - kibanaUrl, + from, + to, }; const worker = new Worker(Path.join(__dirname, './worker.js'), { workerData, }); - worker.on('message', (message: WorkerMessages) => { - switch (message.log) { + + streamManager.trackWorker(worker); + + worker.on('message', ([logLevel, msg]: [string, string]) => { + switch (logLevel) { case LogLevel.debug: - logger.debug.apply({}, message.args); + logger.debug(msg); return; case LogLevel.info: - logger.info.apply({}, message.args); + logger.info(msg); return; - case LogLevel.trace: - logger.debug.apply({}, message.args); + case LogLevel.verbose: + logger.verbose(msg); return; case LogLevel.warn: - logger.warn.apply({}, message.args); + logger.warning(msg); return; case LogLevel.error: - logger.error.apply({}, message.args); + logger.error(msg); return; default: - logger.info(message); + logger.info(msg); } }); worker.on('error', (message) => { @@ -134,7 +158,27 @@ export async function startHistoricalDataUpload({ }); } - const workerServices = range(0, intervals.length).map((index) => runService(intervals[index])); + const workerServices = + intervals.length === 1 + ? // just run in this process. it's hard to attach + // a debugger to a worker_thread, see: + // https://issues.chromium.org/issues/41461728 + [ + indexHistoricalData({ + bucketFrom: intervals[0].bucketFrom, + bucketTo: intervals[0].bucketTo, + clients, + logger, + runOptions, + workerId: 'i', + from, + to, + streamManager, + }), + ] + : range(0, intervals.length).map((index) => runService(intervals[index])); - return Promise.all(workerServices); + await Promise.race(workerServices); + + await teardown(); } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts index 7166bef523f4..eb10a1e5842f 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts @@ -8,156 +8,86 @@ */ import { timerange } from '@kbn/apm-synthtrace-client'; -import { castArray } from 'lodash'; -import { PassThrough, Readable, Writable } from 'stream'; -import { isGeneratorObject } from 'util/types'; -import { SynthtraceEsClient } from '../../lib/shared/base_client'; -import { awaitStream } from '../../lib/utils/wait_until_stream_finished'; +import { castArray, once } from 'lodash'; import { bootstrap } from './bootstrap'; import { getScenario } from './get_scenario'; import { RunOptions } from './parse_run_cli_flags'; +import { StreamManager } from './stream_manager'; export async function startLiveDataUpload({ runOptions, - start, + from, + to, }: { runOptions: RunOptions; - start: Date; + from: number; + to: number; }) { const file = runOptions.file; - const { - logger, - apmEsClient, - logsEsClient, - infraEsClient, - syntheticsEsClient, - otelEsClient, - entitiesEsClient, - entitiesKibanaClient, - } = await bootstrap(runOptions); + const { logger, clients } = await bootstrap(runOptions); const scenario = await getScenario({ file, logger }); const { generate, - bootstrap: scenarioBootsrap, + bootstrap: scenarioBootstrap, teardown: scenarioTearDown, - } = await scenario({ ...runOptions, logger }); + } = await scenario({ ...runOptions, logger, from, to }); - if (scenarioBootsrap) { - await scenarioBootsrap({ - apmEsClient, - logsEsClient, - infraEsClient, - otelEsClient, - syntheticsEsClient, - entitiesEsClient, - entitiesKibanaClient, - }); + const teardown = once(async () => { + if (scenarioTearDown) { + await scenarioTearDown(clients); + } + }); + + const streamManager = new StreamManager(logger, teardown); + + if (scenarioBootstrap) { + await scenarioBootstrap(clients); } const bucketSizeInMs = runOptions.liveBucketSize; - let requestedUntil = start; - - let currentStreams: PassThrough[] = []; - // @ts-expect-error upgrade typescript v4.9.5 - const cachedStreams: WeakMap = new WeakMap(); - - process.on('SIGINT', () => closeStreamsAndTeardown()); - process.on('SIGTERM', () => closeStreamsAndTeardown()); - process.on('SIGQUIT', () => closeStreamsAndTeardown()); - - async function closeStreamsAndTeardown() { - if (scenarioTearDown) { - try { - await scenarioTearDown({ - apmEsClient, - logsEsClient, - infraEsClient, - otelEsClient, - syntheticsEsClient, - entitiesEsClient, - entitiesKibanaClient, - }); - } catch (error) { - logger.error('Error during scenario teardown', error); - } - } - - currentStreams.forEach((stream) => { - stream.end(() => { - process.exit(0); - }); - }); - currentStreams = []; // Reset the stream array - } + let requestedUntil = from; async function uploadNextBatch() { const now = Date.now(); - if (now > requestedUntil.getTime()) { - const bucketFrom = requestedUntil; - const bucketTo = new Date(requestedUntil.getTime() + bucketSizeInMs); + if (now > requestedUntil) { + const bucketCount = Math.floor((now - requestedUntil) / bucketSizeInMs); + + const rangeStart = requestedUntil; + const rangeEnd = rangeStart + bucketCount * bucketSizeInMs; logger.info( - `Requesting ${new Date(bucketFrom).toISOString()} to ${new Date(bucketTo).toISOString()}` + `Requesting ${new Date(rangeStart).toISOString()} to ${new Date( + rangeEnd + ).toISOString()} in ${bucketCount} bucket(s)` ); - const generatorsAndClients = generate({ - range: timerange(bucketFrom.getTime(), bucketTo.getTime()), - clients: { - logsEsClient, - apmEsClient, - infraEsClient, - entitiesEsClient, - syntheticsEsClient, - otelEsClient, - }, - }); + const generatorsAndClients = castArray( + generate({ + range: timerange(rangeStart, rangeEnd, logger), + clients, + }) + ); - const generatorsAndClientsArray = castArray(generatorsAndClients); + await Promise.all( + generatorsAndClients.map(async ({ generator, client }) => { + await streamManager.index(client, generator); + }) + ); - const streams = generatorsAndClientsArray.map(({ client }) => { - let stream: PassThrough; + logger.debug('Indexing completed'); - if (cachedStreams.has(client)) { - stream = cachedStreams.get(client)!; - } else { - stream = new PassThrough({ objectMode: true }); - cachedStreams.set(client, stream); - client.index(stream); - } - - return stream; - }); - - currentStreams = streams; - - const promises = generatorsAndClientsArray.map(({ generator }, i) => { - const concatenatedStream = castArray(generator) - .reverse() - .reduce((prev, current) => { - const currentStream = isGeneratorObject(current) ? Readable.from(current) : current; - return currentStream.pipe(prev); - }, new PassThrough({ objectMode: true })); - - concatenatedStream.pipe(streams[i], { end: false }); - - return awaitStream(concatenatedStream); - }); - - await Promise.all(promises); - - logger.info('Indexing completed'); - - const refreshPromise = generatorsAndClientsArray.map(async ({ client }) => { + const refreshPromise = generatorsAndClients.map(async ({ client }) => { await client.refresh(); }); await Promise.all(refreshPromise); - logger.info('Refreshing completed'); - requestedUntil = bucketTo; + logger.debug('Refreshing completed'); + + requestedUntil = rangeEnd; } } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts new file mode 100644 index 000000000000..4032ebe2b0d2 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { castArray, once, pull } from 'lodash'; +import { ToolingLog } from '@kbn/tooling-log'; +import { PassThrough, Readable, Writable, finished } from 'stream'; +import { Fields } from '@kbn/apm-synthtrace-client'; +import { isGeneratorObject } from 'util/types'; +import { Worker, parentPort } from 'worker_threads'; +import { SynthtraceEsClient } from '../../lib/shared/base_client'; +import { SynthGenerator } from '../../lib/utils/with_client'; +import { awaitStream } from '../../lib/utils/wait_until_stream_finished'; + +// execute a callback when one of the kill signals is received +function attach(logger: ToolingLog, cb: () => Promise) { + const disconnect = () => { + process.off('SIGINT', wrapped); + process.off('SIGTERM', wrapped); + process.off('SIGQUIT', wrapped); + }; + + const wrapped = once(() => { + disconnect(); + cb() + .then(() => { + process.exit(0); + }) + .catch((err) => { + logger.error(err); + process.exit(1); + }); + }); + + process.on('SIGINT', wrapped); + process.on('SIGTERM', wrapped); + process.on('SIGQUIT', wrapped); +} +/** + * StreamManager waits until streams have completed, + * and then calls a teardown callback. + */ + +const asyncNoop = async () => {}; + +export class StreamManager { + private readonly clientStreams: Map, PassThrough> = new Map(); + private readonly trackedStreams: Writable[] = []; + private readonly trackedWorkers: Worker[] = []; + + constructor( + private readonly logger: ToolingLog, + private readonly teardownCallback: () => Promise = asyncNoop + ) { + attach(this.logger, () => this.teardown()); + + parentPort?.on('message', (message) => { + if (message === 'shutdown') { + this.teardown() + .then(() => { + process.exit(0); + }) + .catch(() => { + process.exit(1); + }); + } + }); + } + + trackWorker(worker: Worker) { + const untrack = () => { + pull(this.trackedWorkers, worker); + }; + worker.on('error', () => { + untrack(); + }); + worker.on('exit', () => { + untrack(); + }); + this.trackedWorkers.push(worker); + } + + /** + * Create a single stream per client, and index data + * received from the generator into that stream. + */ + async index( + client: SynthtraceEsClient, + generator: SynthGenerator + ): Promise { + const clientStream = this.createOrReuseClientStream(client); + + const generatorStream = castArray(generator) + .reverse() + .reduce((prev, current) => { + const currentStream = isGeneratorObject(current) ? Readable.from(current) : current; + return currentStream.pipe(prev); + }, new PassThrough({ objectMode: true })); + + // the generator stream should write to the client + // stream, but not end it, as the next buckets will + // create a new generator + generatorStream.pipe(clientStream, { end: false }); + + // track the stream for later to end it if needed + this.trackedStreams.push(generatorStream); + + await awaitStream(generatorStream).finally(() => { + pull(this.trackedStreams, generatorStream); + }); + } + + private createOrReuseClientStream(client: SynthtraceEsClient) { + let stream: PassThrough; + + if (this.clientStreams.has(client)) { + stream = this.clientStreams.get(client)!; + } else { + stream = new PassThrough({ objectMode: true }); + this.clientStreams.set(client, stream); + client.index(stream); + } + return stream; + } + + teardown = once(async () => { + // If a signal is received during teardown, + // we just quit forcefully. + attach(this.logger, async () => { + this.logger.error(`Force-quitting after receiving kill signal`); + process.exit(1); + }); + + this.logger.info('Tearing down after kill signal'); + + // end all streams and listen until they've + // completed + function endStream(stream: Writable) { + return new Promise((resolve, reject) => { + stream.end(); + finished(stream, (err) => { + if (err) { + return reject(err); + } + resolve(); + }); + }); + } + + if (this.trackedStreams.length) { + // ending generator streams + this.logger.debug(`Ending ${this.trackedStreams.length} tracked streams`); + + await Promise.all(this.trackedStreams.map(endStream)); + } + + if (this.trackedWorkers.length) { + // give workers a chance to gracefully shut down + this.logger.debug(`Shutting down ${this.trackedWorkers.length} workers`); + + await Promise.all( + this.trackedWorkers.map((worker) => { + return new Promise((resolve, reject) => { + worker.postMessage('shutdown'); + worker.on('exit', () => { + resolve(); + }); + setTimeout(() => { + reject(`Failed to gracefully shutdown worker in time, terminating`); + }, 10000); + }); + }) + ); + } + + const clientStreams = Array.from(this.clientStreams.values()); + + if (clientStreams.length) { + // ending client streams + this.logger.debug(`Ending ${clientStreams.length} client streams`); + + await Promise.all(clientStreams.map(endStream)); + } + + await this.teardownCallback(); + }); +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index 4a4d0af10e47..1dc0d56954f0 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -7,153 +7,46 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { timerange } from '@kbn/apm-synthtrace-client'; -import { castArray } from 'lodash'; -import { memoryUsage } from 'process'; import { parentPort, workerData } from 'worker_threads'; -import { getApmEsClient } from './get_apm_es_client'; -import { getEntitiesKibanaClient } from './get_entites_kibana_client'; -import { getEntitiesEsClient } from './get_entities_es_client'; -import { getInfraEsClient } from './get_infra_es_client'; -import { getLogsEsClient } from './get_logs_es_client'; -import { getOtelSynthtraceEsClient } from './get_otel_es_client'; -import { getScenario } from './get_scenario'; -import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { bootstrap } from './bootstrap'; +import { indexHistoricalData } from './index_historical_data'; import { loggerProxy } from './logger_proxy'; import { RunOptions } from './parse_run_cli_flags'; +import { StreamManager } from './stream_manager'; export interface WorkerData { + from: number; + to: number; bucketFrom: Date; bucketTo: Date; runOptions: RunOptions; workerId: string; - esUrl: string; - version: string; - kibanaUrl: string; } -const { bucketFrom, bucketTo, runOptions, esUrl, version, kibanaUrl } = workerData as WorkerData; +const { bucketFrom, bucketTo, runOptions, workerId, from, to } = workerData as WorkerData; async function start() { const logger = loggerProxy; - const entitiesEsClient = getEntitiesEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, + + const streamManager = new StreamManager(logger); + + const { clients } = await bootstrap({ + ...runOptions, + skipClientBootstrap: true, + clean: false, + }); + + await indexHistoricalData({ + bucketFrom, + bucketTo, + clients, logger, + runOptions, + workerId, + from, + to, + streamManager, }); - - const entitiesKibanaClient = getEntitiesKibanaClient({ - target: kibanaUrl, - logger, - }); - - const apmEsClient = getApmEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, - logger, - version, - }); - - const logsEsClient = getLogsEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, - logger, - }); - - const infraEsClient = getInfraEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, - logger, - }); - - const syntheticsEsClient = getSyntheticsEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, - logger, - }); - - const otelEsClient = getOtelSynthtraceEsClient({ - concurrency: runOptions.concurrency, - target: esUrl, - logger, - }); - - const file = runOptions.file; - - const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger })); - - logger.info(`Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()}`); - - const { generate, bootstrap, teardown } = await scenario({ ...runOptions, logger }); - - if (bootstrap) { - await bootstrap({ - apmEsClient, - logsEsClient, - infraEsClient, - syntheticsEsClient, - otelEsClient, - entitiesEsClient, - entitiesKibanaClient, - }); - } - - logger.debug('Generating scenario'); - - const generatorsAndClients = logger.perf('generate_scenario', () => - generate({ - range: timerange(bucketFrom, bucketTo), - clients: { - logsEsClient, - apmEsClient, - infraEsClient, - entitiesEsClient, - syntheticsEsClient, - otelEsClient, - }, - }) - ); - - const generatorsAndClientsArray = castArray(generatorsAndClients); - - logger.debug('Indexing scenario'); - - function mb(value: number): string { - return Math.round(value / 1024 ** 2).toString() + 'mb'; - } - - let cpuUsage = process.cpuUsage(); - - setInterval(async () => { - cpuUsage = process.cpuUsage(cpuUsage); - const mem = memoryUsage(); - logger.info( - `cpu time: (user: ${cpuUsage.user}µs, sys: ${cpuUsage.system}µs), memory: ${mb( - mem.heapUsed - )}/${mb(mem.heapTotal)}` - ); - }, 5000); - - await logger.perf('index_scenario', async () => { - const promises = generatorsAndClientsArray.map(async ({ client, generator }) => { - await client.index(generator); - await client.refresh(); - }); - - await Promise.all(promises); - }); - - if (teardown) { - await teardown({ - apmEsClient, - logsEsClient, - infraEsClient, - syntheticsEsClient, - otelEsClient, - entitiesEsClient, - entitiesKibanaClient, - }); - } } parentPort!.on('message', (message) => { diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts index 39277d7d8882..9771cc3324d1 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts @@ -29,7 +29,7 @@ export interface ApmSynthtraceEsClientOptions extends Omit { - private version: string; + public readonly version: string; constructor(options: { client: Client; logger: Logger } & ApmSynthtraceEsClientOptions) { super({ diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts index d7f3bb3ce3cf..e086d54373c0 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts @@ -7,25 +7,22 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import fetch from 'node-fetch'; import pRetry from 'p-retry'; +import { KibanaClient, KibanaClientHttpError } from '../../shared/base_kibana_client'; import { Logger } from '../../utils/create_logger'; -import { kibanaHeaders } from '../../shared/client_headers'; -import { getFetchAgent } from '../../../cli/utils/ssl'; +import { getKibanaClient } from '../../../cli/utils/get_kibana_client'; export class ApmSynthtraceKibanaClient { + private readonly kibanaClient: KibanaClient; private readonly logger: Logger; - private target: string; - private headers: Record; - constructor(options: { logger: Logger; target: string; headers?: Record }) { + constructor(options: { logger: Logger } & ({ target: string } | { kibanaClient: KibanaClient })) { + this.kibanaClient = 'kibanaClient' in options ? options.kibanaClient : getKibanaClient(options); this.logger = options.logger; - this.target = options.target; - this.headers = { ...kibanaHeaders(), ...(options.headers ?? {}) }; } getFleetApmPackagePath(packageVersion?: string): string { - let path = `${this.target}/api/fleet/epm/packages/apm`; + let path = `/api/fleet/epm/packages/apm`; if (packageVersion) { path = `${path}/${packageVersion}`; } @@ -39,26 +36,21 @@ export class ApmSynthtraceKibanaClient { const url = `${this.getFleetApmPackagePath()}?prerelease=${prerelease}`; this.logger.debug(`Fetching from URL: ${url}`); - const response = await fetch(url, { - method: 'GET', - headers: this.headers, - agent: getFetchAgent(url), - }); + const response = await this.kibanaClient + .fetch<{ item: { latestVersion?: string } }>(url, { + method: 'GET', + }) + .catch((error) => { + const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0; + throw new Error( + `Failed to fetch APM package version, received HTTP ${statusCode} and message: ${error.message}` + ); + }); - const responseJson = await response.json(); - - if (response.status !== 200) { - throw new Error( - `Failed to fetch APM package version, received HTTP ${response.status} and message: ${responseJson.message}` - ); + if (!response.item.latestVersion) { + throw new Error(`Failed to fetch APM package version`); } - - // Add support for 7.x stack as latest version is available under different node - if (responseJson.response && responseJson.response.latestVersion) { - return responseJson.response.latestVersion as string; - } - - return responseJson.item.latestVersion as string; + return response.item.latestVersion; }; try { @@ -67,9 +59,7 @@ export class ApmSynthtraceKibanaClient { this.logger.debug( 'Fetching latestes prerelease version failed, retrying with latest GA version' ); - const retryResult = await fetchPackageVersion({ prerelease: false }).catch((retryError) => { - throw retryError; - }); + const retryResult = await fetchPackageVersion({ prerelease: false }); return retryResult; } @@ -84,23 +74,18 @@ export class ApmSynthtraceKibanaClient { const url = this.getFleetApmPackagePath(packageVersion); const response = await pRetry( async () => { - const res = await fetch(url, { - method: 'POST', - headers: this.headers, - body: '{"force":true}', - agent: getFetchAgent(url), - }); + const res = await this.kibanaClient + .fetch<{ items: unknown[] }>(url, { + method: 'POST', + body: '{"force":true}', + }) + .catch((error) => { + const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0; + throw new Error( + `APM package installation returned ${statusCode} status code\nError: ${error.message}` + ); + }); - if (!res.ok) { - const errorJson = await res.json(); - const errorMessage = - typeof errorJson.message === 'string' - ? errorJson.message - : 'An error occurred during APM package installation.'; - throw new Error( - `APM package installation returned ${res.status} status code\nError: ${errorMessage}` - ); - } return res; }, { @@ -113,12 +98,8 @@ export class ApmSynthtraceKibanaClient { } ); - const responseJson = await response.json(); - - if (!responseJson.items) { - throw new Error( - `No installed assets received for APM package version ${packageVersion}, received HTTP ${response.status} for url ${url}` - ); + if (!response.items) { + throw new Error(`No installed assets received for APM package version ${packageVersion}`); } this.logger.info(`Installed APM package ${packageVersion}`); @@ -132,23 +113,18 @@ export class ApmSynthtraceKibanaClient { const url = this.getFleetApmPackagePath(latestApmPackageVersion); const response = await pRetry( async () => { - const res = await fetch(url, { - method: 'DELETE', - headers: this.headers, - body: '{"force":true}', - agent: getFetchAgent(url), - }); + const res = await this.kibanaClient + .fetch<{ items: unknown[] }>(url, { + method: 'DELETE', + body: '{"force":true}', + }) + .catch((error) => { + const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0; + throw new Error( + `APM package uninstallation returned ${statusCode} status code\nError: ${error.message}` + ); + }); - if (!res.ok) { - const errorJson = await res.json(); - const errorMessage = - typeof errorJson.message === 'string' - ? errorJson.message - : 'An error occurred during APM package uninstallation.'; - throw new Error( - `APM package uninstallation returned ${res.status} status code\nError: ${errorMessage}` - ); - } return res; }, { @@ -163,11 +139,9 @@ export class ApmSynthtraceKibanaClient { } ); - const responseJson = await response.json(); - - if (!responseJson.items) { + if (!response.items) { throw new Error( - `No uninstalled assets received for APM package version ${latestApmPackageVersion}, received HTTP ${response.status} for url ${url}` + `No uninstalled assets received for APM package version ${latestApmPackageVersion}` ); } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts index cd683f2831be..152d43ed4470 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts @@ -7,10 +7,9 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import fetch from 'node-fetch'; import { Logger } from '../utils/create_logger'; -import { kibanaHeaders } from '../shared/client_headers'; -import { getFetchAgent } from '../../cli/utils/ssl'; +import { KibanaClient } from '../shared/base_kibana_client'; +import { getKibanaClient } from '../../cli/utils/get_kibana_client'; interface EntityDefinitionResponse { definitions: Array<{ type: string; state: { installed: boolean; running: boolean } }>; @@ -18,21 +17,20 @@ interface EntityDefinitionResponse { export class EntitiesSynthtraceKibanaClient { private readonly logger: Logger; - private target: string; + private readonly kibana: KibanaClient; - constructor(options: { logger: Logger; target: string }) { + constructor(options: { logger: Logger } & ({ target: string } | { kibanaClient: KibanaClient })) { + this.kibana = 'kibanaClient' in options ? options.kibanaClient : getKibanaClient(options); this.logger = options.logger; - this.target = options.target; } async installEntityIndexPatterns() { - const url = `${this.target}/internal/entities/definition?includeState=true`; - const response = await fetch(url, { - method: 'GET', - headers: kibanaHeaders(), - agent: getFetchAgent(url), - }); - const entityDefinition: EntityDefinitionResponse = await response.json(); + const entityDefinition = await this.kibana.fetch( + `/internal/entities/definition?includeState=true`, + { + method: 'GET', + } + ); const hasEntityDefinitionsInstalled = entityDefinition.definitions?.find( (definition) => definition.type === 'service' @@ -42,21 +40,15 @@ export class EntitiesSynthtraceKibanaClient { this.logger.debug('Entity definitions are already defined'); } else { this.logger.debug('Installing Entity definitions'); - const entityEnablementUrl = `${this.target}/internal/entities/managed/enablement?installOnly=true`; - await fetch(entityEnablementUrl, { + await this.kibana.fetch(`/internal/entities/managed/enablement?installOnly=true`, { method: 'PUT', - headers: kibanaHeaders(), - agent: getFetchAgent(url), }); } } async uninstallEntityIndexPatterns() { - const url = `${this.target}/internal/entities/managed/enablement`; - await fetch(url, { + await this.kibana.fetch(`/internal/entities/managed/enablement`, { method: 'DELETE', - headers: kibanaHeaders(), - agent: getFetchAgent(url), }); } } diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_client.ts index 7fd639331af8..4a2871055421 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_client.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_client.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { Client } from '@elastic/elasticsearch'; +import { Client, errors } from '@elastic/elasticsearch'; import { ESDocumentWithOperation, Fields, @@ -19,8 +19,12 @@ import { Readable, Transform } from 'stream'; import { isGeneratorObject } from 'util/types'; import { Logger } from '../utils/create_logger'; import { sequential } from '../utils/stream_utils'; +import { KibanaClient } from './base_kibana_client'; export interface SynthtraceEsClientOptions { + client: Client; + kibana?: KibanaClient; + logger: Logger; concurrency?: number; refreshAfterIndex?: boolean; pipeline: (base: Readable) => NodeJS.WritableStream; @@ -30,6 +34,7 @@ type MaybeArray = T | T[]; export class SynthtraceEsClient { protected readonly client: Client; + protected readonly kibana?: KibanaClient; protected readonly logger: Logger; private readonly concurrency: number; @@ -39,8 +44,9 @@ export class SynthtraceEsClient { protected dataStreams: string[] = []; protected indices: string[] = []; - constructor(options: { client: Client; logger: Logger } & SynthtraceEsClientOptions) { + constructor(options: SynthtraceEsClientOptions) { this.client = options.client; + this.kibana = options.kibana; this.logger = options.logger; this.concurrency = options.concurrency ?? 1; this.refreshAfterIndex = options.refreshAfterIndex ?? false; @@ -67,10 +73,17 @@ export class SynthtraceEsClient { await Promise.all([ ...(this.dataStreams.length ? [ - this.client.indices.deleteDataStream({ - name: this.dataStreams.join(','), - expand_wildcards: ['open', 'hidden'], - }), + this.client.indices + .deleteDataStream({ + name: this.dataStreams.join(','), + expand_wildcards: ['open', 'hidden'], + }) + .catch((error) => { + if (error instanceof errors.ResponseError && error.statusCode === 404) { + return; + } + throw error; + }), ] : []), ...(resolvedIndices.length @@ -105,7 +118,7 @@ export class SynthtraceEsClient { async index( streamOrGenerator: MaybeArray>, pipelineCallback?: (base: Readable) => NodeJS.WritableStream - ) { + ): Promise { this.logger.debug(`Bulk indexing ${castArray(streamOrGenerator).length} stream(s)`); const previousPipelineCallback = this.pipelineCallback; @@ -135,9 +148,9 @@ export class SynthtraceEsClient { count++; if (count % 100000 === 0) { - this.logger.info(`Indexed ${count} documents`); - } else if (count % 1000 === 0) { this.logger.debug(`Indexed ${count} documents`); + } else if (count % 1000 === 0) { + this.logger.verbose(`Indexed ${count} documents`); } if (doc._action) { diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_kibana_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_kibana_client.ts new file mode 100644 index 000000000000..e4d11b07bf76 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_kibana_client.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/* eslint-disable max-classes-per-file*/ + +import fetch from 'node-fetch'; +import { RequestInit } from 'node-fetch'; +import Path from 'path'; +import { kibanaHeaders } from './client_headers'; +import { getFetchAgent } from '../../cli/utils/ssl'; + +type KibanaClientFetchOptions = RequestInit; + +export class KibanaClientHttpError extends Error { + constructor(message: string, public readonly statusCode: number, public readonly data?: unknown) { + super(message); + } +} + +export class KibanaClient { + private target: string; + private headers: Record; + + constructor(options: { target: string; headers?: Record }) { + this.target = options.target; + this.headers = { ...kibanaHeaders(), ...(options.headers ?? {}) }; + } + + fetch(pathname: string, options: KibanaClientFetchOptions): Promise { + const url = Path.join(this.target, pathname); + return fetch(url, { + ...options, + headers: { + ...this.headers, + ...options.headers, + }, + agent: getFetchAgent(url), + }).then(async (response) => { + if (response.status >= 400) { + throw new KibanaClientHttpError( + `Response error for ${options.method?.toUpperCase() ?? 'GET'} ${url}`, + response.status, + await response.json().catch((error) => { + return undefined; + }) + ); + } + return response.json(); + }); + } +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/client_headers.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/client_headers.ts index 3266582c240c..00f93d565fe0 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/client_headers.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/client_headers.ts @@ -15,3 +15,9 @@ export function kibanaHeaders() { 'elastic-api-version': '2023-10-31', }; } + +export function internalKibanaHeaders() { + return { + 'x-elastic-internal-origin': 'kibana', + }; +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/streams/streams_synthtrace_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/streams/streams_synthtrace_client.ts new file mode 100644 index 000000000000..9cb35285e292 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/streams/streams_synthtrace_client.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client'; +import { Condition, StreamUpsertRequest } from '@kbn/streams-schema'; +import { Readable, Transform, pipeline } from 'stream'; +import { Required } from 'utility-types'; +import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; +import { internalKibanaHeaders } from '../shared/client_headers'; +import { getSerializeTransform } from '../shared/get_serialize_transform'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +interface StreamsDocument {} + +export class StreamsSynthtraceClient extends SynthtraceEsClient { + constructor(options: Required, 'kibana'>) { + super({ + ...options, + pipeline: streamsPipeline(), + }); + this.dataStreams = ['logs', 'logs.*']; + } + + async forkStream( + streamName: string, + request: { stream: { name: string }; if: Condition } + ): Promise<{ acknowledged: true }> { + return this.kibana!.fetch(`/api/streams/${streamName}/_fork`, { + method: 'POST', + headers: { + ...internalKibanaHeaders(), + }, + body: JSON.stringify(request), + }); + } + + async putStream( + streamName: string, + request: StreamUpsertRequest + ): Promise<{ acknowledged: true; result: 'created' | 'updated' }> { + return this.kibana!.fetch(`/api/streams/${streamName}`, { + method: 'PUT', + headers: { + ...internalKibanaHeaders(), + }, + body: JSON.stringify(request), + }); + } + + async enable() { + await this.kibana!.fetch('/api/streams/_enable', { + method: 'POST', + headers: { + ...internalKibanaHeaders(), + }, + }); + } + + async disable() { + await this.kibana!.fetch('/api/streams/_disable', { + method: 'POST', + timeout: 5 * 60 * 1000, + headers: { + ...internalKibanaHeaders(), + }, + }); + } + + override async clean(): Promise { + await this.disable(); + await super.clean(); + } + + async clearESCache(): Promise { + await this.client.indices.clearCache(); + } +} + +function streamsRoutingTransform() { + return new Transform({ + objectMode: true, + transform(document: ESDocumentWithOperation, encoding, callback) { + document._index = 'logs'; + callback(null, document); + }, + }); +} + +function streamsPipeline() { + return (base: Readable) => { + return pipeline( + base, + getSerializeTransform(), + streamsRoutingTransform(), + (err: unknown) => { + if (err) { + throw err; + } + } + ); + }; +} diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/create_logger.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/create_logger.ts index 85536b53a093..9c065b37d50c 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/create_logger.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/create_logger.ts @@ -7,58 +7,41 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { ToolingLog, Writer } from '@kbn/tooling-log'; import { logPerf } from './log_perf'; export enum LogLevel { - trace = 0, - debug = 1, - info = 2, - warn = 3, - error = 4, + verbose = 'verbose', + debug = 'debug', + info = 'info', + warn = 'warning', + error = 'error', } -function getTimeString() { - return `[${new Date().toLocaleTimeString()}]`; -} - -export function createLogger(logLevel: LogLevel) { - const logger: Logger = { - perf: (name, callback) => { - return logPerf(logger, logLevel, name, callback); - }, - debug: (...args: any[]) => { - if (logLevel <= LogLevel.debug) { - // eslint-disable-next-line no-console - console.debug(getTimeString(), ...args); - } - }, - info: (...args: any[]) => { - if (logLevel <= LogLevel.info) { - // eslint-disable-next-line no-console - console.log(getTimeString(), ...args); - } - }, - warn: (...args: any[]) => { - if (logLevel <= LogLevel.warn) { - // eslint-disable-next-line no-console - console.warn(getTimeString(), ...args); - } - }, - error: (...args: any[]) => { - if (logLevel <= LogLevel.error) { - // eslint-disable-next-line no-console - console.log(getTimeString(), ...args); - } - }, +export function extendToolingLog(log: ToolingLog, logLevel: LogLevel = LogLevel.verbose): Logger { + const logger = log as Logger; + logger.perf = (name: string, callback: () => any) => { + return logPerf(log, LogLevel.verbose, name, callback); }; return logger; } -export interface Logger { - perf: (name: string, cb: () => T) => T; - debug: (...args: any[]) => void; - info: (...args: any[]) => void; - warn: (...args: any[]) => void; - error: (...args: any[]) => void; +export function createLogger(logLevel: LogLevel, writer?: Writer): Logger { + const log = new ToolingLog({ + level: logLevel, + writeTo: writer ? { write: () => true } : process.stdout, + }) as Logger; + + if (writer) { + log.setWriters([writer]); + } + + extendToolingLog(log, logLevel); + + return log; } + +export type Logger = ToolingLog & { + perf: (name: string, cb: () => T) => T; +}; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/log_perf.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/log_perf.ts index a88a23ef41a6..0414420a8239 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/log_perf.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/log_perf.ts @@ -7,23 +7,24 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { Logger, LogLevel } from './create_logger'; +import { ToolingLog } from '@kbn/tooling-log'; +import { LogLevel } from './create_logger'; function isPromise(val: any): val is Promise { return val && typeof val === 'object' && 'then' in val && typeof val.then === 'function'; } -function logTo(logger: Logger, name: string, start: bigint) { +function logTo(logger: ToolingLog, name: string, start: bigint) { logger.debug(`${name}: ${Number(process.hrtime.bigint() - start) / 1000000}ms`); } export const logPerf = ( - logger: Logger, + logger: ToolingLog, logLevel: LogLevel, name: string, cb: () => T ): T => { - if (logLevel <= LogLevel.trace) { + if (logLevel === LogLevel.verbose) { const start = process.hrtime.bigint(); const val = cb(); if (isPromise(val)) { diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/with_client.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/with_client.ts index b89b6c0fe3f2..2a63b42c664d 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/with_client.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/with_client.ts @@ -7,20 +7,17 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { SynthtraceGenerator } from '@kbn/apm-synthtrace-client'; +import { Fields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client'; import { Readable } from 'stream'; import { SynthtraceEsClient } from '../shared/base_client'; -export type SynthGenerator = - // @ts-expect-error upgrade typescript v4.9.5 +export type SynthGenerator = | SynthtraceGenerator - // @ts-expect-error upgrade typescript v4.9.5 | Array> | Readable; -export const withClient = ( - // @ts-expect-error upgrade typescript v4.9.5 - client: SynthtraceEsClient, +export const withClient = ( + client: SynthtraceEsClient, generator: SynthGenerator ) => { return { @@ -29,4 +26,4 @@ export const withClient = ( }; }; -export type ScenarioReturnType = ReturnType>; +export type ScenarioReturnType = ReturnType>; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/sample_logs.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/sample_logs.ts new file mode 100644 index 000000000000..c9eafa710425 --- /dev/null +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/sample_logs.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { LogDocument, Serializable } from '@kbn/apm-synthtrace-client'; +import { SampleParserClient } from '@kbn/sample-log-parser'; +import { Scenario } from '../cli/scenario'; +import { withClient } from '../lib/utils/with_client'; + +const scenario: Scenario = async (runOptions) => { + const client = new SampleParserClient({ logger: runOptions.logger }); + + const { rpm } = (runOptions.scenarioOpts ?? {}) as { rpm?: number }; + + const generators = await client.getLogGenerators({ + rpm, + }); + + return { + bootstrap: async ({ streamsClient }) => { + await streamsClient.enable(); + }, + generate: ({ range, clients: { streamsClient } }) => { + return withClient( + streamsClient, + range.interval('5s').generator((timestamp) => { + return generators.flatMap((generator) => + generator.next(timestamp).map((doc) => new Serializable(doc)) + ); + }) + ); + }, + }; +}; + +export default scenario; diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/traces_logs_entities.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/traces_logs_entities.ts index 2e860a525c60..471036f77695 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/traces_logs_entities.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/traces_logs_entities.ts @@ -14,7 +14,6 @@ import { Instance, log, entities, - EntityFields, } from '@kbn/apm-synthtrace-client'; import { Readable } from 'stream'; import { Scenario } from '../cli/scenario'; @@ -35,7 +34,7 @@ const SYNTH_JAVA_TRACE_ENTITY_ID = generateShortId(); const SYNTH_NODE_TRACES_LOGS_ENTITY_ID = generateShortId(); const SYNTH_GO_LOGS_ENTITY_ID = generateShortId(); -const scenario: Scenario> = async (runOptions) => { +const scenario: Scenario = async (runOptions) => { const { logger } = runOptions; const { isLogsDb } = parseLogsScenarioOpts(runOptions.scenarioOpts); diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/es_client_indexer.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/es_client_indexer.test.ts index 6bdd0453c5b9..df522a5f79b1 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/es_client_indexer.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/es_client_indexer.test.ts @@ -11,6 +11,7 @@ import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { pick, range, sum } from 'lodash'; import { Readable } from 'stream'; import { ApmSynthtraceEsClient } from '../lib/apm/client/apm_synthtrace_es_client'; +import { ToolingLog } from '@kbn/tooling-log'; describe('Synthtrace ES Client indexer', () => { let apmEsClient: ApmSynthtraceEsClient; @@ -58,7 +59,8 @@ describe('Synthtrace ES Client indexer', () => { const generator = timerange( new Date('2022-01-01T00:00:00.000Z'), - new Date('2022-01-01T01:00:01.000Z') + new Date('2022-01-01T01:00:01.000Z'), + {} as ToolingLog ) .interval('30m') .generator((timestamp) => { @@ -99,7 +101,11 @@ describe('Synthtrace ES Client indexer', () => { }); const getGenerator = () => - timerange(new Date('2022-01-01T00:00:00.000Z'), new Date('2022-01-01T00:59:59.999Z')) + timerange( + new Date('2022-01-01T00:00:00.000Z'), + new Date('2022-01-01T00:59:59.999Z'), + {} as ToolingLog + ) .interval('20m') .rate(10) .generator(generatorCallback); @@ -142,7 +148,8 @@ describe('Synthtrace ES Client indexer', () => { const generator = timerange( new Date('2022-01-01T00:00:00.000Z'), - new Date('2022-01-01T00:06:59.999Z') + new Date('2022-01-01T00:06:59.999Z'), + {} as ToolingLog ) .interval('1m') .rate(RATE) diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts index 692a96df1cad..3880b2e4fd0d 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts @@ -8,6 +8,7 @@ */ import { apm, ApmFields, SynthtraceGenerator, timerange } from '@kbn/apm-synthtrace-client'; +import { ToolingLog } from '@kbn/tooling-log'; describe('simple trace', () => { let iterable: SynthtraceGenerator; @@ -23,7 +24,8 @@ describe('simple trace', () => { const range = timerange( new Date('2021-01-01T00:00:00.000Z'), - new Date('2021-01-01T00:15:00.000Z') + new Date('2021-01-01T00:15:00.000Z'), + {} as ToolingLog ); iterable = range diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/02_transaction_metrics.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/02_transaction_metrics.test.ts index 1967dc19fe8b..46eed9438fe0 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/02_transaction_metrics.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/02_transaction_metrics.test.ts @@ -12,6 +12,7 @@ import { sortBy } from 'lodash'; import { Readable } from 'stream'; import { createTransactionMetricsAggregator } from '../../lib/apm/aggregators/create_transaction_metrics_aggregator'; import { awaitStream } from '../../lib/utils/wait_until_stream_finished'; +import { ToolingLog } from '@kbn/tooling-log'; describe('transaction metrics', () => { let events: Array>; @@ -26,7 +27,8 @@ describe('transaction metrics', () => { const range = timerange( new Date('2021-01-01T00:00:00.000Z'), - new Date('2021-01-01T00:15:00.000Z') + new Date('2021-01-01T00:15:00.000Z'), + {} as ToolingLog ); const span = (timestamp: number) => diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/03_span_destination_metrics.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/03_span_destination_metrics.test.ts index b7acefb48354..f0ac2c666139 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/03_span_destination_metrics.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/03_span_destination_metrics.test.ts @@ -12,6 +12,7 @@ import { sortBy } from 'lodash'; import { Readable } from 'stream'; import { createSpanMetricsAggregator } from '../../lib/apm/aggregators/create_span_metrics_aggregator'; import { awaitStream } from '../../lib/utils/wait_until_stream_finished'; +import { ToolingLog } from '@kbn/tooling-log'; describe('span destination metrics', () => { let events: Array>; @@ -26,7 +27,8 @@ describe('span destination metrics', () => { const range = timerange( new Date('2021-01-01T00:00:00.000Z'), - new Date('2021-01-01T00:15:00.000Z') + new Date('2021-01-01T00:15:00.000Z'), + {} as ToolingLog ); const serialized = [ diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/04_breakdown_metrics.test.ts b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/04_breakdown_metrics.test.ts index 76c2fa25c569..8e0259d5b24a 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/04_breakdown_metrics.test.ts +++ b/src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/04_breakdown_metrics.test.ts @@ -12,6 +12,7 @@ import { Readable } from 'stream'; import { awaitStream } from '../../lib/utils/wait_until_stream_finished'; import { createBreakdownMetricsAggregator } from '../../lib/apm/aggregators/create_breakdown_metrics_aggregator'; import { apm, ApmFields, timerange } from '@kbn/apm-synthtrace-client'; +import { ToolingLog } from '@kbn/tooling-log'; describe('breakdown metrics', () => { let events: ApmFields[]; @@ -32,7 +33,11 @@ describe('breakdown metrics', () => { const start = new Date('2021-01-01T00:00:00.000Z'); - const range = timerange(start, new Date(start.getTime() + INTERVALS * 30 * 1000)); + const range = timerange( + start, + new Date(start.getTime() + INTERVALS * 30 * 1000), + {} as ToolingLog + ); const listSpans = Array.from( range diff --git a/src/platform/packages/shared/kbn-apm-synthtrace/tsconfig.json b/src/platform/packages/shared/kbn-apm-synthtrace/tsconfig.json index b2e5c4b699fb..c991dd309b5d 100644 --- a/src/platform/packages/shared/kbn-apm-synthtrace/tsconfig.json +++ b/src/platform/packages/shared/kbn-apm-synthtrace/tsconfig.json @@ -11,6 +11,9 @@ "@kbn/dev-utils", "@kbn/elastic-agent-utils", "@kbn/zod", + "@kbn/sample-log-parser", + "@kbn/tooling-log", + "@kbn/streams-schema", ], "exclude": [ "target/**/*", diff --git a/tsconfig.base.json b/tsconfig.base.json index 7354cce0bca4..f877594cbc94 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1538,6 +1538,8 @@ "@kbn/safer-lodash-set/*": ["src/platform/packages/shared/kbn-safer-lodash-set/*"], "@kbn/saml-provider-plugin": ["x-pack/test/security_api_integration/plugins/saml_provider"], "@kbn/saml-provider-plugin/*": ["x-pack/test/security_api_integration/plugins/saml_provider/*"], + "@kbn/sample-log-parser": ["x-pack/platform/packages/shared/kbn-sample-parser"], + "@kbn/sample-log-parser/*": ["x-pack/platform/packages/shared/kbn-sample-parser/*"], "@kbn/sample-task-plugin": ["x-pack/test/plugin_api_integration/plugins/sample_task_plugin"], "@kbn/sample-task-plugin/*": ["x-pack/test/plugin_api_integration/plugins/sample_task_plugin/*"], "@kbn/sample-task-plugin-update-by-query": ["x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget"], diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/README.md b/x-pack/platform/packages/shared/kbn-sample-parser/README.md new file mode 100644 index 000000000000..d90a4ba2ae13 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/README.md @@ -0,0 +1,12 @@ +# @kbn/sample-log-parser + +This library downloads, parses and re-generates data from [loghub](https://github.com/logpai/loghub), a collection of sample log data sets. + +## CLI + +Run `AZURE_OPENAI_ENDPOINT=... AZURE_OPENAI_API_KEY=... node x-pack/scripts/loghub_parser.js` to generate and validate Loghub parsers. Every parser exports functions that extract and replace timestamps in log messages from Loghub systems. A parser is considered valid if the extracted timestamp is the same as the replaced timestamp. If a parser does not exist or is not valid, +the LLM is used to re-generate a new one. + +## SampleParserClient + +`SampleParserClient` reads the parsers and the sample datasets, and returns generators that will replay the loghub sample sets with updated timestamps. If the indexing rate is higher than a certain maximum (currently 100rpm), it will replay the sample set at a lower speed. diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/cli/index.ts b/x-pack/platform/packages/shared/kbn-sample-parser/cli/index.ts new file mode 100644 index 000000000000..5147cba8a4ed --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/cli/index.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { Command } from 'commander'; +import pLimit from 'p-limit'; +import { partition } from 'lodash'; +import { ensureLoghubRepo } from '../src/ensure_loghub_repo'; +import { readLoghubSystemFiles } from '../src/read_loghub_system_files'; +import { ensureValidParser } from '../src/ensure_valid_parser'; +import { createOpenAIClient } from '../src/create_openai_client'; +import { ensureValidQueries } from '../src/ensure_valid_queries'; + +async function run({ log }: { log: ToolingLog }) { + await ensureLoghubRepo({ log }); + + const systems = await readLoghubSystemFiles({ log }); + const limiter = pLimit(5); + + const openAIClient = createOpenAIClient(); + + const results = await Promise.all( + systems.map(async (system) => { + return limiter(async () => + Promise.all([ + ensureValidParser({ + openAIClient, + log, + system, + }), + ensureValidQueries({ + openAIClient, + system, + log, + }), + ]) + .then(() => { + return { + name: system.name, + error: null, + }; + }) + .catch((error) => { + return { + name: system.name, + error, + }; + }) + ); + }) + ); + + const [valid, invalid] = partition(results, (result) => !result.error); + if (invalid.length === 0) { + log.info(`Ensured ${valid.length} parsers`); + return; + } + + invalid.forEach((result) => { + log.error(`Failed generating a valid parser for ${result.name}`); + log.error(result.error); + }); + + throw new Error(`${invalid.length} out of ${results.length} parsers are invalid`); +} + +export function cli() { + const program = new Command('bin/kibana-setup'); + + program + .name('loghub-parser') + .description( + 'Generates code to extract and replace timestamps in loglines from Loghub datasets' + ) + .option('-d, --debug', 'Debug logging', false) + .option('-v, --verbose', 'Verbose logging', false) + .option('-s, --silent', 'Prevent all logging', false) + .action(async () => { + const options = program.opts() as { + silent: boolean; + verbose: boolean; + debug: boolean; + }; + + const log = new ToolingLog({ + level: options.silent + ? 'silent' + : options.debug + ? 'debug' + : options.verbose + ? 'verbose' + : 'info', + writeTo: process.stdout, + }); + + return run({ log }); + }) + .parse(process.argv); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.test.ts b/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.test.ts new file mode 100644 index 000000000000..3d2a54a8a8d9 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.test.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// createLoghubGenerator.test.ts +import { ToolingLog } from '@kbn/tooling-log'; +import { LoghubSystem } from '../src/read_loghub_system_files'; +import { LoghubParser } from '../src/types'; +import { createLoghubGenerator } from './create_loghub_generator'; + +describe('createLoghubGenerator', () => { + let system: LoghubSystem; + let parser: LoghubParser; + let log: ToolingLog; + + beforeEach(() => { + parser = { + getTimestamp: (line: string) => parseInt(line, 10), + replaceTimestamp: (line: string, timestamp: number) => line, + }; + }); + + describe('full speed', () => { + beforeEach(() => { + system = { + name: 'TestSystem', + readme: '', + // 2rpm + logLines: ['0', '60000'], + templates: [], + }; + log = new ToolingLog(); + }); + + it('generates the first event at the start time', () => { + const generator = createLoghubGenerator({ system, parser, log, queries: [] }); + const startTime = 100_000; + const docs = generator.next(startTime); + + expect(docs).toHaveLength(1); + + expect(docs[0]['@timestamp']).toBe(startTime); + }); + + it('generates the second event with the right offset', () => { + const generator = createLoghubGenerator({ system, parser, log, queries: [] }); + const startTime = 100_000; + generator.next(startTime); + + const docs = generator.next(startTime + 60000); + expect(docs).toHaveLength(1); + expect(docs[0]['@timestamp']).toBe(startTime + 60000); + }); + + it('returns no events if current time is before the next event', () => { + const generator = createLoghubGenerator({ system, parser, log, queries: [] }); + const startTime = 100_000; + generator.next(startTime); + + const docs = generator.next(startTime + 60000 - 1); + expect(docs).toHaveLength(0); + }); + }); + + describe('throttled', () => { + beforeEach(() => { + system = { + name: 'TestSystem', + readme: '', + // 200rpm + logLines: ['0', '599'], + templates: [], + }; + log = new ToolingLog(); + }); + + it('applies speed throttle when log rate is too high', () => { + const generator = createLoghubGenerator({ system, parser, log, targetRpm: 100, queries: [] }); + + const startTime = 100_000; + const firstBatch = generator.next(startTime); + + expect(firstBatch).toHaveLength(1); + // it starts at the usual time + expect(firstBatch[0]['@timestamp']).toBe(startTime); + + // after that, the delta should be half of what is expected + const secondBatch = generator.next(startTime + 1200); + const expectedTimestamp = startTime + 1200; + + expect(secondBatch.length).toBe(1); + + expect(secondBatch[0]['@timestamp']).toBe(expectedTimestamp); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.ts b/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.ts new file mode 100644 index 000000000000..313aeb2c0978 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { LoghubSystem } from '../src/read_loghub_system_files'; +import { LoghubParser } from '../src/types'; +import { StreamLogDocument, StreamLogGenerator } from './types'; +import { parseDataset } from './parse_dataset'; +import { LoghubQuery } from '../src/validate_queries'; + +export function createLoghubGenerator({ + system, + queries, + parser, + log, + targetRpm, +}: { + system: LoghubSystem; + queries: LoghubQuery[]; + parser: LoghubParser; + log: ToolingLog; + targetRpm?: number; +}): StreamLogGenerator { + let index = 0; + let start = 0; + + const { rpm: systemRpm, lines, min, range } = parseDataset({ system, parser }); + + const count = lines.length; + + const speed = targetRpm === undefined ? 1 : targetRpm / systemRpm; + + const filepath = `${system.name}.log`; + + log.debug( + `Throughput for ${system.name} will be around ${Math.round(targetRpm ?? systemRpm)}rpm` + ); + + return { + name: system.name, + filepath, + queries, + next: (timestamp) => { + if (index === 0) { + start = timestamp; + } + + const docs: StreamLogDocument[] = []; + + while (true) { + const line = lines[index % count]; + + const rotations = Math.floor(index / count); + + const rel = (line.timestamp - min) / range; + + // add 1 ms per rotation to separate start and end events + const delta = (1 / speed) * range * (rel + rotations) + rotations; + + // ES likes its timestamp to be an integer + const simulatedTimestamp = Math.floor(start + delta); + + if (simulatedTimestamp > timestamp) { + break; + } + + index++; + + const next = { + '@timestamp': simulatedTimestamp, + message: parser.replaceTimestamp(line.message, simulatedTimestamp), + filepath, + }; + + docs.push(next); + } + + return docs; + }, + }; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/client/index.ts b/x-pack/platform/packages/shared/kbn-sample-parser/client/index.ts new file mode 100644 index 000000000000..92003e27a36d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/client/index.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { sumBy } from 'lodash'; +import { StreamLogGenerator } from './types'; +import { readLoghubSystemFiles } from '../src/read_loghub_system_files'; +import { ensureLoghubRepo } from '../src/ensure_loghub_repo'; +import { validateParser } from '../src/validate_parser'; +import { getParser } from '../src/get_parser'; +import { createLoghubGenerator } from './create_loghub_generator'; +import { parseDataset } from './parse_dataset'; +import { validateQueries } from '../src/validate_queries'; +import { getQueries } from '../src/get_queries'; + +export class SampleParserClient { + private readonly logger: ToolingLog; + constructor(options: { logger: ToolingLog }) { + this.logger = options.logger; + } + + async getLogGenerators({ + rpm = 10000, + distribution = 'uniform', + }: { + rpm?: number; + distribution?: 'relative' | 'uniform'; + }): Promise { + await ensureLoghubRepo({ log: this.logger }); + const systems = await readLoghubSystemFiles({ log: this.logger }); + + const results = await Promise.all( + systems.map(async (system) => { + await Promise.all([validateParser(system), validateQueries(system)]).catch((error) => { + throw new AggregateError([error], `Parser for ${system.name} is not valid`); + }); + + const [parser, queries] = await Promise.all([getParser(system), getQueries(system)]); + const { rpm: systemRpm } = parseDataset({ system, parser }); + + return { + parser, + system, + rpm: systemRpm, + queries, + }; + }) + ); + + const totalRpm = sumBy(results, ({ rpm: systemRpm }) => systemRpm); + + return await Promise.all( + results.map(({ system, parser, rpm: systemRpm, queries }) => { + let targetRpm: number; + if (distribution === 'relative') { + const share = systemRpm / totalRpm; + targetRpm = rpm === undefined ? Math.min(100, systemRpm) : share * rpm; + } else { + targetRpm = Math.round(rpm / results.length); + } + + return createLoghubGenerator({ + system, + parser, + log: this.logger, + targetRpm: Math.max(1, targetRpm), + queries, + }); + }) + ); + } +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/client/parse_dataset.ts b/x-pack/platform/packages/shared/kbn-sample-parser/client/parse_dataset.ts new file mode 100644 index 000000000000..b4fd0aa4b540 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/client/parse_dataset.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { LoghubSystem } from '../src/read_loghub_system_files'; +import { LoghubParser } from '../src/types'; + +export function parseDataset({ system, parser }: { parser: LoghubParser; system: LoghubSystem }) { + const parsedLogLines = system.logLines.map((line) => { + const timestamp = parser.getTimestamp(line); + return { + timestamp, + message: line, + }; + }); + + const min = parsedLogLines[0].timestamp; + + let minTimestamp = min; + let years = 0; + + // add years for timestamps without years + parsedLogLines.forEach((logLine) => { + if (logLine.timestamp < minTimestamp) { + minTimestamp = logLine.timestamp; + years++; + } + if (years >= 0) { + logLine.timestamp = moment(logLine.timestamp).add(years, 'years').valueOf(); + } + }); + + const max = parsedLogLines[parsedLogLines.length - 1].timestamp; + + const count = parsedLogLines.length; + + const range = max - min; + + const rpm = count / (range / 1000 / 60); + + return { + lines: parsedLogLines, + rpm, + range, + min, + max, + }; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/client/types.ts b/x-pack/platform/packages/shared/kbn-sample-parser/client/types.ts new file mode 100644 index 000000000000..1e6112fad919 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/client/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LoghubQuery } from '../src/validate_queries'; + +export interface StreamLogDocument { + [x: string]: unknown; + filepath: string; + message: string; + '@timestamp': number; +} + +export interface StreamLogGenerator { + name: string; + filepath: string; + queries: LoghubQuery[]; + next: (timestamp: number) => StreamLogDocument[]; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/index.ts b/x-pack/platform/packages/shared/kbn-sample-parser/index.ts new file mode 100644 index 000000000000..0db451cbc5c6 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { cli } from './cli'; +export { SampleParserClient } from './client'; +export { type LoghubQuery, createQueryMatcher, tokenize } from './src/validate_queries'; diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/jest.config.js b/x-pack/platform/packages/shared/kbn-sample-parser/jest.config.js new file mode 100644 index 000000000000..228dffa35933 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../../..', + roots: ['/x-pack/platform/packages/shared/kbn-sample-parser'], +}; diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/kibana.jsonc b/x-pack/platform/packages/shared/kbn-sample-parser/kibana.jsonc new file mode 100644 index 000000000000..86266fd233ab --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/sample-log-parser", + "owner": "@elastic/streams-program-team", + "group": "platform", + "visibility": "shared", + "devOnly": true +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/package.json b/x-pack/platform/packages/shared/kbn-sample-parser/package.json new file mode 100644 index 000000000000..9b61c36fc6e1 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/sample-log-parser", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/parser.ts new file mode 100644 index 000000000000..a1bfd128d6c0 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/parser.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const [_, month, day, hour, minute, second, millisecond] = match; + const dateString = `2023-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`; + return moment.utc(dateString).valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = moment.utc(timestamp).format('MM-DD HH:mm:ss.SSS'); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/queries.json new file mode 100644 index 000000000000..13c5e8b36f68 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/queries.json @@ -0,0 +1,365 @@ +{ + "queries": [ + { + "id": "register_callback", + "title": "Register Callback", + "description": "Identifies log messages related to registering a callback.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "register callback", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "unregister_callback", + "title": "Unregister Callback", + "description": "Identifies log messages related to unregistering a callback.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "unregister callback", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "unregister_callback_null", + "title": "Unregister Callback Null", + "description": "Identifies log messages related to unregistering a callback for null.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "unregister callback null", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cleanup_application_record", + "title": "Clean Up Application Record", + "description": "Identifies log messages related to cleaning up an application record.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cleanUpApplicationRecord", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cleanup_application_record_locked_restart_false", + "title": "Clean Up Application Record Locked (Restart False)", + "description": "Identifies log messages related to cleaning up an application record with restart set to false.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cleanUpApplicationRecordLocked restart false", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cleanup_application_record_locked_reset_pid", + "title": "Clean Up Application Record Locked (Reset PID)", + "description": "Identifies log messages related to cleaning up an application record with reset PID.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cleanUpApplicationRecordLocked reset pid", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "new_process_app", + "title": "New Process App", + "description": "Identifies log messages related to the creation of a new process app.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "new Process app=ProcessRecord", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "process_has_died", + "title": "Process Has Died", + "description": "Identifies log messages related to a process that has died.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Process has died", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "start_proc_for_service", + "title": "Start Process for Service", + "description": "Identifies log messages related to starting a process for a service.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Start proc for service", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cannot_be_cast_to_token", + "title": "Cannot Be Cast to Token", + "description": "Identifies log messages related to casting errors to a Token.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cannot be cast to Token", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "must_execute_in_ui", + "title": "Must Execute in UI", + "description": "Identifies log messages related to execution requirements in the UI.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Must execute in UI", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "bad_activity_token", + "title": "Bad Activity Token", + "description": "Identifies log messages related to bad activity tokens.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Bad activity token", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "unable_to_start_service_intent_not_found", + "title": "Unable to Start Service Intent (Not Found)", + "description": "Identifies log messages related to unable to start service intent not found.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Unable to start service Intent not found", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_app_window_token_going_to_hide", + "title": "Skipping App Window Token (Going to Hide)", + "description": "Identifies log messages related to skipping app window token going to hide.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping AppWindowToken going to hide", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_with_excluded_false_tr_intent_intent", + "title": "Skipping with Excluded False (tr.intent:Intent)", + "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping withExcluded false tr.intent:Intent", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_with_excluded_false_tr_intent_intent_typ", + "title": "Skipping with Excluded False (tr.intent:Intent with Type)", + "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with type.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping withExcluded false tr.intent:Intent", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_with_excluded_false_tr_intent_intent_flg", + "title": "Skipping with Excluded False (tr.intent:Intent with Flag)", + "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping withExcluded false tr.intent:Intent", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_with_excluded_false_tr_intent_intent_flg_cmp", + "title": "Skipping with Excluded False (tr.intent:Intent with Flag and Component)", + "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag and component.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping withExcluded false tr.intent:Intent", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "skipping_with_excluded_false_tr_intent_intent_flg_cmp_bnds", + "title": "Skipping with Excluded False (tr.intent:Intent with Flag, Component, and Bounds)", + "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag, component, and bounds.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Skipping withExcluded false tr.intent:Intent", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/parser.ts new file mode 100644 index 000000000000..b060ee7159e9 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/parser.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const timestampRegex = /\[(\w{3}) (\w{3}) (\d{1,2}) (\d{2}:\d{2}:\d{2}) (\d{4})\]/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(timestampRegex); + if (match) { + const [_, dayOfWeek, month, day, time, year] = match; + const timestampString = `${dayOfWeek} ${month} ${day} ${time} ${year}`; + const date = moment.utc(timestampString, 'ddd MMM DD HH:mm:ss YYYY'); + return date.valueOf(); + } + throw new Error('Invalid log line format'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newDate = moment.utc(timestamp); + const newTimestampString = newDate.format('ddd MMM DD HH:mm:ss YYYY'); + return logLine.replace(timestampRegex, `[${newTimestampString}]`); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/queries.json new file mode 100644 index 000000000000..83918b03ea78 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/queries.json @@ -0,0 +1,118 @@ +{ + "queries": [ + { + "id": "jk2_init_found_child", + "title": "JK2 Init Found Child", + "description": "Returns logs where 'jk2_init() Found child' is found in the scoreboard slot.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "jk2_init() Found child scoreboard slot", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "worker_env_init_ok", + "title": "Worker Environment Init OK", + "description": "Returns logs where 'workerEnv.init() ok' is mentioned.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "workerEnv.init() ok", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mod_jk_child_error_state", + "title": "Mod JK Child WorkerEnv Error State", + "description": "Returns logs where 'mod_jk child workerEnv in error state' is found.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mod_jk child workerEnv error state", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "directory_index_forbidden", + "title": "Directory Index Forbidden", + "description": "Returns logs where 'Directory index forbidden by rule' is mentioned.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Directory index forbidden by rule", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "jk2_init_cant_find_child", + "title": "JK2 Init Can't Find Child", + "description": "Returns logs where 'jk2_init() Can't find child' is found in the scoreboard.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "jk2_init() Can't find child scoreboard", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mod_jk_child_init", + "title": "Mod JK Child Init", + "description": "Returns logs where 'mod_jk child init' is mentioned.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mod_jk child init", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/parser.ts new file mode 100644 index 000000000000..2b85e69dc353 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/parser.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const timestampRegex = /\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}\.\d{2}\.\d{6}/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(timestampRegex); + if (match) { + const timestamp = match[0]; + const momentObj = moment.utc(timestamp, 'YYYY-MM-DD-HH.mm.ss.SSSSSS'); + return momentObj.valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const momentObj = moment.utc(timestamp); + const formattedTimestamp = momentObj.format('YYYY-MM-DD-HH.mm.ss.SSSSSS'); + return logLine.replace(timestampRegex, formattedTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/queries.json new file mode 100644 index 000000000000..4564fa67e572 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/queries.json @@ -0,0 +1,821 @@ +{ + "queries": [ + { + "id": "e1", + "title": "DDR Errors Detected and Corrected with Rank, Symbol, and Seconds", + "description": "Returns log messages indicating DDR errors that were detected and corrected, including rank, symbol, and seconds information.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ddr error(s) detected AND corrected AND rank AND symbol AND seconds", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e2", + "title": "DDR Errors Detected and Corrected with Rank, Symbol, and Bit", + "description": "Returns log messages indicating DDR errors that were detected and corrected, including rank, symbol, and bit information.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ddr errors(s) detected AND corrected AND rank AND symbol AND bit", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e5", + "title": "L3 EDRAM Errors Detected and Corrected", + "description": "Returns log messages indicating L3 EDRAM errors that were detected and corrected.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "L3 EDRAM error(s) detected AND corrected", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e6", + "title": "L3 EDRAM Errors Detected and Corrected with Seconds", + "description": "Returns log messages indicating L3 EDRAM errors that were detected and corrected, including seconds information.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "L3 EDRAM error(s) detected AND corrected AND seconds", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e17", + "title": "Cannot Get Assembly Information for Node Card", + "description": "Returns log messages indicating an inability to get assembly information for a node card.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Can not get assembly information for node card", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e21", + "title": "Error Creating Node Map with Bad File Descriptor", + "description": "Returns log messages indicating an error in creating a node map due to a bad file descriptor.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error creating node map AND Bad file descriptor", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e22", + "title": "Error Creating Node Map with Block Device Required", + "description": "Returns log messages indicating an error in creating a node map due to a block device being required.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error creating node map AND Block device required", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e23", + "title": "Error Creating Node Map with No Child Processes", + "description": "Returns log messages indicating an error in creating a node map due to no child processes.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error creating node map AND No child processes", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e24", + "title": "Error Creating Node Map with Permission Denied", + "description": "Returns log messages indicating an error in creating a node map due to permission being denied.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error creating node map AND Permission denied", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e25", + "title": "Error Loading with Invalid or Missing Program Image and Exec Format Error", + "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and an exec format error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error loading AND invalid or missing program image AND Exec format error", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e26", + "title": "Error Loading with Invalid or Missing Program Image and Permission Denied", + "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and permission being denied.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error loading AND invalid or missing program image AND Permission denied", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e27", + "title": "Error Loading with Program Image Too Big", + "description": "Returns log messages indicating an error in loading due to the program image being too big.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error loading AND program image too big", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e28", + "title": "Error Loading with Invalid or Missing Program Image and No Such File or Directory", + "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and no such file or directory.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error loading AND invalid or missing program image AND No such file or directory", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e29", + "title": "Error Reading Message Prefix with Link Severed", + "description": "Returns log messages indicating an error in reading the message prefix due to the link being severed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error reading message prefix AND Link has been severed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e30", + "title": "Error Reading Message Prefix with Connection Reset by Peer", + "description": "Returns log messages indicating an error in reading the message prefix due to the connection being reset by the peer.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error reading message prefix AND Connection reset by peer", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e31", + "title": "Error Reading Message Prefix with Connection Timed Out", + "description": "Returns log messages indicating an error in reading the message prefix due to the connection timing out.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error reading message prefix AND Connection timed out", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e32", + "title": "Error Reading Message Prefix with Link Severed", + "description": "Returns log messages indicating an error in reading the message prefix due to the link being severed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error reading message prefix AND Link has been severed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e33", + "title": "Failed to Read Message Prefix with Control Stream", + "description": "Returns log messages indicating a failure to read the message prefix due to the control stream.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "failed to read message prefix AND control stream", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e34", + "title": "Generated Core Files for Program", + "description": "Returns log messages indicating that core files were generated for a program.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "generated core files for program", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e40", + "title": "Poll Control Descriptors with Debugger Died", + "description": "Returns log messages indicating an issue with poll control descriptors due to the debugger dying.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pollControlDescriptors AND Detected the debugger died", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e44", + "title": "Critical Input Interrupt with Warning and Torus Y+ Wire", + "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Y+ wire.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "critical input interrupt AND warning AND torus y+ wire", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e45", + "title": "Critical Input Interrupt with Warning and Torus Z- Wire", + "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Z- wire.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "critical input interrupt AND warning AND torus z- wire", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e46", + "title": "Critical Input Interrupt with Warning, Torus Z+ Wire, and Suppressing Further Interrupts", + "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Z+ wire and suppressing further interrupts.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "critical input interrupt AND warning AND torus z+ wire AND suppressing further interrupts", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e47", + "title": "Critical Input Interrupt with Warning, Tree C1 Wire, and Suppressing Further Interrupts", + "description": "Returns log messages indicating a critical input interrupt with a warning related to the tree C1 wire and suppressing further interrupts.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "critical input interrupt AND warning AND tree C1 wire AND suppressing further interrupts", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e51", + "title": "Data Cache Search Parity Error Detected and Attempting to Correct", + "description": "Returns log messages indicating a data cache search parity error that was detected and an attempt to correct it.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "data cache search parity error detected AND attempting to correct", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e77", + "title": "Instruction Cache Parity Error Corrected", + "description": "Returns log messages indicating an instruction cache parity error that was corrected.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "instruction cache parity error corrected", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e80", + "title": "Lustre Mount Failed", + "description": "Returns log messages indicating a failed Lustre mount.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Lustre mount FAILED", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e81", + "title": "Lustre Mount Failed", + "description": "Returns log messages indicating a failed Lustre mount.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Lustre mount FAILED", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e82", + "title": "Machine Check DCR Read Timeout", + "description": "Returns log messages indicating a machine check DCR read timeout.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "MACHINE CHECK DCR read timeout", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e89", + "title": "NFS Mount Failed", + "description": "Returns log messages indicating a failed NFS mount.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "NFS Mount failed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e90", + "title": "Node Card Not Fully Functional", + "description": "Returns log messages indicating that a node card is not fully functional.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Node card is not fully functional", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e94", + "title": "NodeCard Not Fully Functional", + "description": "Returns log messages indicating that a NodeCard is not fully functional.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "NodeCard is not fully functional", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e95", + "title": "PrepareForService Shutting Down Node Card", + "description": "Returns log messages indicating that PrepareForService is shutting down a node card.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PrepareForService shutting down Node card", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e96", + "title": "PrepareForService Shutting Down NodeCard", + "description": "Returns log messages indicating that PrepareForService is shutting down a NodeCard.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PrepareForService shutting down NodeCard", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e107", + "title": "RTS Internal Error", + "description": "Returns log messages indicating an RTS internal error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rts internal error", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e108", + "title": "RTS Panic and Stopping Execution", + "description": "Returns log messages indicating an RTS panic and stopping execution.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rts panic AND stopping execution", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e109", + "title": "RTS Tree/Torus Link Training Failed", + "description": "Returns log messages indicating an RTS tree/torus link training failure.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rts tree/torus link training failed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e111", + "title": "RTS Kernel Terminated for Reason", + "description": "Returns log messages indicating that the RTS kernel was terminated for a reason.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rts kernel terminated for reason", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e113", + "title": "Shutdown Complete", + "description": "Returns log messages indicating that a shutdown is complete.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "shutdown complete", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e117", + "title": "Suppressing Further Interrupts of Same Type", + "description": "Returns log messages indicating that further interrupts of the same type are being suppressed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "suppressing further interrupts of same type", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e118", + "title": "Total DDR Errors Detected and Corrected", + "description": "Returns log messages indicating the total number of DDR errors that were detected and corrected.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "total of ddr error(s) detected AND corrected", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e119", + "title": "Total DDR Errors Detected and Corrected with Seconds", + "description": "Returns log messages indicating the total number of DDR errors that were detected and corrected, including seconds information.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "total of ddr error(s) detected AND corrected AND seconds", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "e120", + "title": "Wait State Enable", + "description": "Returns log messages indicating that the wait state is enabled.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "wait state enable", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/parser.ts new file mode 100644 index 000000000000..71adf3c79af2 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/parser.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const timestampRegex = /(\d{6}) (\d{6}) (\d{1,3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(timestampRegex); + if (match) { + const [_, datePart, timePart, millisPart] = match; + const formattedDate = `20${datePart} ${timePart}.${millisPart.padStart(3, '0')}`; + const momentDate = moment.utc(formattedDate, 'YYYYMMDD HHmmss.SSS'); + return momentDate.valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const match = logLine.match(timestampRegex); + if (match) { + const momentDate = moment.utc(timestamp); + const newTimestamp = momentDate.format('YYMMDD HHmmss SSS'); + return logLine.replace(timestampRegex, newTimestamp); + } + throw new Error('Timestamp not found in log line'); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/queries.json new file mode 100644 index 000000000000..65042d4f542f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/queries.json @@ -0,0 +1,232 @@ +{ + "queries": [ + { + "id": "starting_thread_transfer_block", + "title": "Starting thread to transfer block", + "description": "Matches log messages that indicate the start of a thread to transfer a block.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting thread to transfer block", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "exception_serving_block", + "title": "Got exception while serving block", + "description": "Matches log messages that indicate an exception occurred while serving a block.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Got exception while serving", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "ask_delete_block", + "title": "Ask to delete block", + "description": "Matches log messages that indicate a request to delete a block.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ask to delete blk", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "ask_replicate_block", + "title": "Ask to replicate block", + "description": "Matches log messages that indicate a request to replicate a block.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ask to replicate blk", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "blockmap_updated", + "title": "BlockMap updated", + "description": "Matches log messages that indicate the blockMap has been updated.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "blockMap updated", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "allocate_block", + "title": "Allocate block", + "description": "Matches log messages that indicate a block allocation.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "allocateBlock", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "block_added_invalidset", + "title": "Block is added to invalidSet", + "description": "Matches log messages that indicate a block has been added to the invalid set.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "is added to invalidSet", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "deleting_block", + "title": "Deleting block", + "description": "Matches log messages that indicate a block is being deleted.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Deleting block", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "packetresponder_terminating", + "title": "PacketResponder terminating", + "description": "Matches log messages that indicate a PacketResponder is terminating.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PacketResponder terminating", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "received_block", + "title": "Received block", + "description": "Matches log messages that indicate a block has been received.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Received block", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "receiving_block", + "title": "Receiving block", + "description": "Matches log messages that indicate a block is being received.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Receiving block", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "verification_succeeded_block", + "title": "Verification succeeded for block", + "description": "Matches log messages that indicate a block verification succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Verification succeeded for blk", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/parser.ts new file mode 100644 index 000000000000..b5f20f1c5177 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/parser.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const TIMESTAMP_REGEX = /\b\d{10}\b/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + return parseInt(match[0], 10) * 1000; + } + throw new Error('Timestamp not found'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = Math.floor(timestamp / 1000).toString(); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/queries.json new file mode 100644 index 000000000000..73c7ef47da33 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/queries.json @@ -0,0 +1,330 @@ +{ + "queries": [ + { + "id": "startup_boot_command", + "title": "Startup Message (boot command)", + "description": "Returns log messages that indicate a boot command was issued.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "boot command", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "shutdown_closing", + "title": "Shutdown Message (closing)", + "description": "Returns log messages that indicate a shutdown process with the term 'closing'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "closing" + } + } + ] + } + } + }, + { + "id": "startup_starting", + "title": "Startup Message (starting)", + "description": "Returns log messages that indicate a startup process with the term 'starting'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "starting" + } + } + ] + } + } + }, + { + "id": "shutdown_halt_command", + "title": "Shutdown Message (halt command)", + "description": "Returns log messages that indicate a halt command was issued.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "halt command", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_boot_halt_asserted", + "title": "Error Message (boot command Error: HALT asserted)", + "description": "Returns log messages that indicate a boot command error with HALT asserted.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "boot.*Error: HALT asserted" + } + } + ] + } + } + }, + { + "id": "critical_message", + "title": "Critical Message (critical)", + "description": "Returns log messages that contain the term 'critical'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "critical" + } + } + ] + } + } + }, + { + "id": "error_command_aborted", + "title": "Error Message (Command has been aborted)", + "description": "Returns log messages that indicate a command has been aborted.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "Command has been aborted" + } + } + ] + } + } + }, + { + "id": "error_failed_subcommands", + "title": "Error Message (Failed subcommands)", + "description": "Returns log messages that indicate failed subcommands.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "Failed subcommands" + } + } + ] + } + } + }, + { + "id": "link_error", + "title": "Link Error Message (Link error)", + "description": "Returns log messages that indicate a link error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "Link error" + } + } + ] + } + } + }, + { + "id": "link_error_broadcast_tree", + "title": "Link Error Message (Link error on broadcast tree Interconnect)", + "description": "Returns log messages that indicate a link error on the broadcast tree interconnect.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Link error on broadcast tree Interconnect.*" + } + } + ] + } + } + }, + { + "id": "warning_message", + "title": "Warning Message (warning)", + "description": "Returns log messages that contain the term 'warning'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "warning" + } + } + ] + } + } + }, + { + "id": "temperature_warning", + "title": "Temperature Warning Message (Temperature exceeds warning threshold)", + "description": "Returns log messages that indicate temperature exceeds the warning threshold.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Temperature.*exceeds warning threshold" + } + } + ] + } + } + }, + { + "id": "error_server_filesystem_panic", + "title": "Error Message (ServerFileSystem domain panic)", + "description": "Returns log messages that indicate a ServerFileSystem domain panic.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "ServerFileSystem.*domain panic" + } + } + ] + } + } + }, + { + "id": "error_server_filesystem_full", + "title": "Error Message (ServerFileSystem domain is full)", + "description": "Returns log messages that indicate a ServerFileSystem domain is full.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "ServerFileSystem.*domain.*is full" + } + } + ] + } + } + }, + { + "id": "error_not_responding", + "title": "Error Message (not responding)", + "description": "Returns log messages that indicate a system or service is not responding.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "not responding" + } + } + ] + } + } + }, + { + "id": "error_not_responding_hyphen", + "title": "Error Message (not-responding)", + "description": "Returns log messages that indicate a system or service is not-responding.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "not-responding" + } + } + ] + } + } + }, + { + "id": "power_control_problem", + "title": "Power/Control Problem Message (power/control problem)", + "description": "Returns log messages that indicate a power or control problem.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": "power/control problem" + } + } + ] + } + } + }, + { + "id": "psu_failure", + "title": "PSU Failure Message (psu failure)", + "description": "Returns log messages that indicate a PSU failure.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "psu failure.*" + } + } + ] + } + } + }, + { + "id": "error_risboot_timeout", + "title": "Error Message (risBoot command Error: Timed out)", + "description": "Returns log messages that indicate a risBoot command error due to timeout.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "risBoot.*Error: Timed out" + } + } + ] + } + } + }, + { + "id": "network_connection_failure", + "title": "Network Connection Failure Message (detected a failed network connection)", + "description": "Returns log messages that indicate a failed network connection.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "detected a failed network connection" + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/parser.ts new file mode 100644 index 000000000000..81e7e4e0b802 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/parser.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const timestamp = match[1]; + return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss,SSS').valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss,SSS'); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/queries.json new file mode 100644 index 000000000000..3b35eeb07ddd --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/queries.json @@ -0,0 +1,308 @@ +{ + "queries": [ + { + "id": "failures_on_node", + "title": "Failures on node MININT", + "description": "Finds log messages indicating failures on node MININT.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "failures on node MININT", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "added_attempt_failed_maps", + "title": "Added attempt to list of failed maps", + "description": "Finds log messages indicating an attempt was added to the list of failed maps.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Added attempt to list of failed maps", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "taskattempt_fail_container_cleanup_to_fail_task_cleanup", + "title": "TaskAttempt Transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP", + "description": "Finds log messages indicating a TaskAttempt transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "TaskAttempt Transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "taskattempt_fail_task_cleanup_to_failed", + "title": "TaskAttempt Transitioned from FAIL_TASK_CLEANUP to FAILED", + "description": "Finds log messages indicating a TaskAttempt transitioned from FAIL_TASK_CLEANUP to FAILED.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "TaskAttempt Transitioned from FAIL_TASK_CLEANUP to FAILED", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "taskattempt_running_to_fail_container_cleanup", + "title": "TaskAttempt Transitioned from RUNNING to FAIL_CONTAINER_CLEANUP", + "description": "Finds log messages indicating a TaskAttempt transitioned from RUNNING to FAIL_CONTAINER_CLEANUP.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "TaskAttempt Transitioned from RUNNING to FAIL_CONTAINER_CLEANUP", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "datastreamer_exception", + "title": "DataStreamer Exception", + "description": "Finds log messages indicating a DataStreamer Exception.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "DataStreamer Exception", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "dfsoutputstream_responseprocessor_exception", + "title": "DFSOutputStream ResponseProcessor exception for block", + "description": "Finds log messages indicating a DFSOutputStream ResponseProcessor exception for a block.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "DFSOutputStream ResponseProcessor exception for block", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "diagnostics_report_no_route_to_host", + "title": "Diagnostics report from attempt: No Route to Host", + "description": "Finds log messages indicating a diagnostics report from an attempt with a No Route to Host error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Diagnostics report from attempt Error java.net.NoRouteToHostException No Route to Host", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_contacting_rm", + "title": "ERROR IN CONTACTING RM", + "description": "Finds log messages indicating an error in contacting the Resource Manager (RM).", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ERROR IN CONTACTING RM", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_recovery_bad_datanode", + "title": "Error Recovery for block in pipeline: bad datanode", + "description": "Finds log messages indicating an error recovery for a block in the pipeline due to a bad datanode.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error Recovery for block in pipeline bad datanode", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_writing_history_event", + "title": "Error writing History Event", + "description": "Finds log messages indicating an error writing a History Event.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error writing History Event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "failed_to_renew_lease", + "title": "Failed to renew lease for DFSClient_NONMAPREDUCE", + "description": "Finds log messages indicating a failure to renew a lease for DFSClient_NONMAPREDUCE.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Failed to renew lease for DFSClient_NONMAPREDUCE", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "killing_attempt", + "title": "KILLING attempt", + "description": "Finds log messages indicating an attempt is being killed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "KILLING attempt", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "task_cleanup_failed", + "title": "Task cleanup failed for attempt", + "description": "Finds log messages indicating a task cleanup failed for an attempt.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Task cleanup failed for attempt", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "task_exited_no_route_to_host", + "title": "Task attempt exited: No Route to Host", + "description": "Finds log messages indicating a task attempt exited due to a No Route to Host error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Task attempt exited java.net.NoRouteToHostException No Route to Host", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "thread_threw_exception", + "title": "Thread threw an Exception", + "description": "Finds log messages indicating a thread threw an exception.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Thread Thread threw an Exception", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/invalid_parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/invalid_parser.ts new file mode 100644 index 000000000000..b5083a9dd0c6 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/invalid_parser.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /^(\d{4})(\d{1,2})(\d{1,2})-(\d{1,2}):(\d{1,2}):(\d{1,2}):(\d{1,3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const [_, year, month, day, hour, minute, second, millisecond] = match; + const timestampStr = `${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`; + const timestamp = moment.utc(timestampStr, 'YYYY-MM-DDTHH:mm:ss.SSSZ').valueOf(); + return timestamp; + } + throw new Error('Timestamp not found'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestampStr = moment.utc(timestamp).format('YYYYMMDD-HH:mm:ss:SSS'); + return logLine.replace(TIMESTAMP_REGEX, newTimestampStr); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/parser.ts new file mode 100644 index 000000000000..3b2d3b43c5a1 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/parser.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /(\d{4}\d{2}\d{2}-\d{1,2}:\d{1,2}:\d{1,2}:\d{1,3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + return moment.utc(match[0], 'YYYYMMDD-HH:mm:ss:SSS').valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = moment.utc(timestamp).format('YYYYMMDD-HH:mm:ss:SSS'); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/queries.json new file mode 100644 index 000000000000..7695b0ab026d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/queries.json @@ -0,0 +1,226 @@ +{ + "queries": [ + { + "id": "init_environment", + "title": "Init Environment", + "description": "Returns log messages related to the initialization of the environment.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "initEnviroument", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "init_data_privacy", + "title": "Init Data Privacy", + "description": "Returns log messages indicating that data privacy has been initialized and is set to true.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "initDataPrivacy the dataPrivacy is true", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "init_user_privacy", + "title": "Init User Privacy", + "description": "Returns log messages indicating that user privacy has been initialized and is set to true.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "initUserPrivacy the userPrivacy is true", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "start_timer_auto_sync", + "title": "Start Timer AutoSync", + "description": "Returns log messages indicating that the auto-sync timer has started.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "startTimer start autoSync", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "bulk_save_fail_error_code", + "title": "Bulk Save Fail Error Code", + "description": "Returns log messages indicating a failure in bulk saving health data with an error code.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "insertHiHealthData\\(\\) bulkSaveDetailHiHealthData fail errorCode = .*" + } + } + ] + } + } + }, + { + "id": "save_one_detail_data_fail", + "title": "Save One Detail Data Fail", + "description": "Returns log messages indicating a failure in saving one detail health data with specific data and type.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "saveHealthDetailData\\(\\) saveOneDetailData fail hiHealthData = .*,type = .*" + } + } + ] + } + } + }, + { + "id": "upload_statics_to_db_failed", + "title": "Upload Statics to DB Failed", + "description": "Returns log messages indicating a failure in uploading statics to the database with a message set to true.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "uploadStaticsToDB failed message=true", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "send_sync_failed_broadcast", + "title": "Send Sync Failed Broadcast", + "description": "Returns log messages indicating that a sync failed broadcast was sent.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "sendSyncFailedBroadcast", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "screen_off", + "title": "Screen Off", + "description": "Returns log messages indicating that the screen off action was received.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "onReceive action: android.intent.action.SCREEN_OFF", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "screen_on", + "title": "Screen On", + "description": "Returns log messages indicating that the screen on action was received.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "onReceive action: android.intent.action.SCREEN_ON", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "process_handle_screen_on", + "title": "Process Handle Screen On", + "description": "Returns log messages indicating that the process handled a screen on broadcast action.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "processHandleBroadcastAction action:android.intent.action.SCREEN_ON", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "screen_status_unknown", + "title": "Screen Status Unknown", + "description": "Returns log messages indicating that the screen status is unknown but assumed to be on.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "screen status unknown,think screen on", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/parser.ts new file mode 100644 index 000000000000..3baa320d732a --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/parser.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const [_, month, day, hour, minute, second] = match; + const dateString = `${month} ${day} ${hour}:${minute}:${second} UTC`; + const date = moment.utc(dateString, 'MMM DD HH:mm:ss'); + return date.valueOf(); + } + throw new Error('Timestamp not found'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newDate = moment.utc(timestamp).format('MMM DD HH:mm:ss'); + return logLine.replace(TIMESTAMP_REGEX, newDate); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/queries.json new file mode 100644 index 000000000000..76b1732fd3a5 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/queries.json @@ -0,0 +1,369 @@ +{ + "queries": [ + { + "id": "cupsd_startup", + "title": "CUPS Daemon Startup", + "description": "Extracts log messages indicating that the CUPS daemon startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cupsd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cupsd_shutdown", + "title": "CUPS Daemon Shutdown", + "description": "Extracts log messages indicating that the CUPS daemon shutdown succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "cupsd shutdown succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "warning_client_address", + "title": "Warning: Client Address Issue", + "description": "Extracts log messages indicating a warning related to client address connection reset by peer.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "warning: can't get client address: Connection reset by peer", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "auth_failed_permission_denied", + "title": "Error: Authentication Failed (Permission Denied)", + "description": "Extracts log messages indicating authentication failure due to permission denied in replay cache code.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Authentication failed from .* \\(.*\\): Permission denied in replay cache code" + } + } + ] + } + } + }, + { + "id": "auth_failed_connection_abort", + "title": "Error: Authentication Failed (Connection Abort)", + "description": "Extracts log messages indicating authentication failure due to software caused connection abort.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Authentication failed from .* \\(.*\\): Software caused connection abort" + } + } + ] + } + } + }, + { + "id": "couldnt_authenticate_user", + "title": "Error: Couldn't Authenticate User", + "description": "Extracts log messages indicating failure to authenticate a user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Couldn't authenticate user", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "failure_registering_capabilities", + "title": "Error: Failure Registering Capabilities", + "description": "Extracts log messages indicating failure in registering capabilities with the kernel.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Failure registering capabilities with the kernel", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "kerberos_auth_failed", + "title": "Error: Kerberos Authentication Failed", + "description": "Extracts log messages indicating Kerberos authentication failure.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Kerberos authentication failed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "anonymous_ftp_login", + "title": "Error: Anonymous FTP Login", + "description": "Extracts log messages indicating an anonymous FTP login attempt.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "ANONYMOUS FTP LOGIN FROM .*, \\(anonymous\\)" + } + } + ] + } + } + }, + { + "id": "alert_exited_abnormally", + "title": "Error: Alert Exited Abnormally", + "description": "Extracts log messages indicating an alert that exited abnormally with code 1.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ALERT exited abnormally with [1]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "audit_netlink_socket", + "title": "Audit: Initializing Netlink Socket", + "description": "Extracts log messages indicating that the audit system is initializing a netlink socket.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "audit: initializing netlink socket (disabled)", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "audit_initialized", + "title": "Audit: Initialized", + "description": "Extracts log messages indicating that the audit system has been initialized.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "audit\\(.*\\..*\\:.*\\): initialized" + } + } + ] + } + } + }, + { + "id": "klogd_startup", + "title": "Klogd Startup", + "description": "Extracts log messages indicating that the klogd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "klogd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "syslogd_startup", + "title": "Syslogd Startup", + "description": "Extracts log messages indicating that the syslogd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "syslogd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "rpc_idmapd_startup", + "title": "RPC Idmapd Startup", + "description": "Extracts log messages indicating that the rpc.idmapd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rpc.idmapd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "rpc_statd_startup", + "title": "RPC Statd Startup", + "description": "Extracts log messages indicating that the rpc.statd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rpc.statd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "sdpd_startup", + "title": "SDPD Startup", + "description": "Extracts log messages indicating that the sdpd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "sdpd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "hcid_startup", + "title": "HCID Startup", + "description": "Extracts log messages indicating that the hcid startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "hcid startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "imps2_autodetected", + "title": "IMPS2 Auto-detected", + "description": "Extracts log messages indicating that the IMPS2 auto-detected an IntelliMouse PS/2.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "imps2: Auto-detected intellimouse PS/.*" + } + } + ] + } + } + }, + { + "id": "crypto_api_init", + "title": "Initializing Cryptographic API", + "description": "Extracts log messages indicating that the cryptographic API is being initialized.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Initializing Cryptographic API", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/parser.ts new file mode 100644 index 000000000000..272c3ba4aa7f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/parser.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const [_, month, day, hour, minute, second] = match; + const year = new Date().getFullYear(); // Assume current year + const dateStr = `${year} ${month} ${day} ${hour}:${minute}:${second}`; + const date = moment.utc(dateStr, 'YYYY MMM DD HH:mm:ss'); + return date.valueOf(); // Return epoch milliseconds + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const date = moment.utc(timestamp); + const newTimestamp = date.format('MMM DD HH:mm:ss'); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/queries.json new file mode 100644 index 000000000000..d6af27500bf6 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/queries.json @@ -0,0 +1,457 @@ +{ + "queries": [ + { + "id": "startup_delay_hci_reset", + "title": "Startup Delay HCI Reset", + "description": "Returns log messages related to BroadcomBluetoothHostController SetupController delaying HCI reset.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "BroadcomBluetoothHostController SetupController Delay HCI Reset", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_completed_matched_device", + "title": "Startup Completed Matched Device", + "description": "Returns log messages related to BroadcomBluetoothHostControllerUSBTransport start completed and matched on device.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "BroadcomBluetoothHostControllerUSBTransport start Completed matched on Device", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_calling_register_service", + "title": "Startup Calling Register Service", + "description": "Returns log messages related to IOBluetoothFamily calling registerService during ProcessBluetoothTransportShowsUpActionWL.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "IOBluetoothFamily ProcessBluetoothTransportShowsUpActionWL calling registerService", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_connected_transport_successfully", + "title": "Startup Connected to Transport Successfully", + "description": "Returns log messages related to IOBluetoothFamily successfully connecting to the transport during ProcessBluetoothTransportShowsUpActionWL.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "IOBluetoothFamily ProcessBluetoothTransportShowsUpActionWL Connected to the transport successfully", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_completed_result_true", + "title": "Startup Completed Result TRUE", + "description": "Returns log messages related to IOBluetoothHostControllerUSBTransport start completed with result TRUE.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "IOBluetoothHostControllerUSBTransport start completed result TRUE", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "init_host_controller_published", + "title": "INIT Host Controller Published", + "description": "Returns log messages related to the initialization of the host controller being published.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "INIT Host controller is published", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_import_bailout", + "title": "Error Import Bailout", + "description": "Returns log messages related to ImportBailout error asking to exit for Diskarb.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "ImportBailout.Error Asked to exit for Diskarb", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "warning_sdef_argument", + "title": "Warning SDEF Argument", + "description": "Returns log messages related to sdef warning for argument of command in Microsoft Word Suite.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "sdef warning for argument of command can continue previous list in suite Microsoft Word Suite is not a valid type name", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "warning_dns_config_service", + "title": "Warning DNS Config Service", + "description": "Returns log messages related to DNS config service posix failing to read DnsConfig.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "WARNING dns_config_service_posix.cc Failed to read DnsConfig", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_bluetooth_hid_device_controller", + "title": "Error Bluetooth HID Device Controller", + "description": "Returns log messages related to BluetoothHIDDeviceController error of not finding the disconnected object.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "BluetoothHIDDeviceController ERROR Could not find the disconnected object", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_carddavplugin_offline", + "title": "Error CardDAVPlugin Offline", + "description": "Returns log messages related to CardDAVPlugin error indicating the internet connection appears to be offline.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "CardDAVPlugin-ERROR getPrincipalInfo Error Domain NSURLErrorDomain Code The Internet connection appears to be offline", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_carddavplugin_timeout", + "title": "Error CardDAVPlugin Timeout", + "description": "Returns log messages related to CardDAVPlugin error indicating the request timed out.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "CardDAVPlugin-ERROR getPrincipalInfo Error Domain NSURLErrorDomain Code The request timed out", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_apple_device_management_hid_event_service", + "title": "Error Apple Device Management HID Event Service", + "description": "Returns log messages related to AppleDeviceManagementHIDEventService error of not making a string from connection notification key.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "HID ATC Error AppleDeviceManagementHIDEventService start Could not make a string from out connection notification key", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_timed_out", + "title": "Error Timed Out", + "description": "Returns log messages related to errors indicating a timeout.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": ".*ERROR.*timed out after.*" + } + } + ] + } + } + }, + { + "id": "error_media_validator_unrecognized_codec", + "title": "Error Media Validator Unrecognized Codec", + "description": "Returns log messages related to MediaValidator unrecognized codec failing codec specific check.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "MediaValidator Unrecognized codec Failed codec specific check", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_media_validator_lookup_codec_support", + "title": "Error Media Validator Lookup Codec Support", + "description": "Returns log messages related to MediaValidator mv_LookupCodecSupport unrecognized codec.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "MediaValidator mv_LookupCodecSupport Unrecognized codec", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_daemon_connection_invalidated", + "title": "Error Daemon Connection Invalidated", + "description": "Returns log messages related to daemon connection being invalidated.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Daemon connection invalidated", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_db_no_such_table", + "title": "Error DB No Such Table", + "description": "Returns log messages related to database error indicating no such table.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "DB Error no such table", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_core_drag_remove_receive_handler", + "title": "Error Core Drag Remove Receive Handler", + "description": "Returns log messages related to error in CoreDragRemoveReceiveHandler.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error in CoreDragRemoveReceiveHandler", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_core_drag_remove_tracking_handler", + "title": "Error Core Drag Remove Tracking Handler", + "description": "Returns log messages related to error in CoreDragRemoveTrackingHandler.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error in CoreDragRemoveTrackingHandler", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_iconservicesagent", + "title": "Error IconServicesAgent", + "description": "Returns log messages related to error returned from iconservicesagent.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Error returned from iconservicesagent", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_service_exited_abnormal_code", + "title": "Error Service Exited Abnormal Code", + "description": "Returns log messages related to service exiting with abnormal code.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Service exited with abnormal code", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "warning_hibernate_page_list_setall", + "title": "Warning Hibernate Page List SetAll", + "description": "Returns log messages related to hibernate_page_list_setall skipping xpmapped pages.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "WARNING hibernate_page_list_setall skipped xpmapped pages", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "warning_type1_font_data", + "title": "Warning Type1 Font Data", + "description": "Returns log messages related to Type1 font data not being in the correct format required by the Adobe Type Font Format specification.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "WARNING Type1 font data isn't in the correct format required by the Adobe Type Font Format specification", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/parser.ts new file mode 100644 index 000000000000..0a2cccec4656 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/parser.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const timestampRegex = /(\w{3})\s+(\d{1,2})\s+(\d{2}:\d{2}:\d{2})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(timestampRegex); + if (match) { + const [_, month, day, time] = match; + const dateString = `${month} ${day} ${time}`; + const date = moment.utc(dateString, 'MMM DD HH:mm:ss'); + return date.valueOf(); + } + throw new Error('Invalid log line format'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newDate = moment.utc(timestamp); + const newTimestamp = newDate.format('MMM DD HH:mm:ss'); + return logLine.replace(timestampRegex, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/queries.json new file mode 100644 index 000000000000..8d3df310949c --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/queries.json @@ -0,0 +1,517 @@ +{ + "queries": [ + { + "id": "accepted_password", + "title": "Accepted password for SSH", + "description": "Logs indicating a successful SSH login using a password.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Accepted password for from port ssh2", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "connection_closed_preauth", + "title": "Connection closed by preauth", + "description": "Logs indicating a connection closed by the client before authentication.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Connection closed by [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "no_identification_string", + "title": "No identification string received", + "description": "Logs indicating that the server did not receive an identification string from the client.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Did not receive identification string from", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "too_many_auth_failures_admin", + "title": "Too many authentication failures for admin", + "description": "Logs indicating too many authentication failures for the admin user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Disconnecting: Too many authentication failures for admin [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "too_many_auth_failures_root", + "title": "Too many authentication failures for root", + "description": "Logs indicating too many authentication failures for the root user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Disconnecting: Too many authentication failures for root [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "auth_fail_jsch_exception", + "title": "Auth fail JSchException", + "description": "Logs indicating an authentication failure with a JSchException.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error: Received disconnect from com.jcraft.jsch.JSchException: Auth fail [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "no_more_auth_methods", + "title": "No more user authentication methods", + "description": "Logs indicating no more user authentication methods are available.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error: Received disconnect from No more user authentication methods available. [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "failed_none_invalid_user", + "title": "Failed none for invalid user", + "description": "Logs indicating a failed authentication attempt for an invalid user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Failed none for invalid user from port ssh2", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "failed_password", + "title": "Failed password", + "description": "Logs indicating a failed password authentication attempt.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Failed password for from port ssh2", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "failed_password_invalid_user", + "title": "Failed password for invalid user", + "description": "Logs indicating a failed password authentication attempt for an invalid user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Failed password for invalid user from port ssh2", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "write_failed_connection_reset", + "title": "Write failed: Connection reset by peer", + "description": "Logs indicating a connection reset by peer error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "fatal: Write failed: Connection reset by peer [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "invalid_user_preauth", + "title": "Invalid user preauth", + "description": "Logs indicating an invalid user authentication request before authentication.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "input_userauth_request: invalid user [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "invalid_user", + "title": "Invalid user", + "description": "Logs indicating an invalid user authentication attempt.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Invalid user from", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "message_repeated", + "title": "Message repeated", + "description": "Logs indicating repeated messages of failed password attempts for root.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "message repeated times: [ Failed password for root from port ]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_auth_failure", + "title": "PAM authentication failure", + "description": "Logs indicating a PAM authentication failure.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PAM more authentication failure; logname= uid= euid= tty=ssh ruser= rhost=", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_auth_failures", + "title": "PAM authentication failures", + "description": "Logs indicating multiple PAM authentication failures.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PAM more authentication failures; logname= uid= euid= tty=ssh ruser= rhost=", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_auth_failures_root", + "title": "PAM authentication failures for root", + "description": "Logs indicating multiple PAM authentication failures for the root user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PAM more authentication failures; logname= uid= euid= tty=ssh ruser= rhost= user=root", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_ignoring_max_retries", + "title": "PAM ignoring max retries", + "description": "Logs indicating that the PAM service is ignoring max retries.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "PAM service(sshd) ignoring max retries;", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_unix_auth_failure", + "title": "PAM Unix authentication failure", + "description": "Logs indicating a PAM Unix authentication failure.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pam_unix(sshd:auth): authentication failure; logname= uid= euid= tty=ssh ruser= rhost=", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_unix_auth_failure_user", + "title": "PAM Unix authentication failure for user", + "description": "Logs indicating a PAM Unix authentication failure for a specific user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pam_unix(sshd:auth): authentication failure; logname= uid= euid= tty=ssh ruser= rhost= user=", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_unix_check_pass", + "title": "PAM Unix check pass", + "description": "Logs indicating a PAM Unix check pass for an unknown user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pam_unix(sshd:auth): check pass; user unknown", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_unix_session_closed", + "title": "PAM Unix session closed", + "description": "Logs indicating a PAM Unix session closed for a user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pam_unix(sshd:session): session closed for user", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "pam_unix_session_opened", + "title": "PAM Unix session opened", + "description": "Logs indicating a PAM Unix session opened for a user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "pam_unix(sshd:session): session opened for user by (uid=)", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "received_disconnect_bye_bye", + "title": "Received disconnect: Bye Bye", + "description": "Logs indicating a received disconnect with the message 'Bye Bye'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Received disconnect from Bye Bye [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "received_disconnect_user_request", + "title": "Received disconnect: User request", + "description": "Logs indicating a received disconnect due to user request.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Received disconnect from Closed due to user request. [preauth]", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "received_disconnect_disconnected_by_user", + "title": "Received disconnect: Disconnected by user", + "description": "Logs indicating a received disconnect by the user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Received disconnect from disconnected by user", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "reverse_mapping_failed", + "title": "Reverse mapping failed", + "description": "Logs indicating a failed reverse mapping check, suggesting a possible break-in attempt.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "reverse mapping checking getaddrinfo for failed - POSSIBLE BREAK-IN ATTEMPT!", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/parser.ts new file mode 100644 index 000000000000..e18108b881dc --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/parser.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const timestamp = match[1]; + return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss.SSS').valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss.SSS'); + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/queries.json new file mode 100644 index 000000000000..1442d25a8541 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/queries.json @@ -0,0 +1,207 @@ +{ + "queries": [ + { + "id": "terminate_instance", + "title": "E11: Terminating instance", + "description": "Returns logs where an instance is being terminated.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Terminating instance", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "vm_paused", + "title": "E20: VM Paused (Lifecycle Event)", + "description": "Returns logs where a VM has been paused.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "VM Paused Lifecycle Event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "vm_resumed", + "title": "E21: VM Resumed (Lifecycle Event)", + "description": "Returns logs where a VM has been resumed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "VM Resumed Lifecycle Event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "vm_started", + "title": "E22: VM Started (Lifecycle Event)", + "description": "Returns logs where a VM has been started.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "VM Started Lifecycle Event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "vm_stopped", + "title": "E23: VM Stopped (Lifecycle Event)", + "description": "Returns logs where a VM has been stopped.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "VM Stopped Lifecycle Event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "disk_limit_not_specified", + "title": "E6: disk limit not specified, defaulting to unlimited", + "description": "Returns logs where the disk limit was not specified and defaulted to unlimited.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "disk limit not specified defaulting to unlimited", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "sync_power_state_pending_task", + "title": "E7: During sync_power_state the instance has a pending task (spawning). Skip.", + "description": "Returns logs where an instance has a pending task during sync_power_state.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "During sync_power_state the instance has a pending task spawning Skip", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "vcpu_limit_not_specified", + "title": "E19: vcpu limit not specified, defaulting to unlimited", + "description": "Returns logs where the vcpu limit was not specified and defaulted to unlimited.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "vcpu limit not specified defaulting to unlimited", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "http_exception_no_instances", + "title": "E33: HTTP exception thrown: No instances found for any event", + "description": "Returns logs where an HTTP exception was thrown due to no instances found for any event.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "HTTP exception thrown No instances found for any event", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "instance_sync_host_mismatch", + "title": "E40: The instance sync for host '<*>' did not match. Re-created its InstanceList.", + "description": "Returns logs where the instance sync for a host did not match and its InstanceList was re-created.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "The instance sync for host .* did not match Re-created its InstanceList" + } + } + ] + } + } + }, + { + "id": "sync_instance_power_states_mismatch", + "title": "E43: While synchronizing instance power states, found <*> instances in the database and <*> instances on the hypervisor.", + "description": "Returns logs where there was a mismatch in the number of instances found in the database and on the hypervisor during instance power state synchronization.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "While synchronizing instance power states found .* instances in the database and .* instances on the hypervisor" + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/parser.ts new file mode 100644 index 000000000000..f7fb9046a7ed --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/parser.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /\[(\d{1,2})\.(\d{1,2}) (\d{1,2}):(\d{2}):(\d{2})\]/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const [_, month, day, hour, minute, second] = match; + const dateString = `${new Date().getFullYear()}-${month}-${day} ${hour}:${minute}:${second}`; + const date = moment.utc(dateString, 'YYYY-MM-DD HH:mm:ss'); + return date.valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newDate = moment.utc(timestamp); + const newTimestamp = `[${newDate.format('MM.DD HH:mm:ss')}]`; + return logLine.replace(TIMESTAMP_REGEX, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/queries.json new file mode 100644 index 000000000000..2455631d2b70 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/queries.json @@ -0,0 +1,153 @@ +{ + "queries": [ + { + "id": "open_through_proxy_socks5", + "title": "Open through proxy SOCKS5", + "description": "Returns log messages where the connection is opened through a SOCKS5 proxy.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "open through proxy SOCKS5", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "open_through_proxy_https", + "title": "Open through proxy HTTPS", + "description": "Returns log messages where the connection is opened through an HTTPS proxy.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "open through proxy HTTPS", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_could_not_connect_to_proxy_resolve_error", + "title": "Error: Could not connect to proxy - resolve error", + "description": "Returns log messages where there is an error connecting to a proxy due to a resolution error.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error Could not connect to proxy Could not resolve error", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_could_not_connect_to_proxy_connection_attempt_failed", + "title": "Error: Could not connect to proxy - connection attempt failed", + "description": "Returns log messages where there is an error connecting to a proxy due to a failed connection attempt.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error Could not connect to proxy connection attempt failed with error", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_could_not_connect_through_proxy_status_code", + "title": "Error: Could not connect through proxy - status code", + "description": "Returns log messages where there is an error connecting through a proxy due to a status code issue.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error Could not connect through proxy Proxy server cannot establish a connection with the target status code", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_could_not_connect_through_proxy_closed_connection", + "title": "Error: Could not connect through proxy - closed connection", + "description": "Returns log messages where there is an error connecting through a proxy due to the proxy closing the connection unexpectedly.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error Could not connect through proxy Proxy closed the connection unexpectedly", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_connection_request_canceled", + "title": "Error: Connection request canceled", + "description": "Returns log messages where a connection request was canceled before completion.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "error A connection request was canceled before the completion", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "close_bytes_sent_received_lifetime", + "title": "Close, bytes sent/received, lifetime", + "description": "Returns log messages where a connection is closed, showing bytes sent, bytes received, and the connection lifetime.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": ".* close, .* bytes.*sent, .* bytes.*received, lifetime .*" + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/parser.ts new file mode 100644 index 000000000000..5888c6bec429 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/parser.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX = /^(\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const timestampString = match[1]; + return moment.utc(timestampString, 'YY/MM/DD HH:mm:ss').valueOf(); + } + throw new Error('Timestamp not found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const formattedTimestamp = moment.utc(timestamp).format('YY/MM/DD HH:mm:ss'); + return logLine.replace(TIMESTAMP_REGEX, formattedTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/queries.json new file mode 100644 index 000000000000..9d95e1d976a8 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/queries.json @@ -0,0 +1,280 @@ +{ + "queries": [ + { + "id": "memory_store_started", + "title": "MemoryStore Started", + "description": "Identifies logs where MemoryStore started with a specified capacity.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "MemoryStore started capacity", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "registered_signal_handlers", + "title": "Registered Signal Handlers", + "description": "Identifies logs where signal handlers for TERM, HUP, and INT were registered.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Registered signal handlers TERM HUP INT", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "remoting_started", + "title": "Remoting Started", + "description": "Identifies logs where remoting started and is listening on specified addresses.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Remoting started; listening on addresses :\\[akka\\.tcp://.*\\]" + } + } + ] + } + } + }, + { + "id": "slf4jlogger_started", + "title": "Slf4jLogger Started", + "description": "Identifies logs where Slf4jLogger started.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Slf4jLogger started", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "starting_executor", + "title": "Starting Executor", + "description": "Identifies logs where an executor is starting on a specified host.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting executor ID host", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "starting_remoting", + "title": "Starting Remoting", + "description": "Identifies logs where remoting is starting.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting remoting", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "successfully_registered_with_driver", + "title": "Successfully Registered with Driver", + "description": "Identifies logs where the system successfully registered with the driver.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Successfully registered driver", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "started_netty_block_transfer_service", + "title": "Started Netty Block Transfer Service", + "description": "Identifies logs where the Netty Block Transfer Service started on a specified port.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Successfully started service 'org\\.apache\\.spark\\.network\\.netty\\.NettyBlockTransferService' on port .*\\." + } + } + ] + } + } + }, + { + "id": "started_spark_executor_actor_system", + "title": "Started Spark Executor Actor System", + "description": "Identifies logs where the Spark Executor Actor System started on a specified port.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Successfully started service 'sparkExecutorActorSystem' on port .*\\." + } + } + ] + } + } + }, + { + "id": "partition_rdd_not_found", + "title": "Partition RDD Not Found", + "description": "Identifies logs where a partition RDD was not found and is being computed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Partition rdd not found computing", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mapred_job_id_deprecated", + "title": "Mapred Job ID Deprecated", + "description": "Identifies logs where the mapred.job.id is deprecated and suggests using mapreduce.job.id instead.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mapred.job.id deprecated mapreduce.job.id", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mapred_task_id_deprecated", + "title": "Mapred Task ID Deprecated", + "description": "Identifies logs where the mapred.task.id is deprecated and suggests using mapreduce.task.attempt.id instead.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mapred.task.id deprecated mapreduce.task.attempt.id", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mapred_task_is_map_deprecated", + "title": "Mapred Task Is Map Deprecated", + "description": "Identifies logs where the mapred.task.is.map is deprecated and suggests using mapreduce.task.ismap instead.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mapred.task.is.map deprecated mapreduce.task.ismap", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mapred_task_partition_deprecated", + "title": "Mapred Task Partition Deprecated", + "description": "Identifies logs where the mapred.task.partition is deprecated and suggests using mapreduce.task.partition instead.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mapred.task.partition deprecated mapreduce.task.partition", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "mapred_tip_id_deprecated", + "title": "Mapred Tip ID Deprecated", + "description": "Identifies logs where the mapred.tip.id is deprecated and suggests using mapreduce.task.id instead.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "mapred.tip.id deprecated mapreduce.task.id", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/parser.ts new file mode 100644 index 000000000000..7260856015a7 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/parser.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const TIMESTAMP_REGEX = /^- (\d+) (\d{4}\.\d{2}\.\d{2} .*)/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const timestamp = parseInt(match[1], 10); + return timestamp * 1000; // Convert to milliseconds + } + throw new Error('Timestamp not found'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const match = logLine.match(TIMESTAMP_REGEX); + if (match) { + const newTimestamp = Math.floor(timestamp / 1000); // Convert to seconds + return `- ${newTimestamp} ${match[2]}`; + } + throw new Error('Timestamp not found'); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/queries.json new file mode 100644 index 000000000000..e721ae1a3b2c --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/queries.json @@ -0,0 +1,360 @@ +{ + "queries": [ + { + "id": "gmetad_startup", + "title": "Gmetad Startup Succeeded", + "description": "Identifies logs where gmetad startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "gmetad startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "portmap_startup", + "title": "Portmap Startup Succeeded", + "description": "Identifies logs where portmap startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "portmap startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "rpc_statd_startup", + "title": "RPC.Statd Startup Succeeded", + "description": "Identifies logs where rpc.statd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "rpc.statd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "syslog_ng_startup", + "title": "Syslog-NG Startup Succeeded", + "description": "Identifies logs where syslog-ng startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "syslog-ng startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "xinetd_startup", + "title": "Xinetd Startup Succeeded", + "description": "Identifies logs where xinetd startup succeeded.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "xinetd startup succeeded", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "bind_port_failed", + "title": "Bind to Port Failed", + "description": "Identifies logs where binding to a port failed due to address already in use.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "error: Bind to port .* on .* failed: Address already in use." + } + } + ] + } + } + }, + { + "id": "local_disconnected", + "title": "Local Disconnected", + "description": "Identifies logs where a local connection was disconnected.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Local disconnected: Connection closed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "connection_lost", + "title": "Connection Lost", + "description": "Identifies logs where a connection was lost.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "connection lost: 'Connection closed.'", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cannot_open_file", + "title": "Cannot Open File", + "description": "Identifies logs where a file could not be opened for writing.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Cannot open file /dev/logsurfer for writing (No such file or directory)", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "resolve_data_source_failed", + "title": "Failed to Resolve Data Source", + "description": "Identifies logs where resolving a data source name failed.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Warning: we failed to resolve data source name .*" + } + } + ] + } + } + }, + { + "id": "resolve_trusted_host_failed", + "title": "Failed to Resolve Trusted Host", + "description": "Identifies logs where resolving a trusted host name failed.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Warning: we failed to resolve trusted host name .*" + } + } + ] + } + } + }, + { + "id": "wait_for_ready_failed", + "title": "Wait for Ready Failed", + "description": "Identifies logs where waiting for ready state before probe failed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Wait for ready failed before probe !", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "cache_data_failed", + "title": "Cache Data Request Failed", + "description": "Identifies logs where asking for cache data failed.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "asking for cache data failed", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "probe_failed", + "title": "Probe Failed", + "description": "Identifies logs where probing vesafb0 failed with an error.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "probe of vesafb0 failed with error .*" + } + } + ] + } + } + }, + { + "id": "dhcpdiscover_no_free_leases", + "title": "DHCPDISCOVER No Free Leases", + "description": "Identifies logs where a DHCPDISCOVER request found no free leases.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "DHCPDISCOVER from .* via eth1: network A_net: no free leases" + } + } + ] + } + } + }, + { + "id": "dhcprequest_unknown_lease", + "title": "DHCPREQUEST Unknown Lease", + "description": "Identifies logs where a DHCPREQUEST was made for an unknown lease.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "DHCPREQUEST for .* \\(.*\\) from .* via eth1: unknown lease .*" + } + } + ] + } + } + }, + { + "id": "unqualified_host_name", + "title": "Unqualified Host Name", + "description": "Identifies logs where an unqualified host name was unknown, leading to a retry.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "My unqualified host name \\(.*\\) unknown; sleeping for retry" + } + } + ] + } + } + }, + { + "id": "unable_to_qualify_domain", + "title": "Unable to Qualify Domain", + "description": "Identifies logs where the system was unable to qualify its own domain name.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "unable to qualify my own domain name \\(.*\\) -- using short name" + } + } + ] + } + } + }, + { + "id": "session_closed_root", + "title": "Session Closed for Root", + "description": "Identifies logs where a session was closed for the root user.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "session closed for user root", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "session_opened_root", + "title": "Session Opened for Root", + "description": "Identifies logs where a session was opened for the root user by uid=0.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "session opened for user root by (uid=0)", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/parser.ts new file mode 100644 index 000000000000..b5ae351fc0f0 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/parser.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const TIMESTAMP_REGEX_1 = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/; +const TIMESTAMP_REGEX_2 = /\d{4}\/\d{1,2}\/\d{1,2}:\d{2}:\d{2}:\d{2}\.\d{3}/; + +export function getTimestamp(logLine: string): number { + const match1 = logLine.match(TIMESTAMP_REGEX_1); + if (match1) { + return moment.utc(match1[0], 'YYYY-MM-DD HH:mm:ss').valueOf(); + } + + const match2 = logLine.match(TIMESTAMP_REGEX_2); + if (match2) { + return moment.utc(match2[0], 'YYYY/M/D:HH:mm:ss.SSS').valueOf(); + } + + throw new Error('No valid timestamp found in log line'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const match1 = logLine.match(TIMESTAMP_REGEX_1); + if (match1) { + const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'); + return logLine.replace(TIMESTAMP_REGEX_1, newTimestamp); + } + + const match2 = logLine.match(TIMESTAMP_REGEX_2); + if (match2) { + const newTimestamp = moment.utc(timestamp).format('YYYY/M/D:HH:mm:ss.SSS'); + return logLine.replace(TIMESTAMP_REGEX_2, newTimestamp); + } + + throw new Error('No valid timestamp found in log line'); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/queries.json new file mode 100644 index 000000000000..1f5c28c6019e --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/queries.json @@ -0,0 +1,268 @@ +{ + "queries": [ + { + "id": "startup_trustedinstaller_main_loop", + "title": "Startup TrustedInstaller Main Loop", + "description": "Returns logs where the TrustedInstaller main loop is starting.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting TrustedInstaller main loop", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_trustedinstaller_finalization", + "title": "Startup TrustedInstaller Finalization", + "description": "Returns logs where the TrustedInstaller finalization is starting.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting TrustedInstaller finalization", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_trustedinstaller_initialization", + "title": "Startup TrustedInstaller Initialization", + "description": "Returns logs where the TrustedInstaller initialization is starting.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting TrustedInstaller initialization", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "trustedinstaller_service_start", + "title": "TrustedInstaller Service Start", + "description": "Returns logs where the TrustedInstaller service starts successfully.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "TrustedInstaller service starts successfully", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "shutdown_trustedinstaller_main_loop", + "title": "Shutdown TrustedInstaller Main Loop", + "description": "Returns logs where the TrustedInstaller main loop is ending.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Ending TrustedInstaller main loop", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "shutdown_trustedinstaller_finalization", + "title": "Shutdown TrustedInstaller Finalization", + "description": "Returns logs where the TrustedInstaller finalization is ending.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Ending TrustedInstaller finalization", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "shutdown_trustedinstaller_initialization", + "title": "Shutdown TrustedInstaller Initialization", + "description": "Returns logs where the TrustedInstaller initialization is ending.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Ending TrustedInstaller initialization", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_manifest_invalid_item", + "title": "Error Manifest Invalid Item", + "description": "Returns logs where there is an error related to an invalid item in the manifest.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Expecting attribute name \\[HRESULT = .* - CBS_E_MANIFEST_INVALID_ITEM\\]" + } + } + ] + } + } + }, + { + "id": "error_backup_log_cab", + "title": "Error Backup Log Cab", + "description": "Returns logs where there is an error creating a backup log cab.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Failed to create backup log cab\\. \\[HRESULT = .* - ERROR_INVALID_FUNCTION\\]" + } + } + ] + } + } + }, + { + "id": "error_next_element", + "title": "Error Next Element", + "description": "Returns logs where there is an error getting the next element.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Failed to get next element \\[HRESULT = .* - CBS_E_MANIFEST_INVALID_ITEM\\]" + } + } + ] + } + } + }, + { + "id": "error_open_package", + "title": "Error Open Package", + "description": "Returns logs where there is an error opening a package internally.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Failed to internally open package\\. \\[HRESULT = .* - CBS_E_INVALID_PACKAGE\\]" + } + } + ] + } + } + }, + { + "id": "error_sqm_sample_upload", + "title": "Error SQM Sample Upload", + "description": "Returns logs where there is an error starting a standard sample upload.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "SQM: Failed to start standard sample upload\\. \\[HRESULT = .* - E_FAIL\\]" + } + } + ] + } + } + }, + { + "id": "error_sqm_upload_file_pattern", + "title": "Error SQM Upload File Pattern", + "description": "Returns logs where there is an error starting an upload with a specific file pattern.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "SQM: Failed to start upload with file pattern: .* flags: .* \\[HRESULT = .* - E_FAIL\\]" + } + } + ] + } + } + }, + { + "id": "warning_sqm_unsent_reports", + "title": "Warning SQM Unsent Reports", + "description": "Returns logs where there is a warning about failing to upload all unsent reports.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "SQM: Warning: Failed to upload all unsent reports\\. \\[HRESULT = .* - E_FAIL\\]" + } + } + ] + } + } + }, + { + "id": "warning_unrecognized_package_attribute", + "title": "Warning Unrecognized Package Attribute", + "description": "Returns logs where there is a warning about an unrecognized packageExtended attribute.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Warning Unrecognized packageExtended attribute", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/parser.ts new file mode 100644 index 000000000000..45b1d96d4a48 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/parser.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +const timestampRegex = /(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}),(\d{3})/; + +export function getTimestamp(logLine: string): number { + const match = logLine.match(timestampRegex); + if (match) { + const [_, year, month, day, hour, minute, second, millisecond] = match; + const timestamp = `${year}-${month}-${day} ${hour}:${minute}:${second}.${millisecond}`; + return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss.SSS').valueOf(); + } + throw new Error('Invalid log line format'); +} + +export function replaceTimestamp(logLine: string, timestamp: number): string { + const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss,SSS'); + return logLine.replace(timestampRegex, newTimestamp); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/queries.json b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/queries.json new file mode 100644 index 000000000000..17e20fd4ce32 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/queries.json @@ -0,0 +1,201 @@ +{ + "queries": [ + { + "id": "shutdown_message", + "title": "Shutdown Message", + "description": "Identifies log messages indicating a shutdown event with the keyword 'GOODBYE'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "GOODBYE", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "error_message", + "title": "Error Message", + "description": "Identifies log messages indicating an error related to opening a channel at an election address.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Cannot open channel to .* at election address /.*:.*" + } + } + ] + } + } + }, + { + "id": "exception_message", + "title": "Exception Message", + "description": "Identifies log messages indicating an 'end of stream' exception.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "caught end of stream exception", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "connection_broken_message", + "title": "Connection Broken Message", + "description": "Identifies log messages indicating a broken connection for a specific ID.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Connection broken for id .*, my id = .*, error =" + } + } + ] + } + } + }, + { + "id": "ioexception_message", + "title": "IOException Message", + "description": "Identifies log messages indicating a session close due to a java.io.IOException with the message 'ZooKeeperServer not running'.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Exception causing close of session .* due to java.io.IOException: ZooKeeperServer not running" + } + } + ] + } + } + }, + { + "id": "session_expiry_message", + "title": "Session Expiry Message", + "description": "Identifies log messages indicating a session expiry due to a timeout.", + "query": { + "bool": { + "filter": [ + { + "regexp": { + "message": "Expiring session .*, timeout of .*ms exceeded" + } + } + ] + } + } + }, + { + "id": "startup_message", + "title": "Startup Message", + "description": "Identifies log messages indicating a startup event with the keyword 'starting up'.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "starting up", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "shutdown_complete_message", + "title": "Shutdown Complete Message", + "description": "Identifies log messages indicating the completion of a shutdown process.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "shutdown of request processor complete", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "startup_quorum_peer_message", + "title": "Startup Quorum Peer Message", + "description": "Identifies log messages indicating the startup of a quorum peer.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Starting quorum peer", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "unexpected_exception_shutdown_message", + "title": "Unexpected Exception Causing Shutdown Message", + "description": "Identifies log messages indicating an unexpected exception causing a shutdown while the socket is still open.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Unexpected exception causing shutdown while sock still open", + "operator": "AND" + } + } + } + ] + } + } + }, + { + "id": "unexpected_exception_message", + "title": "Unexpected Exception Message", + "description": "Identifies log messages indicating an unexpected exception.", + "query": { + "bool": { + "filter": [ + { + "match": { + "message": { + "query": "Unexpected Exception", + "operator": "AND" + } + } + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/constants.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/constants.ts new file mode 100644 index 000000000000..0ab81f071f3d --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/constants.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Path from 'path'; +import { REPO_ROOT } from '@kbn/repo-info'; + +export const LOGHUB_DIR = Path.join(REPO_ROOT, '../loghub'); +export const LOGHUB_REPO = 'https://github.com/logpai/loghub.git'; +export const LOGHUB_PARSER_DIR = Path.join(__dirname, '../parsers'); diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/create_openai_client.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/create_openai_client.ts new file mode 100644 index 000000000000..eb3045dd7fdc --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/create_openai_client.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AzureOpenAI } from 'openai'; + +export interface OpenAIClient extends AzureOpenAI { + model: string; +} + +export function createOpenAIClient(): OpenAIClient { + const apiKey = process.env.AZURE_OPENAI_API_KEY; + const endpoint = process.env.AZURE_OPENAI_ENDPOINT; + const deployment = process.env.AZURE_OPENAI_DEPLOYMENT; + const apiVersion = process.env.AZURE_OPENAI_API_VERSION ?? '2025-01-01-preview'; + + const openAIClient = new AzureOpenAI({ + apiKey, + endpoint, + deployment, + apiVersion, + }) as OpenAIClient; + const model = process.env.AZURE_OPENAI_MODEL || `gpt-4o`; + + // chat complete methods require a model parameter, + // so we expose it on the client in order to ~fully + // encapsulate config + openAIClient.model = model; + + return openAIClient; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_loghub_repo.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_loghub_repo.ts new file mode 100644 index 000000000000..1977a6bc8e18 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_loghub_repo.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ToolingLog } from '@kbn/tooling-log'; +import { promises as Fs } from 'fs'; +import simpleGit from 'simple-git'; +import { LOGHUB_DIR, LOGHUB_REPO } from './constants'; + +export async function ensureLoghubRepo({ log }: { log: ToolingLog }) { + const dirExists = await Fs.stat(LOGHUB_DIR) + .then((stat) => stat.isDirectory()) + .catch(() => false); + + if (!dirExists) { + log.info(`Directory "${LOGHUB_DIR}" does not exist. Cloning repository...`); + await simpleGit().clone(LOGHUB_REPO, LOGHUB_DIR, ['--depth', '1']); + } + + const repoGit = simpleGit(LOGHUB_DIR); + + log.debug(`Fetching from logai/loghub`); + + await repoGit.fetch(); + + const defaultBranch = + (await repoGit.revparse(['--abbrev-ref', 'origin/HEAD'])).replace('origin/', '') || 'master'; + + const currentBranch = (await repoGit.revparse(['--abbrev-ref', 'HEAD'])) || defaultBranch; + + if (currentBranch !== defaultBranch) { + log.info(`Checking out ${defaultBranch}`); + + await repoGit.checkout(defaultBranch); + } + + const status = await repoGit.status(); + if (status.behind && status.behind > 0) { + log.info(`Local branch is behind by ${status.behind} commit(s); pulling changes.`); + await repoGit.pull('origin', defaultBranch); + } else { + log.debug(`Local branch is up-to-date; no pull needed.`); + } +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_parser.ts new file mode 100644 index 000000000000..71667b81a8cd --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_parser.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import pRetry from 'p-retry'; +import { OpenAIClient } from './create_openai_client'; +import { type LoghubSystem } from './read_loghub_system_files'; +import { getParserFilename, writeFileRecursively } from './utils'; +import { validateParser } from './validate_parser'; + +async function generateParser({ + openAIClient, + system, + error, + log, +}: { + openAIClient: OpenAIClient; + system: LoghubSystem; + error?: Error; + log: ToolingLog; +}): Promise { + log.info(`Attempting to generate a parser for ${system.name}`); + + const systemPrompt = `You are given a system's documentation and log files. + + Your goal is to write a TypeScript files that exports two functions: + + - \`getTimestamp ( logLine:string ):number\`: extract the timestamp + from the logline and return it as epoch milliseconds + - \`replaceTimestamp ( logLine:string, timestamp:number ):string\`: + replace the timestamp with the new timestamp, in the format that is + used in the log line. + + Generally, you will want to generate + a regular expression that can be used to A) extract the values + to parse the date, B) replace the original timestamp with the + formatted injected timestamp. + + You can use \`moment\`, but not any other libraries. + + Some notes: + - in some cases, leading 0s are stripped. Take this into account + when generating a regex (e.g. use \d{2,3} instead of \d{2}). + `; + + const reasoningResponse = await openAIClient.chat.completions.create({ + model: openAIClient.model, + messages: [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: `Analyze the log lines and reason about the date format + in the log lines. Find the piece of data that is likely the + timestamp. Reason about regular expressions, timezones, date + formatting/parsing from the format in the log line to and from + epoch ms. You don't have to write the actual code yet, that + happens in a follow-up, but you can write snippets. If an error + occurred, reason about how the error should be fixed. + + ${ + error + ? `# Error + The following error occurred on a previous attempt: ${error.message}` + : '' + } + + ## README.md + + ${system.readme ?? 'Empty'} + + ## Log file + + ${system.logLines.slice(0, 500).join('\n')} + `, + }, + ], + temperature: 0.2, + }); + + const analysis = reasoningResponse.choices[0].message.content; + + const output = await openAIClient.chat.completions.create({ + model: openAIClient.model, + messages: [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: `Based on previous reasoning, output the + typescript file in the following format. Make sure + any dates are parsed as UTC if the time zone is not + specified, regardless of where the script runs: + + export function getTimestamp ( logLine:string ):number { + // function implementation here + } + + export function replaceTimestamp ( logLine:string, timestamp:number ) { + // function implementation here + } + + DO NOT output anything else, including any markdown backticks, + just the file contents. The result of your output will be written + to disk directly. If you use \`moment\`, add the following import + statement to the top of the file: + + \`\`\`import moment from "moment";\`\`\` + + If you use a regex that is shared, add it to the top of the file + as a constant. + + # Reasoning + + ${analysis} + + ## README.md + + ${system.readme ?? 'Empty'} + + ## Log file + + ${system.logLines.slice(0, 500).join('\n')} + `, + }, + ], + temperature: 0.2, + }); + + const file = output.choices[0].message.content; + + if (!file) { + throw new Error(`No content received from the LLM`); + } + + return file; +} + +export async function ensureValidParser({ + openAIClient, + system, + log, +}: { + openAIClient: OpenAIClient; + system: LoghubSystem; + log: ToolingLog; +}) { + let error: Error | undefined; + + const isValid = await validateParser(system) + .then(() => true) + .catch(() => false); + + if (isValid) { + return; + } + + await pRetry( + async () => { + const file = await generateParser({ system, error, log, openAIClient }); + + await writeFileRecursively(getParserFilename(system), file); + + await validateParser(system); + }, + { retries: 5 } + ); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_queries.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_queries.ts new file mode 100644 index 000000000000..071e61dc6566 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_queries.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import pRetry from 'p-retry'; +import { OpenAIClient } from './create_openai_client'; +import { type LoghubSystem } from './read_loghub_system_files'; +import { getQueriesFilename, writeFileRecursively } from './utils'; +import { queryFileSchema, validateQueries } from './validate_queries'; + +async function generateQueries({ + openAIClient, + system, + error, + log, +}: { + openAIClient: OpenAIClient; + system: LoghubSystem; + error?: Error; + log: ToolingLog; +}): Promise { + log.info(`Attempting to generate queries for ${system.name}`); + + const systemPrompt = `You are an expert in Elasticsearch DSL, log patterns + and log analytics.`; + + const reasoningResponse = await openAIClient.chat.completions.create({ + model: openAIClient.model, + messages: [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: `Given the following log message dataset and templates, + generate queries using ES DSL. These queries should return log + messages that are "interesting", such as startup/shutdown related + messages, or warning/error/fatal messages. For each query, + use a single \`match\` or \`regexp\` query. Keep the following + things in mind: + + - Each query should return a single template. Do not group templates + together. + - Use the \`message\` field. + - In the templates, \`<*>\` refers to a _variable_ in the actual log + message. DO NOT include this in your query. Instead, just skip over + the word in your match query, or group it as a variable in your + regexp query. + - The default operator for a \`match\` query is \`OR\`. Keep this in + mind, most likely you'll want \`AND\`. Use capitalized letters. + + ${ + error + ? `# Error + The following error occurred on a previous attempt: ${error.message}` + : '' + } + # Readme + + ${system.readme} + + # Templates + + ${system.templates.join('\n')} + `, + }, + ], + temperature: 0.2, + }); + + const analysis = reasoningResponse.choices[0].message.content; + + log.verbose(`Analysis for ${system.name}: + + ${analysis}`); + + const output = await openAIClient.chat.completions.create({ + model: openAIClient.model, + messages: [ + { + role: 'system', + content: systemPrompt, + }, + { + role: 'user', + content: `Based on previous reasoning, output the + queries in a structured format. + + # Analysis + ${analysis}`, + }, + ], + tools: [ + { + function: { + name: 'output', + description: + 'Output the queries in structured data. Make sure you output the query DSL in `queries[n].query`', + strict: true, + parameters: { + type: 'object', + additionalProperties: false, + properties: { + queries: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + properties: { + id: { + type: 'string', + description: + 'The id of the query. Use [a-z_]+. Make sure the id is descriptive of the query, but keep it short.', + }, + query: { + type: 'string', + description: + 'The Elasticsearch Query DSL for the query, as serialized JSON. Wrap in { bool: { filter: [...] } }', + }, + title: { + type: 'string', + description: 'A short title for the query', + }, + description: { + type: 'string', + description: 'A human-readable description of what the query returns', + }, + }, + required: ['id', 'query', 'title', 'description'], + }, + }, + }, + required: ['queries'], + }, + }, + type: 'function', + }, + ], + tool_choice: { + function: { + name: 'output', + }, + type: 'function', + }, + temperature: 0.2, + }); + + const message = output.choices[0].message; + + const args = message.tool_calls?.[0]?.function.arguments; + + if (!args) { + throw new Error(`Expected tool call, received message: ${message.content}`); + } + + const queries = queryFileSchema.parse(JSON.parse(args)); + + return JSON.stringify(queries, null, 2); +} + +export async function ensureValidQueries({ + openAIClient, + system, + log, +}: { + openAIClient: OpenAIClient; + system: LoghubSystem; + log: ToolingLog; +}) { + let error: Error | undefined; + + const isValid = await validateQueries(system) + .then(() => true) + .catch(() => false); + + if (isValid) { + return; + } + + await pRetry( + async () => { + const file = await generateQueries({ system, error, log, openAIClient }); + + await writeFileRecursively(getQueriesFilename(system), file); + + await validateQueries(system); + }, + { + retries: 5, + onFailedAttempt(err) { + log.debug(`Error generating queries for ${system.name}`); + log.debug(err); + }, + } + ); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/get_parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/get_parser.ts new file mode 100644 index 000000000000..7c60b2e1907f --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/get_parser.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type LoghubSystem } from './read_loghub_system_files'; +import { LoghubParser } from './types'; +import { getParserFilename } from './utils'; + +export async function getParser(system: LoghubSystem): Promise { + const fileName = getParserFilename(system); + + return (await import(fileName)) as LoghubParser; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/get_queries.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/get_queries.ts new file mode 100644 index 000000000000..fb411f9ddced --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/get_queries.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type LoghubSystem } from './read_loghub_system_files'; +import { getQueriesFilename } from './utils'; +import { LoghubQuery } from './validate_queries'; + +export async function getQueries(system: LoghubSystem): Promise { + const fileName = getQueriesFilename(system); + + const { queries } = (await import(fileName)) as { queries: LoghubQuery[] }; + return queries; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/read_loghub_system_files.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/read_loghub_system_files.ts new file mode 100644 index 000000000000..111433aff55c --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/read_loghub_system_files.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { promises as Fs } from 'fs'; +import Path from 'path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { LOGHUB_DIR } from './constants'; +import { getFileOrThrow } from './utils'; + +export interface LoghubSystem { + name: string; + readme: string; + templates: string[]; + logLines: string[]; +} + +function toLines(contents: string) { + return contents + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); +} + +export async function readLoghubSystemFiles({ log }: { log: ToolingLog }): Promise { + log.debug(`Reading loghub files from ${LOGHUB_DIR}`); + + const systemFolders = await Fs.readdir(LOGHUB_DIR, { withFileTypes: true }); + + const systems = await Promise.all( + systemFolders.flatMap(async (dir): Promise => { + if (!dir.isDirectory() || dir.name.startsWith('.')) { + return []; + } + + const dirName = Path.join(LOGHUB_DIR, dir.name); + + const fileNames = await Fs.readdir(dirName); + const readmeFileName = fileNames.find((fileName) => fileName.toLowerCase() === 'readme.md'); + const logFileName = fileNames.find((fileName) => Path.extname(fileName) === '.log'); + const templateFileName = fileNames.find((fileName) => fileName.endsWith('_templates.csv')); + + if (!logFileName) { + throw new Error(`Could not find log file in ${dir.name}`); + } + + if (!templateFileName) { + throw new Error(`Could not find template file in ${dir.name}`); + } + + const [readmeFile, logFile, templateFile] = await Promise.all( + [readmeFileName, logFileName, templateFileName].map(async (fileName) => { + if (!fileName) { + return ''; + } + return getFileOrThrow(Path.join(dirName, fileName)); + }) + ); + + return [ + { + name: dir.name, + logLines: toLines(logFile), + readme: readmeFile, + templates: toLines(templateFile), + }, + ]; + }) + ); + + return systems.flat(); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/types.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/types.ts new file mode 100644 index 000000000000..b5c4518a1bdc --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface LoghubParser { + getTimestamp: (logLine: string) => number; + replaceTimestamp: (logLine: string, timestamp: number) => string; +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/utils.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/utils.ts new file mode 100644 index 000000000000..68ec531b0456 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/utils.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { promises as Fs } from 'fs'; +import Path from 'path'; +import { LOGHUB_PARSER_DIR } from './constants'; +import { type LoghubSystem } from './read_loghub_system_files'; + +export async function getFileOrThrow(filename: string): Promise { + return await Fs.readFile(filename, 'utf-8'); +} +export async function writeFileRecursively(filename: string, content: string) { + const dir = Path.dirname(filename); + + // recursive doesn't throw if the directory already exists + await Fs.mkdir(dir, { recursive: true }); + + await Fs.writeFile(filename, content, 'utf8'); +} + +export function getParserFilename(system: LoghubSystem) { + return Path.join(LOGHUB_PARSER_DIR, system.name, 'parser.ts'); +} + +export function getQueriesFilename(system: LoghubSystem) { + return Path.join(LOGHUB_PARSER_DIR, system.name, 'queries.json'); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_parser.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_parser.ts new file mode 100644 index 000000000000..fff5916eeaad --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_parser.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getParser } from './get_parser'; +import { type LoghubSystem } from './read_loghub_system_files'; +import { getFileOrThrow, getParserFilename } from './utils'; + +export async function validateParser(system: LoghubSystem): Promise { + const [{ getTimestamp, replaceTimestamp }, parserFileContents] = await Promise.all([ + getParser(system), + getFileOrThrow(getParserFilename(system)), + ]); + + let successfullyParsed = 0; + system.logLines.forEach((logLine, index) => { + try { + const timestamp = getTimestamp(logLine); + if (isNaN(timestamp)) { + throw new Error(`getTimestamp: no valid date extracted`); + } + + const next = replaceTimestamp(logLine, timestamp); + + const extracted = getTimestamp(next); + + const isEqual = extracted === timestamp; + + if (!isEqual) { + throw new Error(`replaceTimestamp: expected ${next}`); + } + } catch (error) { + error.message += ` + Line: "${logLine}" + Index: ${index} + Lines successfully parsed: ${successfullyParsed} + Example of successfully parsed line: ${system.logLines[index - 1] ?? '-'} + Source: + + ${parserFileContents}`; + throw error; + } + + successfullyParsed++; + }); +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_queries.ts b/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_queries.ts new file mode 100644 index 000000000000..a3f17c4ab90b --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/src/validate_queries.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { isEmpty } from 'lodash'; +import { LoghubSystem } from './read_loghub_system_files'; +import { getFileOrThrow, getQueriesFilename } from './utils'; + +interface DslQuery { + bool: { + filter: Array< + | { + match: { + message: + | { + query: string; + operator?: 'AND' | 'OR'; + } + | string; + }; + } + | { + regexp: { + message: string; + }; + } + >; + }; +} + +export interface LoghubQuery { + id: string; + title: string; + description: string; + query: DslQuery; +} + +export const querySchema: z.ZodSchema = z.object({ + id: z.string(), + title: z.string(), + description: z.string(), + query: z + .union([ + z.object({}, {}).passthrough(), + z.string().transform((arg) => JSON.parse(arg) as object), + ]) + .refine( + (arg): arg is Record => { + return !isEmpty(arg); + }, + { message: `Object cannot be empty` } + ) + .pipe( + z.object({ + bool: z.object({ + filter: z.array( + z.union([ + z.object({ regexp: z.object({ message: z.string() }) }), + z.object({ + match: z.object({ + message: z.union([ + z.string(), + z.object({ + query: z.string(), + operator: z + .string() + .toUpperCase() + .pipe(z.union([z.literal('AND'), z.literal('OR')])) + .optional(), + }), + ]), + }), + }), + ]) + ), + }), + }) + ), +}); + +export const queryFileSchema = z.object({ + queries: z.array(querySchema), +}); + +export function tokenize(text: string): string[] { + return text + .toLowerCase() + .split(/\W+/) + .filter((token) => token.length > 0); +} + +type StructuredInputMatchFunction = (input: { tokens: string[]; raw: string }) => boolean; + +export function createQueryMatcher(query: DslQuery): StructuredInputMatchFunction { + const validators = query.bool.filter.map((queryContainer): StructuredInputMatchFunction => { + if ('match' in queryContainer) { + const { query: q, operator = 'OR' } = + typeof queryContainer.match.message === 'string' + ? { query: queryContainer.match.message, operator: 'OR' } + : queryContainer.match.message; + + const tokens = tokenize(q); + return (input) => { + return operator === 'AND' + ? tokens.every((token) => input.tokens.includes(token)) + : tokens.some((token) => input.tokens.includes(token)); + }; + } + const regex = new RegExp(queryContainer.regexp.message); + return (input) => !!input.raw.match(regex); + }); + + return (line) => validators.every((validator) => validator(line)); +} + +function executeQuery(query: DslQuery, system: LoghubSystem) { + const matcher = createQueryMatcher(query); + + return system.logLines.filter((line) => { + const input = { tokens: tokenize(line), raw: line }; + return matcher(input); + }); +} + +export async function validateQueries(system: LoghubSystem): Promise { + const { queries } = queryFileSchema.parse( + JSON.parse(await getFileOrThrow(getQueriesFilename(system))) + ); + + const results = queries.map(({ id, query }) => { + const hits = executeQuery(query, system); + + return { + id, + query: JSON.stringify(query.bool.filter[0]), + hits: hits.length, + }; + }); + + const hasSomeHits = results.some((result) => result.hits > 0); + + if (!hasSomeHits) { + throw new Error('No query returned any hits'); + } +} diff --git a/x-pack/platform/packages/shared/kbn-sample-parser/tsconfig.json b/x-pack/platform/packages/shared/kbn-sample-parser/tsconfig.json new file mode 100644 index 000000000000..0c5a0fc34257 --- /dev/null +++ b/x-pack/platform/packages/shared/kbn-sample-parser/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/tooling-log", + "@kbn/repo-info", + "@kbn/zod", + ] +} diff --git a/x-pack/scripts/sample_log_parser.js b/x-pack/scripts/sample_log_parser.js new file mode 100755 index 000000000000..00f1c58a3de1 --- /dev/null +++ b/x-pack/scripts/sample_log_parser.js @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +require('../../src/setup_node_env'); +require('@kbn/sample-log-parser').cli(); diff --git a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/setup_synthtrace.ts b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/setup_synthtrace.ts index 1674ec5ce8e5..279b3f0c14f1 100644 --- a/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/setup_synthtrace.ts +++ b/x-pack/solutions/observability/plugins/observability_ai_assistant_app/scripts/evaluation/setup_synthtrace.ts @@ -12,8 +12,7 @@ import { ApmSynthtraceKibanaClient, } from '@kbn/apm-synthtrace'; import { ToolingLog } from '@kbn/tooling-log'; -import { isPromise } from 'util/types'; -import { Logger } from '@kbn/apm-synthtrace/src/lib/utils/create_logger'; +import { extendToolingLog } from '@kbn/apm-synthtrace'; import { Client } from '@elastic/elasticsearch'; export interface SynthtraceEsClients { @@ -31,30 +30,7 @@ export async function setupSynthtrace({ client: Client; target: string; }): Promise { - const logger: Logger = { - debug: (...args) => log.debug(...args), - info: (...args) => log.info(...args), - warn: (...args) => log.warning(...args), - error: (...args) => log.error(args.map((arg) => arg.toString()).join(' ')), - perf: (name, cb) => { - const now = performance.now(); - - const result = cb(); - - function measure() { - const after = performance.now(); - log.debug(`[${name}] took ${after - now} ms`); - } - - if (isPromise(result)) { - result.finally(measure); - } else { - measure(); - } - - return result; - }, - }; + const logger = extendToolingLog(log); const kibanaClient = new ApmSynthtraceKibanaClient({ target, logger, diff --git a/yarn.lock b/yarn.lock index 9421307af556..021d15dd5cf0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6923,6 +6923,10 @@ version "0.0.0" uid "" +"@kbn/sample-log-parser@link:x-pack/platform/packages/shared/kbn-sample-parser": + version "0.0.0" + uid "" + "@kbn/sample-task-plugin-update-by-query@link:x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget": version "0.0.0" uid ""