[8.x] [Streams] Replay loghub data with synthtrace (#212120) (#214916)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Streams] Replay loghub data with synthtrace
(#212120)](https://github.com/elastic/kibana/pull/212120)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Dario
Gieselaar","email":"dario.gieselaar@elastic.co"},"sourceCommit":{"committedDate":"2025-03-11T12:30:06Z","message":"[Streams]
Replay loghub data with synthtrace (#212120)\n\nDownload, parse and
replay loghub data with Synthtrace, for use in the\nStreams project. In
summary:\n\n- adds a `@kbn/sample-log-parser` package which parses
Loghub sample\ndata, creates valid parsers for extracting and replacing
timestamps,\nusing the LLM\n- add a `sample_logs` scenario which uses
the parsed data sets to replay\nLoghub data continuously as if it were
live data\n- refactor some parts of Synthtrace (follow-up work captured
in\nhttps://github.com/elastic/kibana/issues/212179)\n\n## Synthtrace
changes\n\n- Replace custom Logger object with Kibana-standard
ToolingLog\n- Report progress and estimated time to completion for
long-running jobs\n- Simplify scenarioOpts (allow comma-separated
key-value pairs instead\nof just JSON)\n- Simplify client
initialization\n- When using workers, only bootstrap once (in the main
thread)\n- Allow workers to gracefully shutdown\n- Downgrade some
logging levels for less noise\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"ba13e86a70c331275d40ed8f84c3f264845afc6e","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport
missing","v9.0.0","ci:project-deploy-observability","Team:obs-ux-infra_services","backport:version","Feature:Streams","v9.1.0","v8.19.0"],"title":"[Streams]
Replay loghub data with
synthtrace","number":212120,"url":"https://github.com/elastic/kibana/pull/212120","mergeCommit":{"message":"[Streams]
Replay loghub data with synthtrace (#212120)\n\nDownload, parse and
replay loghub data with Synthtrace, for use in the\nStreams project. In
summary:\n\n- adds a `@kbn/sample-log-parser` package which parses
Loghub sample\ndata, creates valid parsers for extracting and replacing
timestamps,\nusing the LLM\n- add a `sample_logs` scenario which uses
the parsed data sets to replay\nLoghub data continuously as if it were
live data\n- refactor some parts of Synthtrace (follow-up work captured
in\nhttps://github.com/elastic/kibana/issues/212179)\n\n## Synthtrace
changes\n\n- Replace custom Logger object with Kibana-standard
ToolingLog\n- Report progress and estimated time to completion for
long-running jobs\n- Simplify scenarioOpts (allow comma-separated
key-value pairs instead\nof just JSON)\n- Simplify client
initialization\n- When using workers, only bootstrap once (in the main
thread)\n- Allow workers to gracefully shutdown\n- Downgrade some
logging levels for less noise\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"ba13e86a70c331275d40ed8f84c3f264845afc6e"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.x"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/212120","number":212120,"mergeCommit":{"message":"[Streams]
Replay loghub data with synthtrace (#212120)\n\nDownload, parse and
replay loghub data with Synthtrace, for use in the\nStreams project. In
summary:\n\n- adds a `@kbn/sample-log-parser` package which parses
Loghub sample\ndata, creates valid parsers for extracting and replacing
timestamps,\nusing the LLM\n- add a `sample_logs` scenario which uses
the parsed data sets to replay\nLoghub data continuously as if it were
live data\n- refactor some parts of Synthtrace (follow-up work captured
in\nhttps://github.com/elastic/kibana/issues/212179)\n\n## Synthtrace
changes\n\n- Replace custom Logger object with Kibana-standard
ToolingLog\n- Report progress and estimated time to completion for
long-running jobs\n- Simplify scenarioOpts (allow comma-separated
key-value pairs instead\nof just JSON)\n- Simplify client
initialization\n- When using workers, only bootstrap once (in the main
thread)\n- Allow workers to gracefully shutdown\n- Downgrade some
logging levels for less noise\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"ba13e86a70c331275d40ed8f84c3f264845afc6e"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -776,6 +776,7 @@ x-pack/platform/plugins/shared/rule_registry @elastic/response-ops @elastic/obs-
 x-pack/platform/plugins/private/runtime_fields @elastic/kibana-management
 src/platform/packages/shared/kbn-safer-lodash-set @elastic/kibana-security
 x-pack/test/security_api_integration/plugins/saml_provider @elastic/kibana-security
+x-pack/platform/packages/shared/kbn-sample-parser @elastic/streams-program-team
 x-pack/test/plugin_api_integration/plugins/sample_task_plugin @elastic/response-ops
 x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget @elastic/response-ops
 src/platform/test/plugin_functional/plugins/saved_object_export_transforms @elastic/kibana-core

package.json

@@ -1499,6 +1499,7 @@
     "@kbn/repo-source-classifier": "link:packages/kbn-repo-source-classifier",
     "@kbn/repo-source-classifier-cli": "link:packages/kbn-repo-source-classifier-cli",
     "@kbn/reporting-mocks-server": "link:src/platform/packages/private/kbn-reporting/mocks_server",
+    "@kbn/sample-log-parser": "link:x-pack/platform/packages/shared/kbn-sample-parser",
     "@kbn/scout": "link:src/platform/packages/shared/kbn-scout",
     "@kbn/scout-info": "link:src/platform/packages/private/kbn-scout-info",
     "@kbn/scout-oblt": "link:x-pack/solutions/observability/packages/kbn-scout-oblt",

src/platform/packages/private/kbn-journeys/services/synthtrace.ts

@@ -15,7 +15,7 @@ import {
 } from '@kbn/apm-synthtrace';
 import { ToolingLog } from '@kbn/tooling-log';
 import Url from 'url';
-import { Logger } from '@kbn/apm-synthtrace/src/lib/utils/create_logger';
+import { type Logger, extendToolingLog } from '@kbn/apm-synthtrace';
 import { Auth, Es } from '.';
 import { KibanaUrl } from './kibana_url';
 
@@ -39,45 +39,9 @@ export async function getSynthtraceClient(
   }
 }
 
-// Adapting ToolingLog instance to Logger interface
-class LoggerAdapter implements Logger {
-  private log: ToolingLog;
-  private joiner = ', ';
-
-  constructor(log: ToolingLog) {
-    this.log = log;
-  }
-
-  debug(...args: any[]): void {
-    this.log.debug(args.join(this.joiner));
-  }
-
-  info(...args: any[]): void {
-    this.log.info(args.join(this.joiner));
-  }
-
-  warn(...args: any[]): void {
-    this.log.warning(args.join(this.joiner));
-  }
-
-  error(arg: string | Error): void {
-    this.log.error(arg);
-  }
-
-  perf<T>(name: string, cb: () => T): T {
-    const startTime = Date.now();
-    const result = cb();
-    const duration = Date.now() - startTime;
-    const durationInSeconds = duration / 1000;
-    const formattedTime = durationInSeconds.toFixed(3) + 's';
-    this.log.info(`${name} took ${formattedTime}.`);
-    return result;
-  }
-}
-
 async function initInfraSynthtraceClient(options: SynthtraceClientOptions) {
   const { log, es, auth, kbnUrl } = options;
-  const logger: Logger = new LoggerAdapter(log);
+  const logger: Logger = extendToolingLog(log);
 
   const synthKbnClient = new InfraSynthtraceKibanaClient({
     logger,
@@ -99,7 +63,7 @@ async function initInfraSynthtraceClient(options: SynthtraceClientOptions) {
 
 async function initApmSynthtraceClient(options: SynthtraceClientOptions) {
   const { log, es, auth, kbnUrl } = options;
-  const logger: Logger = new LoggerAdapter(log);
+  const logger: Logger = extendToolingLog(log);
   const kibanaUrl = new URL(kbnUrl.get());
   const kibanaUrlWithAuth = Url.format({
     protocol: kibanaUrl.protocol,

src/platform/packages/shared/kbn-apm-synthtrace-client/index.ts

@@ -27,7 +27,7 @@ export { Entity } from './src/lib/entity';
 export { infra, type InfraDocument } from './src/lib/infra';
 export { parseInterval } from './src/lib/interval';
 export { monitoring, type MonitoringDocument } from './src/lib/monitoring';
-export type { Serializable } from './src/lib/serializable';
+export { Serializable } from './src/lib/serializable';
 export { timerange } from './src/lib/timerange';
 export type { Timerange } from './src/lib/timerange';
 export { dedot } from './src/lib/utils/dedot';

src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/interval.ts

@@ -7,11 +7,13 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ToolingLog } from '@kbn/tooling-log';
 import { castArray } from 'lodash';
 import moment, { unitOfTime } from 'moment';
 import { SynthtraceGenerator } from '../types';
 import { Fields } from './entity';
 import { Serializable } from './serializable';
+import { TimerangeProgressReporter } from './timerange_progress_reporter';
 
 export function parseInterval(interval: string): {
   intervalAmount: number;
@@ -32,6 +34,7 @@ interface IntervalOptions {
   to: Date;
   interval: string;
   rate?: number;
+  log?: ToolingLog;
 }
 
 interface StepDetails {
@@ -86,10 +89,22 @@ export class Interval<TFields extends Fields = Fields> {
     };
 
     let index = 0;
+    const calculateEvery = 10;
+
+    const reporter = this.options.log
+      ? new TimerangeProgressReporter({
+          log: this.options.log,
+          reportEvery: 5000,
+          total: timestamps.length,
+        })
+      : undefined;
 
     for (const timestamp of timestamps) {
       const events = castArray(map(timestamp, index, stepDetails));
       index++;
+      if (index % calculateEvery === 0) {
+        reporter?.next(index);
+      }
       for (const event of events) {
         yield event;
       }

src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange.ts

@@ -9,15 +9,20 @@
 
 import datemath from '@kbn/datemath';
 import type { Moment } from 'moment';
+import { ToolingLog } from '@kbn/tooling-log';
 import { GaussianEvents } from './gaussian_events';
 import { Interval } from './interval';
 import { PoissonEvents } from './poisson_events';
 
 export class Timerange {
-  constructor(public readonly from: Date, public readonly to: Date) {}
+  constructor(
+    public readonly from: Date,
+    public readonly to: Date,
+    private readonly log?: ToolingLog
+  ) {}
 
   interval(interval: string) {
-    return new Interval({ from: this.from, to: this.to, interval });
+    return new Interval({ from: this.from, to: this.to, interval, log: this.log });
   }
 
   ratePerMinute(rate: number) {
@@ -39,7 +44,7 @@ export class Timerange {
     return Array.from({ length: segmentCount }, (_, i) => {
       const from = new Date(this.from.getTime() + i * segmentDuration);
       const to = new Date(from.getTime() + segmentDuration);
-      return new Timerange(from, to);
+      return new Timerange(from, to, this.log);
     });
   }
 
@@ -65,7 +70,7 @@ function getDateFrom(date: DateLike, now: Date): Date {
   return date.toDate();
 }
 
-export function timerange(from: DateLike, to: DateLike) {
+export function timerange(from: DateLike, to: DateLike, log?: ToolingLog) {
   const now = new Date();
-  return new Timerange(getDateFrom(from, now), getDateFrom(to, now));
+  return new Timerange(getDateFrom(from, now), getDateFrom(to, now), log);
 }

src/platform/packages/shared/kbn-apm-synthtrace-client/src/lib/timerange_progress_reporter.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { first, last } from 'lodash';
+import moment from 'moment';
+
+interface TimerangeProgressOptions {
+  log: ToolingLog;
+  total: number;
+  reportEvery: number;
+}
+
+export class TimerangeProgressReporter {
+  private readonly startOfRun: number = performance.now();
+
+  private measurements: Array<{
+    measuredAt: number;
+    index: number;
+  }> = [
+    {
+      measuredAt: this.startOfRun,
+      index: 0,
+    },
+  ];
+
+  private lastReported: number = this.startOfRun;
+
+  constructor(private readonly options: TimerangeProgressOptions) {}
+
+  next(index: number) {
+    const now = performance.now();
+
+    this.measurements.unshift({
+      index,
+      measuredAt: now,
+    });
+
+    this.measurements.length = Math.min(10, this.measurements.length);
+
+    const timeSinceLastReported = now - this.lastReported;
+
+    if (timeSinceLastReported >= this.options.reportEvery) {
+      this.report(now);
+    }
+  }
+
+  private report(now: number) {
+    this.lastReported = now;
+
+    const firstMeasurement = first(this.measurements)!;
+    const lastMeasurement = last(this.measurements)!;
+
+    const totalDurationFormatted = moment.duration(now - this.startOfRun).humanize();
+
+    const indicesLeft = this.options.total - lastMeasurement.index;
+
+    const measuredIndicesProcessed = lastMeasurement.index - firstMeasurement.index;
+
+    const measuredDuration = lastMeasurement.measuredAt - firstMeasurement.measuredAt;
+
+    const indicesPerMs = measuredIndicesProcessed / measuredDuration;
+
+    const timeLeft = indicesLeft / indicesPerMs;
+
+    const timeLeftFormatted = moment.duration(timeLeft).humanize(true);
+
+    const totalProgress = lastMeasurement.index / this.options.total;
+
+    this.options.log.info(
+      `progress=${(totalProgress * 100).toPrecision(
+        3
+      )}%, duration=${totalDurationFormatted}, eta=${timeLeftFormatted}`
+    );
+  }
+}

src/platform/packages/shared/kbn-apm-synthtrace-client/tsconfig.json

@@ -15,5 +15,6 @@
   "kbn_references": [
     "@kbn/datemath",
     "@kbn/safer-lodash-set",
+    "@kbn/tooling-log",
   ]
 }

src/platform/packages/shared/kbn-apm-synthtrace/index.ts

@@ -7,7 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export { createLogger, LogLevel } from './src/lib/utils/create_logger';
+export {
+  createLogger,
+  LogLevel,
+  type Logger,
+  extendToolingLog,
+} from './src/lib/utils/create_logger';
 
 export { ApmSynthtraceEsClient } from './src/lib/apm/client/apm_synthtrace_es_client';
 export { ApmSynthtraceKibanaClient } from './src/lib/apm/client/apm_synthtrace_kibana_client';

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/run_synthtrace.ts

@@ -67,15 +67,55 @@ function options(y: Argv) {
       number: true,
       default: 1,
     })
+    .option('debug', {
+      describe: 'Use a debug log level',
+      boolean: true,
+    })
+    .option('verbose', {
+      describe: 'Use a verbose log level',
+      boolean: true,
+    })
     .option('logLevel', {
       describe: 'Log level',
-      choices: ['trace', 'debug', 'info', 'error'],
+      choices: ['verbose', 'debug', 'info', 'error'],
       default: 'info',
     })
     .option('scenarioOpts', {
       describe: 'Options specific to the scenario',
-      coerce: (arg) => {
-        return arg as Record<string, any> | undefined;
+      type: 'string',
+      coerce: (arg: string): Record<string, unknown> => {
+        if (!arg) {
+          return {};
+        }
+
+        let scenarioOptions: Record<string, unknown> = {};
+
+        try {
+          scenarioOptions = JSON.parse(arg);
+        } catch (error) {
+          scenarioOptions = Object.fromEntries(
+            arg.split(',').map((kv) => {
+              const [key, value] = kv
+                .trim()
+                .split('=')
+                .map((part) => part.trim());
+              if (value === 'true') {
+                return [key, true];
+              }
+              if (value === 'false') {
+                return [key, false];
+              }
+
+              if (!isNaN(Number(value))) {
+                return [key, Number(value)];
+              }
+
+              return [key, value];
+            })
+          );
+        }
+
+        return scenarioOptions;
       },
     })
     .option('assume-package-version', {
@@ -95,20 +135,18 @@ function options(y: Argv) {
 async function run(argv: RunCliFlags) {
   const runOptions = parseRunCliFlags(argv);
 
-  const toMs = datemath.parse(String(argv.to ?? 'now'))!.valueOf();
-  const to = new Date(toMs);
+  const to = datemath.parse(String(argv.to ?? 'now'))!.valueOf();
 
   const defaultTimeRange = '1m';
 
-  const fromMs = argv.from
+  const from = argv.from
     ? datemath.parse(String(argv.from))!.valueOf()
-    : toMs - intervalToMs(defaultTimeRange);
-  const from = new Date(fromMs);
+    : to - intervalToMs(defaultTimeRange);
 
   const live = argv.live;
 
   if (live) {
-    await startLiveDataUpload({ runOptions, start: from });
+    await startLiveDataUpload({ runOptions, from, to });
   } else {
     await startHistoricalDataUpload({ runOptions, from, to });
   }

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/scenario.ts

@@ -7,40 +7,22 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Timerange } from '@kbn/apm-synthtrace-client';
-import {
-  ApmSynthtraceEsClient,
-  InfraSynthtraceEsClient,
-  LogsSynthtraceEsClient,
-  SyntheticsSynthtraceEsClient,
-  OtelSynthtraceEsClient,
-  EntitiesSynthtraceEsClient,
-} from '../..';
+import { Fields, Timerange } from '@kbn/apm-synthtrace-client';
 import { Logger } from '../lib/utils/create_logger';
 import { ScenarioReturnType } from '../lib/utils/with_client';
+import { SynthtraceClients } from './utils/get_clients';
 import { RunOptions } from './utils/parse_run_cli_flags';
-import { EntitiesSynthtraceKibanaClient } from '../lib/entities/entities_synthtrace_kibana_client';
 
-interface EsClients {
-  apmEsClient: ApmSynthtraceEsClient;
-  logsEsClient: LogsSynthtraceEsClient;
-  infraEsClient: InfraSynthtraceEsClient;
-  syntheticsEsClient: SyntheticsSynthtraceEsClient;
-  otelEsClient: OtelSynthtraceEsClient;
-  entitiesEsClient: EntitiesSynthtraceEsClient;
-}
+export type ScenarioInitOptions = RunOptions & { logger: Logger; from: number; to: number };
+export type ScenarioPhaseOptions = SynthtraceClients;
 
-interface KibanaClients {
-  entitiesKibanaClient: EntitiesSynthtraceKibanaClient;
-}
-
-type Generate<TFields> = (options: {
+type Generate<TFields extends Fields> = (options: {
   range: Timerange;
-  clients: EsClients;
+  clients: SynthtraceClients;
 }) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
 
-export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
-  bootstrap?: (options: EsClients & KibanaClients) => Promise<void>;
+export type Scenario<TFields extends Fields = Fields> = (options: ScenarioInitOptions) => Promise<{
+  bootstrap?: (options: ScenarioPhaseOptions) => Promise<void>;
   generate: Generate<TFields>;
-  teardown?: (options: EsClients & KibanaClients) => Promise<void>;
+  teardown?: (options: ScenarioPhaseOptions) => Promise<void>;
 }>;

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts

@@ -7,23 +7,20 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { Client, HttpConnection } from '@elastic/elasticsearch';
 import { createLogger } from '../../lib/utils/create_logger';
-import { getApmEsClient } from './get_apm_es_client';
-import { getLogsEsClient } from './get_logs_es_client';
-import { getInfraEsClient } from './get_infra_es_client';
+import { getClients } from './get_clients';
 import { getKibanaClient } from './get_kibana_client';
 import { getServiceUrls } from './get_service_urls';
 import { RunOptions } from './parse_run_cli_flags';
-import { getSyntheticsEsClient } from './get_synthetics_es_client';
-import { getOtelSynthtraceEsClient } from './get_otel_es_client';
-import { getEntitiesEsClient } from './get_entities_es_client';
-import { getEntitiesKibanaClient } from './get_entites_kibana_client';
+import { getEsClientTlsSettings } from './ssl';
 
-export async function bootstrap(runOptions: RunOptions) {
+export async function bootstrap({
+  skipClientBootstrap,
+  ...runOptions
+}: RunOptions & { skipClientBootstrap?: boolean }) {
   const logger = createLogger(runOptions.logLevel);
 
-  let version = runOptions['assume-package-version'];
-
   const { kibanaUrl, esUrl } = await getServiceUrls({ ...runOptions, logger });
 
   const kibanaClient = getKibanaClient({
@@ -31,76 +28,37 @@ export async function bootstrap(runOptions: RunOptions) {
     logger,
   });
 
-  if (!version) {
-    version = await kibanaClient.fetchLatestApmPackageVersion();
-    await kibanaClient.installApmPackage(version);
-  } else if (version === 'latest') {
-    version = await kibanaClient.fetchLatestApmPackageVersion();
-  }
-
-  logger.info(`Using package version: ${version}`);
-
-  const apmEsClient = getApmEsClient({
-    target: esUrl,
-    logger,
-    concurrency: runOptions.concurrency,
-    version,
+  const client = new Client({
+    node: esUrl,
+    tls: getEsClientTlsSettings(esUrl),
+    Connection: HttpConnection,
+    requestTimeout: 30_000,
   });
 
-  const logsEsClient = getLogsEsClient({
-    target: esUrl,
+  const clients = await getClients({
     logger,
-    concurrency: runOptions.concurrency,
-  });
-
-  const infraEsClient = getInfraEsClient({
-    target: esUrl,
-    logger,
-    concurrency: runOptions.concurrency,
-  });
-
-  const entitiesEsClient = getEntitiesEsClient({
-    target: esUrl,
-    logger,
-    concurrency: runOptions.concurrency,
-  });
-
-  const entitiesKibanaClient = getEntitiesKibanaClient({
-    target: kibanaUrl,
-    logger,
-  });
-
-  const syntheticsEsClient = getSyntheticsEsClient({
-    target: esUrl,
-    logger,
-    concurrency: runOptions.concurrency,
-  });
-  const otelEsClient = getOtelSynthtraceEsClient({
-    target: esUrl,
-    logger,
-    concurrency: runOptions.concurrency,
+    packageVersion: runOptions['assume-package-version'],
+    options: {
+      client,
+      logger,
+      concurrency: runOptions.concurrency,
+      kibana: kibanaClient,
+    },
+    skipBootstrap: skipClientBootstrap,
   });
 
   if (runOptions.clean) {
-    await apmEsClient.clean();
-    await logsEsClient.clean();
-    await infraEsClient.clean();
-    await entitiesEsClient.clean();
-    await syntheticsEsClient.clean();
-    await otelEsClient.clean();
+    for (const synthtraceClient of Object.values(clients)) {
+      if ('clean' in synthtraceClient) {
+        await synthtraceClient.clean();
+      }
+    }
   }
 
   return {
+    clients,
     logger,
-    apmEsClient,
-    logsEsClient,
-    infraEsClient,
-    entitiesEsClient,
-    syntheticsEsClient,
-    otelEsClient,
-    version,
     kibanaUrl,
     esUrl,
-    entitiesKibanaClient,
   };
 }

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts

@@ -1,39 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { ApmSynthtraceEsClient } from '../../..';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-
-export function getApmEsClient({
-  target,
-  logger,
-  version,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  version: string;
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  const apmEsClient = new ApmSynthtraceEsClient({
-    client,
-    logger,
-    version,
-    concurrency,
-  });
-
-  return apmEsClient;
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_clients.ts

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Required } from 'utility-types';
+import { Client } from '@elastic/elasticsearch';
+import { ApmSynthtraceEsClient } from '../../lib/apm/client/apm_synthtrace_es_client';
+import { ApmSynthtraceKibanaClient } from '../../lib/apm/client/apm_synthtrace_kibana_client';
+import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client';
+import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client';
+import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client';
+import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client';
+import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client';
+import { SynthtraceEsClientOptions } from '../../lib/shared/base_client';
+import { StreamsSynthtraceClient } from '../../lib/streams/streams_synthtrace_client';
+import { SyntheticsSynthtraceEsClient } from '../../lib/synthetics/synthetics_synthtrace_es_client';
+import { Logger } from '../../lib/utils/create_logger';
+
+export interface SynthtraceClients {
+  apmEsClient: ApmSynthtraceEsClient;
+  entitiesEsClient: EntitiesSynthtraceEsClient;
+  infraEsClient: InfraSynthtraceEsClient;
+  logsEsClient: LogsSynthtraceEsClient;
+  otelEsClient: OtelSynthtraceEsClient;
+  streamsClient: StreamsSynthtraceClient;
+  syntheticsEsClient: SyntheticsSynthtraceEsClient;
+  entitiesKibanaClient: EntitiesSynthtraceKibanaClient;
+  esClient: Client;
+}
+
+export async function getClients({
+  logger,
+  options,
+  packageVersion,
+  skipBootstrap,
+}: {
+  logger: Logger;
+  options: Required<Omit<SynthtraceEsClientOptions, 'pipeline'>, 'kibana'>;
+  packageVersion?: string;
+  skipBootstrap?: boolean;
+}): Promise<SynthtraceClients> {
+  const apmKibanaClient = new ApmSynthtraceKibanaClient({
+    logger,
+    kibanaClient: options.kibana,
+  });
+
+  let version = packageVersion;
+
+  if (!version) {
+    version = await apmKibanaClient.fetchLatestApmPackageVersion();
+    if (!skipBootstrap) {
+      await apmKibanaClient.installApmPackage(version);
+    }
+  } else if (version === 'latest') {
+    version = await apmKibanaClient.fetchLatestApmPackageVersion();
+  }
+
+  logger.debug(`Using package version: ${version}`);
+
+  const apmEsClient = new ApmSynthtraceEsClient({
+    ...options,
+    version,
+  });
+
+  const logsEsClient = new LogsSynthtraceEsClient(options);
+  const infraEsClient = new InfraSynthtraceEsClient(options);
+  const entitiesEsClient = new EntitiesSynthtraceEsClient(options);
+
+  const entitiesKibanaClient = new EntitiesSynthtraceKibanaClient({
+    ...options,
+    kibanaClient: options.kibana,
+  });
+
+  const syntheticsEsClient = new SyntheticsSynthtraceEsClient(options);
+  const otelEsClient = new OtelSynthtraceEsClient(options);
+
+  const streamsClient = new StreamsSynthtraceClient(options);
+
+  return {
+    apmEsClient,
+    entitiesEsClient,
+    infraEsClient,
+    logsEsClient,
+    otelEsClient,
+    streamsClient,
+    syntheticsEsClient,
+    entitiesKibanaClient,
+    esClient: options.client,
+  };
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entites_kibana_client.ts

@@ -1,20 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { EntitiesSynthtraceKibanaClient } from '../../lib/entities/entities_synthtrace_kibana_client';
-import { Logger } from '../../lib/utils/create_logger';
-
-export function getEntitiesKibanaClient({ target, logger }: { target: string; logger: Logger }) {
-  const kibanaClient = new EntitiesSynthtraceKibanaClient({
-    logger,
-    target,
-  });
-
-  return kibanaClient;
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_entities_es_client.ts

@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { EntitiesSynthtraceEsClient } from '../../lib/entities/entities_synthtrace_es_client';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-
-export function getEntitiesEsClient({
-  target,
-  logger,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  return new EntitiesSynthtraceEsClient({
-    client,
-    logger,
-    concurrency,
-    refreshAfterIndex: true,
-  });
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts

@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-
-export function getInfraEsClient({
-  target,
-  logger,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  return new InfraSynthtraceEsClient({
-    client,
-    logger,
-    concurrency,
-    refreshAfterIndex: true,
-  });
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_kibana_client.ts

@@ -7,12 +7,11 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { ApmSynthtraceKibanaClient } from '../../lib/apm/client/apm_synthtrace_kibana_client';
+import { KibanaClient } from '../../lib/shared/base_kibana_client';
 import { Logger } from '../../lib/utils/create_logger';
 
 export function getKibanaClient({ target, logger }: { target: string; logger: Logger }) {
-  const kibanaClient = new ApmSynthtraceKibanaClient({
-    logger,
+  const kibanaClient = new KibanaClient({
     target,
   });
 

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-
-export function getLogsEsClient({
-  target,
-  logger,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  return new LogsSynthtraceEsClient({
-    client,
-    logger,
-    concurrency,
-  });
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client';
-
-export function getOtelSynthtraceEsClient({
-  target,
-  logger,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  return new OtelSynthtraceEsClient({
-    client,
-    logger,
-    concurrency,
-  });
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.test.ts

@@ -178,7 +178,7 @@ describe('getServiceUrls', () => {
       const target = 'https://elastic_serverless:changeme@127.0.0.1:9200';
       const kibana = 'https://elastic_serverless:changeme@localhost:5601';
 
-      const warnSpy = jest.spyOn(logger, 'warn');
+      const warnSpy = jest.spyOn(logger, 'warning');
       mockFetchWithAllowedSegments([target, kibana]);
       await expectServiceUrls(target, kibana, {
         esUrl: 'https://elastic_serverless:changeme@127.0.0.1:9200',

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts

@@ -98,7 +98,7 @@ async function getKibanaUrl({
       );
     }
 
-    logger.info(`Discovered kibana running at: ${stripAuthIfCi(discoveredKibanaUrlWithAuth)}`);
+    logger.debug(`Discovered kibana running at: ${stripAuthIfCi(discoveredKibanaUrlWithAuth)}`);
 
     return discoveredKibanaUrlWithAuth.replace(/\/$/, '');
   } catch (error) {
@@ -183,7 +183,7 @@ function logCertificateWarningsIfNeeded(parsedTarget: Url, parsedKibanaUrl: Url,
     (parsedTarget.protocol === 'https:' || parsedKibanaUrl.protocol === 'https:') &&
     (parsedTarget.hostname === '127.0.0.1' || parsedKibanaUrl.hostname === '127.0.0.1')
   ) {
-    logger.warn(
+    logger.warning(
       `WARNING: Self-signed certificate may not work with hostname: '127.0.0.1'. Consider using 'localhost' instead.`
     );
   }
@@ -193,7 +193,7 @@ export async function getServiceUrls({ logger, target, kibana }: RunOptions & {
   if (!target) {
     if (!kibana) {
       kibana = 'http://localhost:5601';
-      logger.info(`No target provided, defaulting Kibana to ${kibana}`);
+      logger.debug(`No target provided, defaulting Kibana to ${kibana}`);
     }
     target = await discoverTargetFromKibanaUrl(kibana);
   }

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/get_synthetics_es_client.ts

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-import { Client } from '@elastic/elasticsearch';
-import { Logger } from '../../lib/utils/create_logger';
-import { RunOptions } from './parse_run_cli_flags';
-import { getEsClientTlsSettings } from './ssl';
-import { SyntheticsSynthtraceEsClient } from '../../lib/synthetics/synthetics_synthtrace_es_client';
-
-export function getSyntheticsEsClient({
-  target,
-  logger,
-  concurrency,
-}: Pick<RunOptions, 'concurrency'> & {
-  target: string;
-  logger: Logger;
-}) {
-  const client = new Client({
-    node: target,
-    tls: getEsClientTlsSettings(target),
-  });
-
-  return new SyntheticsSynthtraceEsClient({
-    client,
-    logger,
-    concurrency,
-  });
-}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/index_historical_data.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { castArray } from 'lodash';
+import { memoryUsage } from 'process';
+import { timerange } from '@kbn/apm-synthtrace-client';
+import { Logger } from '../../lib/utils/create_logger';
+import { SynthtraceClients } from './get_clients';
+import { getScenario } from './get_scenario';
+import { WorkerData } from './synthtrace_worker';
+import { StreamManager } from './stream_manager';
+
+export async function indexHistoricalData({
+  bucketFrom,
+  bucketTo,
+  runOptions,
+  workerId,
+  logger,
+  clients,
+  from,
+  to,
+  streamManager,
+}: WorkerData & { logger: Logger; clients: SynthtraceClients; streamManager: StreamManager }) {
+  const file = runOptions.file;
+
+  const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
+
+  logger.info(
+    `Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()} (pid: ${
+      process.pid
+    })`
+  );
+
+  const { generate } = await scenario({ ...runOptions, logger, from, to });
+
+  logger.debug('Generating scenario');
+
+  const generatorsAndClients = castArray(
+    logger.perf('generate_scenario', () =>
+      generate({
+        range: timerange(bucketFrom, bucketTo, logger),
+        clients,
+      })
+    )
+  );
+
+  logger.debug('Indexing scenario');
+
+  function mb(value: number): string {
+    return Math.round(value / 1024 ** 2).toString() + 'mb';
+  }
+
+  let cpuUsage = process.cpuUsage();
+
+  const intervalId = setInterval(async () => {
+    cpuUsage = process.cpuUsage(cpuUsage);
+    const mem = memoryUsage();
+    logger.debug(
+      `cpu time: (user: ${Math.round(cpuUsage.user / 1000)}mss, sys: ${Math.round(
+        cpuUsage.system / 1000
+      )}ms), memory: ${mb(mem.heapUsed)}/${mb(mem.heapTotal)}`
+    );
+  }, 5000);
+
+  await logger.perf('index_scenario', async () => {
+    await Promise.all(
+      generatorsAndClients.map(async ({ client, generator }) => {
+        await streamManager.index(client, generator);
+      })
+    ).finally(() => {
+      clearInterval(intervalId);
+    });
+  });
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/logger_proxy.ts

@@ -7,38 +7,17 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import util from 'util';
 import { parentPort, isMainThread, workerData } from 'worker_threads';
 import { createLogger, Logger, LogLevel } from '../../lib/utils/create_logger';
-import { logPerf } from '../../lib/utils/log_perf';
 import { WorkerData } from './synthtrace_worker';
 
 const { workerId } = isMainThread ? { workerId: -1 } : (workerData as WorkerData);
-
-function getLogMethod(log: LogLevel) {
-  return (...args: any) => {
-    parentPort?.postMessage({
-      log,
-      args: [`[${workerId}]`].concat(
-        args.map((arg: any) =>
-          typeof arg === 'string' || typeof arg === 'number'
-            ? arg
-            : util.inspect(arg, { depth: 10 })
-        )
-      ),
-    });
-  };
-}
-
 // logging proxy to main thread, ensures we see real time logging
 export const loggerProxy: Logger = isMainThread
-  ? createLogger(LogLevel.trace)
-  : {
-      perf: <T extends any>(name: string, cb: () => T): T => {
-        return logPerf(loggerProxy, LogLevel.trace, name, cb);
+  ? createLogger(LogLevel.verbose)
+  : createLogger(LogLevel.verbose, {
+      write: (msg) => {
+        parentPort?.postMessage([msg.type, [`[${workerId}]`, msg.args[0]].join(' ')]);
+        return true;
       },
-      debug: getLogMethod(LogLevel.debug),
-      info: getLogMethod(LogLevel.info),
-      warn: getLogMethod(LogLevel.warn),
-      error: getLogMethod(LogLevel.error),
-    };
+    });

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/parse_run_cli_flags.ts

@@ -29,26 +29,25 @@ function getParsedFile(flags: RunCliFlags) {
     path.resolve(__dirname, '../../scenarios', `${parsedFile}.js`),
   ].find((p) => existsSync(p));
 
-  if (filepath) {
-    // eslint-disable-next-line no-console
-    console.log(`Loading scenario from ${filepath}`);
-    return filepath;
+  if (!filepath) {
+    throw new Error(`Could not find scenario file: "${parsedFile}"`);
   }
 
-  throw new Error(`Could not find scenario file: "${parsedFile}"`);
+  return filepath;
 }
 
 export function parseRunCliFlags(flags: RunCliFlags) {
-  const { logLevel, target } = flags;
+  const { logLevel, target, debug, verbose } = flags;
   if (target?.includes('.kb.')) {
     throw new Error(`Target URL seems to be a Kibana URL, please provide Elasticsearch URL`);
   }
   const parsedFile = getParsedFile(flags);
 
-  let parsedLogLevel = LogLevel.info;
+  let parsedLogLevel = verbose ? LogLevel.verbose : debug ? LogLevel.debug : LogLevel.info;
+
   switch (logLevel) {
-    case 'trace':
-      parsedLogLevel = LogLevel.trace;
+    case 'verbose':
+      parsedLogLevel = LogLevel.verbose;
       break;
 
     case 'info':
@@ -67,13 +66,11 @@ export function parseRunCliFlags(flags: RunCliFlags) {
       parsedLogLevel = LogLevel.error;
       break;
   }
-
   return {
     ...pick(
       flags,
       'target',
       'workers',
-      'scenarioOpts',
       'kibana',
       'concurrency',
       'versionOverride',
@@ -81,6 +78,7 @@ export function parseRunCliFlags(flags: RunCliFlags) {
       'assume-package-version',
       'liveBucketSize'
     ),
+    scenarioOpts: flags.scenarioOpts as unknown as Record<string, any>,
     logLevel: parsedLogLevel,
     file: parsedFile,
   };

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_historical_data_upload.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { range } from 'lodash';
+import { once, range } from 'lodash';
 import moment from 'moment';
 import { cpus } from 'os';
 import Path from 'path';
@@ -16,6 +16,9 @@ import { LogLevel } from '../../..';
 import { bootstrap } from './bootstrap';
 import { RunOptions } from './parse_run_cli_flags';
 import { WorkerData } from './synthtrace_worker';
+import { getScenario } from './get_scenario';
+import { StreamManager } from './stream_manager';
+import { indexHistoricalData } from './index_historical_data';
 
 export async function startHistoricalDataUpload({
   runOptions,
@@ -23,23 +26,45 @@ export async function startHistoricalDataUpload({
   to,
 }: {
   runOptions: RunOptions;
-  from: Date;
-  to: Date;
+  from: number;
+  to: number;
 }) {
-  const { logger, esUrl, version, kibanaUrl } = await bootstrap(runOptions);
+  const { logger, clients } = await bootstrap(runOptions);
+
+  const file = runOptions.file;
+
+  const scenario = await logger.perf('get_scenario', async () => {
+    const fn = await getScenario({ file, logger });
+    return fn({
+      ...runOptions,
+      logger,
+      from,
+      to,
+    });
+  });
+
+  const teardown = once(async () => {
+    if (scenario.teardown) {
+      await scenario.teardown(clients);
+    }
+  });
+
+  const streamManager = new StreamManager(logger, teardown);
+
+  if (scenario.bootstrap) {
+    await scenario.bootstrap(clients);
+  }
 
   const cores = cpus().length;
 
   let workers = Math.min(runOptions.workers ?? 10, cores - 1);
 
-  const rangeEnd = to;
-
-  const diff = moment(from).diff(rangeEnd);
+  const diff = moment(from).diff(to);
 
   const d = moment.duration(Math.abs(diff), 'ms');
 
-  // make sure ranges cover at least 100k documents
-  const minIntervalSpan = moment.duration(60, 'm');
+  // make sure ranges cover at least 1m
+  const minIntervalSpan = moment.duration(1, 'm');
 
   const minNumberOfRanges = d.asMilliseconds() / minIntervalSpan.asMilliseconds();
   if (minNumberOfRanges < workers) {
@@ -52,15 +77,12 @@ export async function startHistoricalDataUpload({
     logger.info(`updating maxWorkers to ${workers} to ensure each worker does enough work`);
   }
 
-  logger.info(`Generating data from ${from.toISOString()} to ${rangeEnd.toISOString()}`);
-
-  interface WorkerMessages {
-    log: LogLevel;
-    args: any[];
-  }
+  logger.info(
+    `Generating data from ${new Date(from).toISOString()} to ${new Date(to).toISOString()}`
+  );
 
   function rangeStep(interval: number) {
-    if (from > rangeEnd) return moment(from).subtract(interval, 'ms').toDate();
+    if (from > to) return moment(from).subtract(interval, 'ms').toDate();
     return moment(from).add(interval, 'ms').toDate();
   }
 
@@ -91,32 +113,34 @@ export async function startHistoricalDataUpload({
         bucketFrom,
         bucketTo,
         workerId: workerIndex.toString(),
-        esUrl,
-        version,
-        kibanaUrl,
+        from,
+        to,
       };
       const worker = new Worker(Path.join(__dirname, './worker.js'), {
         workerData,
       });
-      worker.on('message', (message: WorkerMessages) => {
-        switch (message.log) {
+
+      streamManager.trackWorker(worker);
+
+      worker.on('message', ([logLevel, msg]: [string, string]) => {
+        switch (logLevel) {
           case LogLevel.debug:
-            logger.debug.apply({}, message.args);
+            logger.debug(msg);
             return;
           case LogLevel.info:
-            logger.info.apply({}, message.args);
+            logger.info(msg);
             return;
-          case LogLevel.trace:
-            logger.debug.apply({}, message.args);
+          case LogLevel.verbose:
+            logger.verbose(msg);
             return;
           case LogLevel.warn:
-            logger.warn.apply({}, message.args);
+            logger.warning(msg);
             return;
           case LogLevel.error:
-            logger.error.apply({}, message.args);
+            logger.error(msg);
             return;
           default:
-            logger.info(message);
+            logger.info(msg);
         }
       });
       worker.on('error', (message) => {
@@ -134,7 +158,27 @@ export async function startHistoricalDataUpload({
     });
   }
 
-  const workerServices = range(0, intervals.length).map((index) => runService(intervals[index]));
+  const workerServices =
+    intervals.length === 1
+      ? // just run in this process. it's hard to attach
+        // a debugger to a worker_thread, see:
+        // https://issues.chromium.org/issues/41461728
+        [
+          indexHistoricalData({
+            bucketFrom: intervals[0].bucketFrom,
+            bucketTo: intervals[0].bucketTo,
+            clients,
+            logger,
+            runOptions,
+            workerId: 'i',
+            from,
+            to,
+            streamManager,
+          }),
+        ]
+      : range(0, intervals.length).map((index) => runService(intervals[index]));
 
-  return Promise.all(workerServices);
+  await Promise.race(workerServices);
+
+  await teardown();
 }

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts

@@ -8,156 +8,86 @@
  */
 
 import { timerange } from '@kbn/apm-synthtrace-client';
-import { castArray } from 'lodash';
-import { PassThrough, Readable, Writable } from 'stream';
-import { isGeneratorObject } from 'util/types';
-import { SynthtraceEsClient } from '../../lib/shared/base_client';
-import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
+import { castArray, once } from 'lodash';
 import { bootstrap } from './bootstrap';
 import { getScenario } from './get_scenario';
 import { RunOptions } from './parse_run_cli_flags';
+import { StreamManager } from './stream_manager';
 
 export async function startLiveDataUpload({
   runOptions,
-  start,
+  from,
+  to,
 }: {
   runOptions: RunOptions;
-  start: Date;
+  from: number;
+  to: number;
 }) {
   const file = runOptions.file;
 
-  const {
-    logger,
-    apmEsClient,
-    logsEsClient,
-    infraEsClient,
-    syntheticsEsClient,
-    otelEsClient,
-    entitiesEsClient,
-    entitiesKibanaClient,
-  } = await bootstrap(runOptions);
+  const { logger, clients } = await bootstrap(runOptions);
 
   const scenario = await getScenario({ file, logger });
   const {
     generate,
-    bootstrap: scenarioBootsrap,
+    bootstrap: scenarioBootstrap,
     teardown: scenarioTearDown,
-  } = await scenario({ ...runOptions, logger });
+  } = await scenario({ ...runOptions, logger, from, to });
 
-  if (scenarioBootsrap) {
-    await scenarioBootsrap({
-      apmEsClient,
-      logsEsClient,
-      infraEsClient,
-      otelEsClient,
-      syntheticsEsClient,
-      entitiesEsClient,
-      entitiesKibanaClient,
-    });
+  const teardown = once(async () => {
+    if (scenarioTearDown) {
+      await scenarioTearDown(clients);
+    }
+  });
+
+  const streamManager = new StreamManager(logger, teardown);
+
+  if (scenarioBootstrap) {
+    await scenarioBootstrap(clients);
   }
 
   const bucketSizeInMs = runOptions.liveBucketSize;
-  let requestedUntil = start;
-
-  let currentStreams: PassThrough[] = [];
-  // @ts-expect-error upgrade typescript v4.9.5
-  const cachedStreams: WeakMap<SynthtraceEsClient, PassThrough> = new WeakMap();
-
-  process.on('SIGINT', () => closeStreamsAndTeardown());
-  process.on('SIGTERM', () => closeStreamsAndTeardown());
-  process.on('SIGQUIT', () => closeStreamsAndTeardown());
-
-  async function closeStreamsAndTeardown() {
-    if (scenarioTearDown) {
-      try {
-        await scenarioTearDown({
-          apmEsClient,
-          logsEsClient,
-          infraEsClient,
-          otelEsClient,
-          syntheticsEsClient,
-          entitiesEsClient,
-          entitiesKibanaClient,
-        });
-      } catch (error) {
-        logger.error('Error during scenario teardown', error);
-      }
-    }
-
-    currentStreams.forEach((stream) => {
-      stream.end(() => {
-        process.exit(0);
-      });
-    });
-    currentStreams = []; // Reset the stream array
-  }
+  let requestedUntil = from;
 
   async function uploadNextBatch() {
     const now = Date.now();
 
-    if (now > requestedUntil.getTime()) {
-      const bucketFrom = requestedUntil;
-      const bucketTo = new Date(requestedUntil.getTime() + bucketSizeInMs);
+    if (now > requestedUntil) {
+      const bucketCount = Math.floor((now - requestedUntil) / bucketSizeInMs);
+
+      const rangeStart = requestedUntil;
+      const rangeEnd = rangeStart + bucketCount * bucketSizeInMs;
 
       logger.info(
-        `Requesting ${new Date(bucketFrom).toISOString()} to ${new Date(bucketTo).toISOString()}`
+        `Requesting ${new Date(rangeStart).toISOString()} to ${new Date(
+          rangeEnd
+        ).toISOString()} in ${bucketCount} bucket(s)`
       );
 
-      const generatorsAndClients = generate({
-        range: timerange(bucketFrom.getTime(), bucketTo.getTime()),
-        clients: {
-          logsEsClient,
-          apmEsClient,
-          infraEsClient,
-          entitiesEsClient,
-          syntheticsEsClient,
-          otelEsClient,
-        },
-      });
+      const generatorsAndClients = castArray(
+        generate({
+          range: timerange(rangeStart, rangeEnd, logger),
+          clients,
+        })
+      );
 
-      const generatorsAndClientsArray = castArray(generatorsAndClients);
+      await Promise.all(
+        generatorsAndClients.map(async ({ generator, client }) => {
+          await streamManager.index(client, generator);
+        })
+      );
 
-      const streams = generatorsAndClientsArray.map(({ client }) => {
-        let stream: PassThrough;
+      logger.debug('Indexing completed');
 
-        if (cachedStreams.has(client)) {
-          stream = cachedStreams.get(client)!;
-        } else {
-          stream = new PassThrough({ objectMode: true });
-          cachedStreams.set(client, stream);
-          client.index(stream);
-        }
-
-        return stream;
-      });
-
-      currentStreams = streams;
-
-      const promises = generatorsAndClientsArray.map(({ generator }, i) => {
-        const concatenatedStream = castArray(generator)
-          .reverse()
-          .reduce<Writable>((prev, current) => {
-            const currentStream = isGeneratorObject(current) ? Readable.from(current) : current;
-            return currentStream.pipe(prev);
-          }, new PassThrough({ objectMode: true }));
-
-        concatenatedStream.pipe(streams[i], { end: false });
-
-        return awaitStream(concatenatedStream);
-      });
-
-      await Promise.all(promises);
-
-      logger.info('Indexing completed');
-
-      const refreshPromise = generatorsAndClientsArray.map(async ({ client }) => {
+      const refreshPromise = generatorsAndClients.map(async ({ client }) => {
         await client.refresh();
       });
 
       await Promise.all(refreshPromise);
-      logger.info('Refreshing completed');
 
-      requestedUntil = bucketTo;
+      logger.debug('Refreshing completed');
+
+      requestedUntil = rangeEnd;
     }
   }
 

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/stream_manager.ts

@@ -0,0 +1,192 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { castArray, once, pull } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
+import { PassThrough, Readable, Writable, finished } from 'stream';
+import { Fields } from '@kbn/apm-synthtrace-client';
+import { isGeneratorObject } from 'util/types';
+import { Worker, parentPort } from 'worker_threads';
+import { SynthtraceEsClient } from '../../lib/shared/base_client';
+import { SynthGenerator } from '../../lib/utils/with_client';
+import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
+
+// execute a callback when one of the kill signals is received
+function attach(logger: ToolingLog, cb: () => Promise<void>) {
+  const disconnect = () => {
+    process.off('SIGINT', wrapped);
+    process.off('SIGTERM', wrapped);
+    process.off('SIGQUIT', wrapped);
+  };
+
+  const wrapped = once(() => {
+    disconnect();
+    cb()
+      .then(() => {
+        process.exit(0);
+      })
+      .catch((err) => {
+        logger.error(err);
+        process.exit(1);
+      });
+  });
+
+  process.on('SIGINT', wrapped);
+  process.on('SIGTERM', wrapped);
+  process.on('SIGQUIT', wrapped);
+}
+/**
+ * StreamManager waits until streams have completed,
+ * and then calls a teardown callback.
+ */
+
+const asyncNoop = async () => {};
+
+export class StreamManager {
+  private readonly clientStreams: Map<SynthtraceEsClient<Fields>, PassThrough> = new Map();
+  private readonly trackedStreams: Writable[] = [];
+  private readonly trackedWorkers: Worker[] = [];
+
+  constructor(
+    private readonly logger: ToolingLog,
+    private readonly teardownCallback: () => Promise<void> = asyncNoop
+  ) {
+    attach(this.logger, () => this.teardown());
+
+    parentPort?.on('message', (message) => {
+      if (message === 'shutdown') {
+        this.teardown()
+          .then(() => {
+            process.exit(0);
+          })
+          .catch(() => {
+            process.exit(1);
+          });
+      }
+    });
+  }
+
+  trackWorker(worker: Worker) {
+    const untrack = () => {
+      pull(this.trackedWorkers, worker);
+    };
+    worker.on('error', () => {
+      untrack();
+    });
+    worker.on('exit', () => {
+      untrack();
+    });
+    this.trackedWorkers.push(worker);
+  }
+
+  /**
+   * Create a single stream per client, and index data
+   * received from the generator into that stream.
+   */
+  async index(
+    client: SynthtraceEsClient<Fields>,
+    generator: SynthGenerator<Fields>
+  ): Promise<void> {
+    const clientStream = this.createOrReuseClientStream(client);
+
+    const generatorStream = castArray(generator)
+      .reverse()
+      .reduce<Writable>((prev, current) => {
+        const currentStream = isGeneratorObject(current) ? Readable.from(current) : current;
+        return currentStream.pipe(prev);
+      }, new PassThrough({ objectMode: true }));
+
+    // the generator stream should write to the client
+    // stream, but not end it, as the next buckets will
+    // create a new generator
+    generatorStream.pipe(clientStream, { end: false });
+
+    // track the stream for later to end it if needed
+    this.trackedStreams.push(generatorStream);
+
+    await awaitStream(generatorStream).finally(() => {
+      pull(this.trackedStreams, generatorStream);
+    });
+  }
+
+  private createOrReuseClientStream(client: SynthtraceEsClient<Fields>) {
+    let stream: PassThrough;
+
+    if (this.clientStreams.has(client)) {
+      stream = this.clientStreams.get(client)!;
+    } else {
+      stream = new PassThrough({ objectMode: true });
+      this.clientStreams.set(client, stream);
+      client.index(stream);
+    }
+    return stream;
+  }
+
+  teardown = once(async () => {
+    // If a signal is received during teardown,
+    // we just quit forcefully.
+    attach(this.logger, async () => {
+      this.logger.error(`Force-quitting after receiving kill signal`);
+      process.exit(1);
+    });
+
+    this.logger.info('Tearing down after kill signal');
+
+    // end all streams and listen until they've
+    // completed
+    function endStream(stream: Writable) {
+      return new Promise<void>((resolve, reject) => {
+        stream.end();
+        finished(stream, (err) => {
+          if (err) {
+            return reject(err);
+          }
+          resolve();
+        });
+      });
+    }
+
+    if (this.trackedStreams.length) {
+      // ending generator streams
+      this.logger.debug(`Ending ${this.trackedStreams.length} tracked streams`);
+
+      await Promise.all(this.trackedStreams.map(endStream));
+    }
+
+    if (this.trackedWorkers.length) {
+      // give workers a chance to gracefully shut down
+      this.logger.debug(`Shutting down ${this.trackedWorkers.length} workers`);
+
+      await Promise.all(
+        this.trackedWorkers.map((worker) => {
+          return new Promise<void>((resolve, reject) => {
+            worker.postMessage('shutdown');
+            worker.on('exit', () => {
+              resolve();
+            });
+            setTimeout(() => {
+              reject(`Failed to gracefully shutdown worker in time, terminating`);
+            }, 10000);
+          });
+        })
+      );
+    }
+
+    const clientStreams = Array.from(this.clientStreams.values());
+
+    if (clientStreams.length) {
+      // ending client streams
+      this.logger.debug(`Ending ${clientStreams.length} client streams`);
+
+      await Promise.all(clientStreams.map(endStream));
+    }
+
+    await this.teardownCallback();
+  });
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts

@@ -7,153 +7,46 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { timerange } from '@kbn/apm-synthtrace-client';
-import { castArray } from 'lodash';
-import { memoryUsage } from 'process';
 import { parentPort, workerData } from 'worker_threads';
-import { getApmEsClient } from './get_apm_es_client';
-import { getEntitiesKibanaClient } from './get_entites_kibana_client';
-import { getEntitiesEsClient } from './get_entities_es_client';
-import { getInfraEsClient } from './get_infra_es_client';
-import { getLogsEsClient } from './get_logs_es_client';
-import { getOtelSynthtraceEsClient } from './get_otel_es_client';
-import { getScenario } from './get_scenario';
-import { getSyntheticsEsClient } from './get_synthetics_es_client';
+import { bootstrap } from './bootstrap';
+import { indexHistoricalData } from './index_historical_data';
 import { loggerProxy } from './logger_proxy';
 import { RunOptions } from './parse_run_cli_flags';
+import { StreamManager } from './stream_manager';
 
 export interface WorkerData {
+  from: number;
+  to: number;
   bucketFrom: Date;
   bucketTo: Date;
   runOptions: RunOptions;
   workerId: string;
-  esUrl: string;
-  version: string;
-  kibanaUrl: string;
 }
 
-const { bucketFrom, bucketTo, runOptions, esUrl, version, kibanaUrl } = workerData as WorkerData;
+const { bucketFrom, bucketTo, runOptions, workerId, from, to } = workerData as WorkerData;
 
 async function start() {
   const logger = loggerProxy;
-  const entitiesEsClient = getEntitiesEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
+
+  const streamManager = new StreamManager(logger);
+
+  const { clients } = await bootstrap({
+    ...runOptions,
+    skipClientBootstrap: true,
+    clean: false,
+  });
+
+  await indexHistoricalData({
+    bucketFrom,
+    bucketTo,
+    clients,
     logger,
+    runOptions,
+    workerId,
+    from,
+    to,
+    streamManager,
   });
-
-  const entitiesKibanaClient = getEntitiesKibanaClient({
-    target: kibanaUrl,
-    logger,
-  });
-
-  const apmEsClient = getApmEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
-    logger,
-    version,
-  });
-
-  const logsEsClient = getLogsEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
-    logger,
-  });
-
-  const infraEsClient = getInfraEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
-    logger,
-  });
-
-  const syntheticsEsClient = getSyntheticsEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
-    logger,
-  });
-
-  const otelEsClient = getOtelSynthtraceEsClient({
-    concurrency: runOptions.concurrency,
-    target: esUrl,
-    logger,
-  });
-
-  const file = runOptions.file;
-
-  const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
-
-  logger.info(`Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()}`);
-
-  const { generate, bootstrap, teardown } = await scenario({ ...runOptions, logger });
-
-  if (bootstrap) {
-    await bootstrap({
-      apmEsClient,
-      logsEsClient,
-      infraEsClient,
-      syntheticsEsClient,
-      otelEsClient,
-      entitiesEsClient,
-      entitiesKibanaClient,
-    });
-  }
-
-  logger.debug('Generating scenario');
-
-  const generatorsAndClients = logger.perf('generate_scenario', () =>
-    generate({
-      range: timerange(bucketFrom, bucketTo),
-      clients: {
-        logsEsClient,
-        apmEsClient,
-        infraEsClient,
-        entitiesEsClient,
-        syntheticsEsClient,
-        otelEsClient,
-      },
-    })
-  );
-
-  const generatorsAndClientsArray = castArray(generatorsAndClients);
-
-  logger.debug('Indexing scenario');
-
-  function mb(value: number): string {
-    return Math.round(value / 1024 ** 2).toString() + 'mb';
-  }
-
-  let cpuUsage = process.cpuUsage();
-
-  setInterval(async () => {
-    cpuUsage = process.cpuUsage(cpuUsage);
-    const mem = memoryUsage();
-    logger.info(
-      `cpu time: (user: ${cpuUsage.user}µs, sys: ${cpuUsage.system}µs), memory: ${mb(
-        mem.heapUsed
-      )}/${mb(mem.heapTotal)}`
-    );
-  }, 5000);
-
-  await logger.perf('index_scenario', async () => {
-    const promises = generatorsAndClientsArray.map(async ({ client, generator }) => {
-      await client.index(generator);
-      await client.refresh();
-    });
-
-    await Promise.all(promises);
-  });
-
-  if (teardown) {
-    await teardown({
-      apmEsClient,
-      logsEsClient,
-      infraEsClient,
-      syntheticsEsClient,
-      otelEsClient,
-      entitiesEsClient,
-      entitiesKibanaClient,
-    });
-  }
 }
 
 parentPort!.on('message', (message) => {

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_es_client/index.ts

@@ -29,7 +29,7 @@ export interface ApmSynthtraceEsClientOptions extends Omit<SynthtraceEsClientOpt
 }
 
 export class ApmSynthtraceEsClient extends SynthtraceEsClient<ApmFields> {
-  private version: string;
+  public readonly version: string;
 
   constructor(options: { client: Client; logger: Logger } & ApmSynthtraceEsClientOptions) {
     super({

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts

@@ -7,25 +7,22 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import fetch from 'node-fetch';
 import pRetry from 'p-retry';
+import { KibanaClient, KibanaClientHttpError } from '../../shared/base_kibana_client';
 import { Logger } from '../../utils/create_logger';
-import { kibanaHeaders } from '../../shared/client_headers';
-import { getFetchAgent } from '../../../cli/utils/ssl';
+import { getKibanaClient } from '../../../cli/utils/get_kibana_client';
 
 export class ApmSynthtraceKibanaClient {
+  private readonly kibanaClient: KibanaClient;
   private readonly logger: Logger;
-  private target: string;
-  private headers: Record<string, string>;
 
-  constructor(options: { logger: Logger; target: string; headers?: Record<string, string> }) {
+  constructor(options: { logger: Logger } & ({ target: string } | { kibanaClient: KibanaClient })) {
+    this.kibanaClient = 'kibanaClient' in options ? options.kibanaClient : getKibanaClient(options);
     this.logger = options.logger;
-    this.target = options.target;
-    this.headers = { ...kibanaHeaders(), ...(options.headers ?? {}) };
   }
 
   getFleetApmPackagePath(packageVersion?: string): string {
-    let path = `${this.target}/api/fleet/epm/packages/apm`;
+    let path = `/api/fleet/epm/packages/apm`;
     if (packageVersion) {
       path = `${path}/${packageVersion}`;
     }
@@ -39,26 +36,21 @@ export class ApmSynthtraceKibanaClient {
       const url = `${this.getFleetApmPackagePath()}?prerelease=${prerelease}`;
       this.logger.debug(`Fetching from URL: ${url}`);
 
-      const response = await fetch(url, {
-        method: 'GET',
-        headers: kibanaHeaders(),
-        agent: getFetchAgent(url),
-      });
+      const response = await this.kibanaClient
+        .fetch<{ item: { latestVersion?: string } }>(url, {
+          method: 'GET',
+        })
+        .catch((error) => {
+          const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0;
+          throw new Error(
+            `Failed to fetch APM package version, received HTTP ${statusCode} and message: ${error.message}`
+          );
+        });
 
-      const responseJson = await response.json();
-
-      if (response.status !== 200) {
-        throw new Error(
-          `Failed to fetch APM package version, received HTTP ${response.status} and message: ${responseJson.message}`
-        );
+      if (!response.item.latestVersion) {
+        throw new Error(`Failed to fetch APM package version`);
       }
-
-      // Add support for 7.x stack as latest version is available under different node
-      if (responseJson.response && responseJson.response.latestVersion) {
-        return responseJson.response.latestVersion as string;
-      }
-
-      return responseJson.item.latestVersion as string;
+      return response.item.latestVersion;
     };
 
     try {
@@ -67,9 +59,7 @@ export class ApmSynthtraceKibanaClient {
       this.logger.debug(
         'Fetching latestes prerelease version failed, retrying with latest GA version'
       );
-      const retryResult = await fetchPackageVersion({ prerelease: false }).catch((retryError) => {
-        throw retryError;
-      });
+      const retryResult = await fetchPackageVersion({ prerelease: false });
 
       return retryResult;
     }
@@ -84,23 +74,18 @@ export class ApmSynthtraceKibanaClient {
     const url = this.getFleetApmPackagePath(packageVersion);
     const response = await pRetry(
       async () => {
-        const res = await fetch(url, {
-          method: 'POST',
-          headers: this.headers,
-          body: '{"force":true}',
-          agent: getFetchAgent(url),
-        });
+        const res = await this.kibanaClient
+          .fetch<{ items: unknown[] }>(url, {
+            method: 'POST',
+            body: '{"force":true}',
+          })
+          .catch((error) => {
+            const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0;
+            throw new Error(
+              `APM package installation returned ${statusCode} status code\nError: ${error.message}`
+            );
+          });
 
-        if (!res.ok) {
-          const errorJson = await res.json();
-          const errorMessage =
-            typeof errorJson.message === 'string'
-              ? errorJson.message
-              : 'An error occurred during APM package installation.';
-          throw new Error(
-            `APM package installation returned ${res.status} status code\nError: ${errorMessage}`
-          );
-        }
         return res;
       },
       {
@@ -113,12 +98,8 @@ export class ApmSynthtraceKibanaClient {
       }
     );
 
-    const responseJson = await response.json();
-
-    if (!responseJson.items) {
-      throw new Error(
-        `No installed assets received for APM package version ${packageVersion}, received HTTP ${response.status} for url ${url}`
-      );
+    if (!response.items) {
+      throw new Error(`No installed assets received for APM package version ${packageVersion}`);
     }
 
     this.logger.info(`Installed APM package ${packageVersion}`);
@@ -132,23 +113,18 @@ export class ApmSynthtraceKibanaClient {
     const url = this.getFleetApmPackagePath(latestApmPackageVersion);
     const response = await pRetry(
       async () => {
-        const res = await fetch(url, {
-          method: 'DELETE',
-          headers: this.headers,
-          body: '{"force":true}',
-          agent: getFetchAgent(url),
-        });
+        const res = await this.kibanaClient
+          .fetch<{ items: unknown[] }>(url, {
+            method: 'DELETE',
+            body: '{"force":true}',
+          })
+          .catch((error) => {
+            const statusCode = error instanceof KibanaClientHttpError ? error.statusCode : 0;
+            throw new Error(
+              `APM package uninstallation returned ${statusCode} status code\nError: ${error.message}`
+            );
+          });
 
-        if (!res.ok) {
-          const errorJson = await res.json();
-          const errorMessage =
-            typeof errorJson.message === 'string'
-              ? errorJson.message
-              : 'An error occurred during APM package uninstallation.';
-          throw new Error(
-            `APM package uninstallation returned ${res.status} status code\nError: ${errorMessage}`
-          );
-        }
         return res;
       },
       {
@@ -163,11 +139,9 @@ export class ApmSynthtraceKibanaClient {
       }
     );
 
-    const responseJson = await response.json();
-
-    if (!responseJson.items) {
+    if (!response.items) {
       throw new Error(
-        `No uninstalled assets received for APM package version ${latestApmPackageVersion}, received HTTP ${response.status} for url ${url}`
+        `No uninstalled assets received for APM package version ${latestApmPackageVersion}`
       );
     }
 

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_kibana_client.ts

@@ -7,10 +7,9 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import fetch from 'node-fetch';
 import { Logger } from '../utils/create_logger';
-import { kibanaHeaders } from '../shared/client_headers';
-import { getFetchAgent } from '../../cli/utils/ssl';
+import { KibanaClient } from '../shared/base_kibana_client';
+import { getKibanaClient } from '../../cli/utils/get_kibana_client';
 
 interface EntityDefinitionResponse {
   definitions: Array<{ type: string; state: { installed: boolean; running: boolean } }>;
@@ -18,21 +17,20 @@ interface EntityDefinitionResponse {
 
 export class EntitiesSynthtraceKibanaClient {
   private readonly logger: Logger;
-  private target: string;
+  private readonly kibana: KibanaClient;
 
-  constructor(options: { logger: Logger; target: string }) {
+  constructor(options: { logger: Logger } & ({ target: string } | { kibanaClient: KibanaClient })) {
+    this.kibana = 'kibanaClient' in options ? options.kibanaClient : getKibanaClient(options);
     this.logger = options.logger;
-    this.target = options.target;
   }
 
   async installEntityIndexPatterns() {
-    const url = `${this.target}/internal/entities/definition?includeState=true`;
-    const response = await fetch(url, {
-      method: 'GET',
-      headers: kibanaHeaders(),
-      agent: getFetchAgent(url),
-    });
-    const entityDefinition: EntityDefinitionResponse = await response.json();
+    const entityDefinition = await this.kibana.fetch<EntityDefinitionResponse>(
+      `/internal/entities/definition?includeState=true`,
+      {
+        method: 'GET',
+      }
+    );
 
     const hasEntityDefinitionsInstalled = entityDefinition.definitions?.find(
       (definition) => definition.type === 'service'
@@ -42,21 +40,15 @@ export class EntitiesSynthtraceKibanaClient {
       this.logger.debug('Entity definitions are already defined');
     } else {
       this.logger.debug('Installing Entity definitions');
-      const entityEnablementUrl = `${this.target}/internal/entities/managed/enablement?installOnly=true`;
-      await fetch(entityEnablementUrl, {
+      await this.kibana.fetch(`/internal/entities/managed/enablement?installOnly=true`, {
         method: 'PUT',
-        headers: kibanaHeaders(),
-        agent: getFetchAgent(url),
       });
     }
   }
 
   async uninstallEntityIndexPatterns() {
-    const url = `${this.target}/internal/entities/managed/enablement`;
-    await fetch(url, {
+    await this.kibana.fetch(`/internal/entities/managed/enablement`, {
       method: 'DELETE',
-      headers: kibanaHeaders(),
-      agent: getFetchAgent(url),
     });
   }
 }

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_client.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Client } from '@elastic/elasticsearch';
+import { Client, errors } from '@elastic/elasticsearch';
 import {
   ESDocumentWithOperation,
   Fields,
@@ -19,8 +19,12 @@ import { Readable, Transform } from 'stream';
 import { isGeneratorObject } from 'util/types';
 import { Logger } from '../utils/create_logger';
 import { sequential } from '../utils/stream_utils';
+import { KibanaClient } from './base_kibana_client';
 
 export interface SynthtraceEsClientOptions {
+  client: Client;
+  kibana?: KibanaClient;
+  logger: Logger;
   concurrency?: number;
   refreshAfterIndex?: boolean;
   pipeline: (base: Readable) => NodeJS.WritableStream;
@@ -30,6 +34,7 @@ type MaybeArray<T> = T | T[];
 
 export class SynthtraceEsClient<TFields extends Fields> {
   protected readonly client: Client;
+  protected readonly kibana?: KibanaClient;
   protected readonly logger: Logger;
 
   private readonly concurrency: number;
@@ -39,8 +44,9 @@ export class SynthtraceEsClient<TFields extends Fields> {
   protected dataStreams: string[] = [];
   protected indices: string[] = [];
 
-  constructor(options: { client: Client; logger: Logger } & SynthtraceEsClientOptions) {
+  constructor(options: SynthtraceEsClientOptions) {
     this.client = options.client;
+    this.kibana = options.kibana;
     this.logger = options.logger;
     this.concurrency = options.concurrency ?? 1;
     this.refreshAfterIndex = options.refreshAfterIndex ?? false;
@@ -67,10 +73,17 @@ export class SynthtraceEsClient<TFields extends Fields> {
     await Promise.all([
       ...(this.dataStreams.length
         ? [
-            this.client.indices.deleteDataStream({
-              name: this.dataStreams.join(','),
-              expand_wildcards: ['open', 'hidden'],
-            }),
+            this.client.indices
+              .deleteDataStream({
+                name: this.dataStreams.join(','),
+                expand_wildcards: ['open', 'hidden'],
+              })
+              .catch((error) => {
+                if (error instanceof errors.ResponseError && error.statusCode === 404) {
+                  return;
+                }
+                throw error;
+              }),
           ]
         : []),
       ...(resolvedIndices.length
@@ -105,7 +118,7 @@ export class SynthtraceEsClient<TFields extends Fields> {
   async index(
     streamOrGenerator: MaybeArray<Readable | SynthtraceGenerator<TFields>>,
     pipelineCallback?: (base: Readable) => NodeJS.WritableStream
-  ) {
+  ): Promise<void> {
     this.logger.debug(`Bulk indexing ${castArray(streamOrGenerator).length} stream(s)`);
 
     const previousPipelineCallback = this.pipelineCallback;
@@ -135,9 +148,9 @@ export class SynthtraceEsClient<TFields extends Fields> {
         count++;
 
         if (count % 100000 === 0) {
-          this.logger.info(`Indexed ${count} documents`);
-        } else if (count % 1000 === 0) {
           this.logger.debug(`Indexed ${count} documents`);
+        } else if (count % 1000 === 0) {
+          this.logger.verbose(`Indexed ${count} documents`);
         }
 
         if (doc._action) {

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/base_kibana_client.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/* eslint-disable max-classes-per-file*/
+
+import fetch from 'node-fetch';
+import { RequestInit } from 'node-fetch';
+import Path from 'path';
+import { kibanaHeaders } from './client_headers';
+import { getFetchAgent } from '../../cli/utils/ssl';
+
+type KibanaClientFetchOptions = RequestInit;
+
+export class KibanaClientHttpError extends Error {
+  constructor(message: string, public readonly statusCode: number, public readonly data?: unknown) {
+    super(message);
+  }
+}
+
+export class KibanaClient {
+  private target: string;
+  private headers: Record<string, string>;
+
+  constructor(options: { target: string; headers?: Record<string, string> }) {
+    this.target = options.target;
+    this.headers = { ...kibanaHeaders(), ...(options.headers ?? {}) };
+  }
+
+  fetch<T>(pathname: string, options: KibanaClientFetchOptions): Promise<T> {
+    const url = Path.join(this.target, pathname);
+    return fetch(url, {
+      ...options,
+      headers: {
+        ...this.headers,
+        ...options.headers,
+      },
+      agent: getFetchAgent(url),
+    }).then(async (response) => {
+      if (response.status >= 400) {
+        throw new KibanaClientHttpError(
+          `Response error for ${options.method?.toUpperCase() ?? 'GET'} ${url}`,
+          response.status,
+          await response.json().catch((error) => {
+            return undefined;
+          })
+        );
+      }
+      return response.json();
+    });
+  }
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/shared/client_headers.ts

@@ -15,3 +15,9 @@ export function kibanaHeaders() {
     'elastic-api-version': '2023-10-31',
   };
 }
+
+export function internalKibanaHeaders() {
+  return {
+    'x-elastic-internal-origin': 'kibana',
+  };
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/streams/streams_synthtrace_client.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client';
+import { Condition, StreamUpsertRequest } from '@kbn/streams-schema';
+import { Readable, Transform, pipeline } from 'stream';
+import { Required } from 'utility-types';
+import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
+import { internalKibanaHeaders } from '../shared/client_headers';
+import { getSerializeTransform } from '../shared/get_serialize_transform';
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+interface StreamsDocument {}
+
+export class StreamsSynthtraceClient extends SynthtraceEsClient<StreamsDocument> {
+  constructor(options: Required<Omit<SynthtraceEsClientOptions, 'pipeline'>, 'kibana'>) {
+    super({
+      ...options,
+      pipeline: streamsPipeline(),
+    });
+    this.dataStreams = ['logs', 'logs.*'];
+  }
+
+  async forkStream(
+    streamName: string,
+    request: { stream: { name: string }; if: Condition }
+  ): Promise<{ acknowledged: true }> {
+    return this.kibana!.fetch(`/api/streams/${streamName}/_fork`, {
+      method: 'POST',
+      headers: {
+        ...internalKibanaHeaders(),
+      },
+      body: JSON.stringify(request),
+    });
+  }
+
+  async putStream(
+    streamName: string,
+    request: StreamUpsertRequest
+  ): Promise<{ acknowledged: true; result: 'created' | 'updated' }> {
+    return this.kibana!.fetch(`/api/streams/${streamName}`, {
+      method: 'PUT',
+      headers: {
+        ...internalKibanaHeaders(),
+      },
+      body: JSON.stringify(request),
+    });
+  }
+
+  async enable() {
+    await this.kibana!.fetch('/api/streams/_enable', {
+      method: 'POST',
+      headers: {
+        ...internalKibanaHeaders(),
+      },
+    });
+  }
+
+  async disable() {
+    await this.kibana!.fetch('/api/streams/_disable', {
+      method: 'POST',
+      timeout: 5 * 60 * 1000,
+      headers: {
+        ...internalKibanaHeaders(),
+      },
+    });
+  }
+
+  override async clean(): Promise<void> {
+    await this.disable();
+    await super.clean();
+  }
+
+  async clearESCache(): Promise<void> {
+    await this.client.indices.clearCache();
+  }
+}
+
+function streamsRoutingTransform() {
+  return new Transform({
+    objectMode: true,
+    transform(document: ESDocumentWithOperation<StreamsDocument>, encoding, callback) {
+      document._index = 'logs';
+      callback(null, document);
+    },
+  });
+}
+
+function streamsPipeline() {
+  return (base: Readable) => {
+    return pipeline(
+      base,
+      getSerializeTransform<StreamsDocument>(),
+      streamsRoutingTransform(),
+      (err: unknown) => {
+        if (err) {
+          throw err;
+        }
+      }
+    );
+  };
+}

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/create_logger.ts

@@ -7,58 +7,41 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { ToolingLog, Writer } from '@kbn/tooling-log';
 import { logPerf } from './log_perf';
 
 export enum LogLevel {
-  trace = 0,
-  debug = 1,
-  info = 2,
-  warn = 3,
-  error = 4,
+  verbose = 'verbose',
+  debug = 'debug',
+  info = 'info',
+  warn = 'warning',
+  error = 'error',
 }
 
-function getTimeString() {
-  return `[${new Date().toLocaleTimeString()}]`;
-}
-
-export function createLogger(logLevel: LogLevel) {
-  const logger: Logger = {
-    perf: (name, callback) => {
-      return logPerf(logger, logLevel, name, callback);
-    },
-    debug: (...args: any[]) => {
-      if (logLevel <= LogLevel.debug) {
-        // eslint-disable-next-line no-console
-        console.debug(getTimeString(), ...args);
-      }
-    },
-    info: (...args: any[]) => {
-      if (logLevel <= LogLevel.info) {
-        // eslint-disable-next-line no-console
-        console.log(getTimeString(), ...args);
-      }
-    },
-    warn: (...args: any[]) => {
-      if (logLevel <= LogLevel.warn) {
-        // eslint-disable-next-line no-console
-        console.warn(getTimeString(), ...args);
-      }
-    },
-    error: (...args: any[]) => {
-      if (logLevel <= LogLevel.error) {
-        // eslint-disable-next-line no-console
-        console.log(getTimeString(), ...args);
-      }
-    },
+export function extendToolingLog(log: ToolingLog, logLevel: LogLevel = LogLevel.verbose): Logger {
+  const logger = log as Logger;
+  logger.perf = (name: string, callback: () => any) => {
+    return logPerf(log, LogLevel.verbose, name, callback);
   };
 
   return logger;
 }
 
-export interface Logger {
-  perf: <T>(name: string, cb: () => T) => T;
-  debug: (...args: any[]) => void;
-  info: (...args: any[]) => void;
-  warn: (...args: any[]) => void;
-  error: (...args: any[]) => void;
+export function createLogger(logLevel: LogLevel, writer?: Writer): Logger {
+  const log = new ToolingLog({
+    level: logLevel,
+    writeTo: writer ? { write: () => true } : process.stdout,
+  }) as Logger;
+
+  if (writer) {
+    log.setWriters([writer]);
+  }
+
+  extendToolingLog(log, logLevel);
+
+  return log;
 }
+
+export type Logger = ToolingLog & {
+  perf: <T>(name: string, cb: () => T) => T;
+};

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/log_perf.ts

@@ -7,23 +7,24 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Logger, LogLevel } from './create_logger';
+import { ToolingLog } from '@kbn/tooling-log';
+import { LogLevel } from './create_logger';
 
 function isPromise(val: any): val is Promise<any> {
   return val && typeof val === 'object' && 'then' in val && typeof val.then === 'function';
 }
 
-function logTo(logger: Logger, name: string, start: bigint) {
+function logTo(logger: ToolingLog, name: string, start: bigint) {
   logger.debug(`${name}: ${Number(process.hrtime.bigint() - start) / 1000000}ms`);
 }
 
 export const logPerf = <T extends any>(
-  logger: Logger,
+  logger: ToolingLog,
   logLevel: LogLevel,
   name: string,
   cb: () => T
 ): T => {
-  if (logLevel <= LogLevel.trace) {
+  if (logLevel === LogLevel.verbose) {
     const start = process.hrtime.bigint();
     const val = cb();
     if (isPromise(val)) {

src/platform/packages/shared/kbn-apm-synthtrace/src/lib/utils/with_client.ts

@@ -7,20 +7,17 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { SynthtraceGenerator } from '@kbn/apm-synthtrace-client';
+import { Fields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client';
 import { Readable } from 'stream';
 import { SynthtraceEsClient } from '../shared/base_client';
 
-export type SynthGenerator<TFields> =
-  // @ts-expect-error upgrade typescript v4.9.5
+export type SynthGenerator<TFields extends Fields> =
   | SynthtraceGenerator<TFields>
-  // @ts-expect-error upgrade typescript v4.9.5
   | Array<SynthtraceGenerator<TFields>>
   | Readable;
 
-export const withClient = <TFields>(
-  // @ts-expect-error upgrade typescript v4.9.5
-  client: SynthtraceEsClient,
+export const withClient = <TFields extends Fields>(
+  client: SynthtraceEsClient<TFields>,
   generator: SynthGenerator<TFields>
 ) => {
   return {
@@ -29,4 +26,4 @@ export const withClient = <TFields>(
   };
 };
 
-export type ScenarioReturnType<TFields> = ReturnType<typeof withClient<TFields>>;
+export type ScenarioReturnType<TFields extends Fields> = ReturnType<typeof withClient<TFields>>;

src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/sample_logs.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { LogDocument, Serializable } from '@kbn/apm-synthtrace-client';
+import { SampleParserClient } from '@kbn/sample-log-parser';
+import { Scenario } from '../cli/scenario';
+import { withClient } from '../lib/utils/with_client';
+
+const scenario: Scenario<LogDocument> = async (runOptions) => {
+  const client = new SampleParserClient({ logger: runOptions.logger });
+
+  const { rpm } = (runOptions.scenarioOpts ?? {}) as { rpm?: number };
+
+  const generators = await client.getLogGenerators({
+    rpm,
+  });
+
+  return {
+    bootstrap: async ({ streamsClient }) => {
+      await streamsClient.enable();
+    },
+    generate: ({ range, clients: { streamsClient } }) => {
+      return withClient(
+        streamsClient,
+        range.interval('5s').generator((timestamp) => {
+          return generators.flatMap((generator) =>
+            generator.next(timestamp).map((doc) => new Serializable(doc))
+          );
+        })
+      );
+    },
+  };
+};
+
+export default scenario;

src/platform/packages/shared/kbn-apm-synthtrace/src/scenarios/traces_logs_entities.ts

@@ -14,7 +14,6 @@ import {
   Instance,
   log,
   entities,
-  EntityFields,
 } from '@kbn/apm-synthtrace-client';
 import { Readable } from 'stream';
 import { Scenario } from '../cli/scenario';
@@ -35,7 +34,7 @@ const SYNTH_JAVA_TRACE_ENTITY_ID = generateShortId();
 const SYNTH_NODE_TRACES_LOGS_ENTITY_ID = generateShortId();
 const SYNTH_GO_LOGS_ENTITY_ID = generateShortId();
 
-const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
+const scenario: Scenario = async (runOptions) => {
   const { logger } = runOptions;
   const { isLogsDb } = parseLogsScenarioOpts(runOptions.scenarioOpts);
 

src/platform/packages/shared/kbn-apm-synthtrace/src/test/es_client_indexer.test.ts

@@ -11,6 +11,7 @@ import { apm, timerange } from '@kbn/apm-synthtrace-client';
 import { pick, range, sum } from 'lodash';
 import { Readable } from 'stream';
 import { ApmSynthtraceEsClient } from '../lib/apm/client/apm_synthtrace_es_client';
+import { ToolingLog } from '@kbn/tooling-log';
 
 describe('Synthtrace ES Client indexer', () => {
   let apmEsClient: ApmSynthtraceEsClient;
@@ -58,7 +59,8 @@ describe('Synthtrace ES Client indexer', () => {
 
     const generator = timerange(
       new Date('2022-01-01T00:00:00.000Z'),
-      new Date('2022-01-01T01:00:01.000Z')
+      new Date('2022-01-01T01:00:01.000Z'),
+      {} as ToolingLog
     )
       .interval('30m')
       .generator((timestamp) => {
@@ -99,7 +101,11 @@ describe('Synthtrace ES Client indexer', () => {
     });
 
     const getGenerator = () =>
-      timerange(new Date('2022-01-01T00:00:00.000Z'), new Date('2022-01-01T00:59:59.999Z'))
+      timerange(
+        new Date('2022-01-01T00:00:00.000Z'),
+        new Date('2022-01-01T00:59:59.999Z'),
+        {} as ToolingLog
+      )
         .interval('20m')
         .rate(10)
         .generator(generatorCallback);
@@ -142,7 +148,8 @@ describe('Synthtrace ES Client indexer', () => {
 
     const generator = timerange(
       new Date('2022-01-01T00:00:00.000Z'),
-      new Date('2022-01-01T00:06:59.999Z')
+      new Date('2022-01-01T00:06:59.999Z'),
+      {} as ToolingLog
     )
       .interval('1m')
       .rate(RATE)

src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/01_simple_trace.test.ts

@@ -8,6 +8,7 @@
  */
 
 import { apm, ApmFields, SynthtraceGenerator, timerange } from '@kbn/apm-synthtrace-client';
+import { ToolingLog } from '@kbn/tooling-log';
 
 describe('simple trace', () => {
   let iterable: SynthtraceGenerator<ApmFields>;
@@ -23,7 +24,8 @@ describe('simple trace', () => {
 
     const range = timerange(
       new Date('2021-01-01T00:00:00.000Z'),
-      new Date('2021-01-01T00:15:00.000Z')
+      new Date('2021-01-01T00:15:00.000Z'),
+      {} as ToolingLog
     );
 
     iterable = range

src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/02_transaction_metrics.test.ts

@@ -12,6 +12,7 @@ import { sortBy } from 'lodash';
 import { Readable } from 'stream';
 import { createTransactionMetricsAggregator } from '../../lib/apm/aggregators/create_transaction_metrics_aggregator';
 import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
+import { ToolingLog } from '@kbn/tooling-log';
 
 describe('transaction metrics', () => {
   let events: Array<Record<string, any>>;
@@ -26,7 +27,8 @@ describe('transaction metrics', () => {
 
     const range = timerange(
       new Date('2021-01-01T00:00:00.000Z'),
-      new Date('2021-01-01T00:15:00.000Z')
+      new Date('2021-01-01T00:15:00.000Z'),
+      {} as ToolingLog
     );
 
     const span = (timestamp: number) =>

src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/03_span_destination_metrics.test.ts

@@ -12,6 +12,7 @@ import { sortBy } from 'lodash';
 import { Readable } from 'stream';
 import { createSpanMetricsAggregator } from '../../lib/apm/aggregators/create_span_metrics_aggregator';
 import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
+import { ToolingLog } from '@kbn/tooling-log';
 
 describe('span destination metrics', () => {
   let events: Array<Record<string, any>>;
@@ -26,7 +27,8 @@ describe('span destination metrics', () => {
 
     const range = timerange(
       new Date('2021-01-01T00:00:00.000Z'),
-      new Date('2021-01-01T00:15:00.000Z')
+      new Date('2021-01-01T00:15:00.000Z'),
+      {} as ToolingLog
     );
 
     const serialized = [

src/platform/packages/shared/kbn-apm-synthtrace/src/test/scenarios/04_breakdown_metrics.test.ts

@@ -12,6 +12,7 @@ import { Readable } from 'stream';
 import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
 import { createBreakdownMetricsAggregator } from '../../lib/apm/aggregators/create_breakdown_metrics_aggregator';
 import { apm, ApmFields, timerange } from '@kbn/apm-synthtrace-client';
+import { ToolingLog } from '@kbn/tooling-log';
 
 describe('breakdown metrics', () => {
   let events: ApmFields[];
@@ -32,7 +33,11 @@ describe('breakdown metrics', () => {
 
     const start = new Date('2021-01-01T00:00:00.000Z');
 
-    const range = timerange(start, new Date(start.getTime() + INTERVALS * 30 * 1000));
+    const range = timerange(
+      start,
+      new Date(start.getTime() + INTERVALS * 30 * 1000),
+      {} as ToolingLog
+    );
 
     const listSpans = Array.from(
       range

src/platform/packages/shared/kbn-apm-synthtrace/tsconfig.json

@@ -11,6 +11,9 @@
     "@kbn/dev-utils",
     "@kbn/elastic-agent-utils",
     "@kbn/zod",
+    "@kbn/sample-log-parser",
+    "@kbn/tooling-log",
+    "@kbn/streams-schema",
   ],
   "exclude": [
     "target/**/*",

tsconfig.base.json

@@ -1546,6 +1546,8 @@
       "@kbn/safer-lodash-set/*": ["src/platform/packages/shared/kbn-safer-lodash-set/*"],
       "@kbn/saml-provider-plugin": ["x-pack/test/security_api_integration/plugins/saml_provider"],
       "@kbn/saml-provider-plugin/*": ["x-pack/test/security_api_integration/plugins/saml_provider/*"],
+      "@kbn/sample-log-parser": ["x-pack/platform/packages/shared/kbn-sample-parser"],
+      "@kbn/sample-log-parser/*": ["x-pack/platform/packages/shared/kbn-sample-parser/*"],
       "@kbn/sample-task-plugin": ["x-pack/test/plugin_api_integration/plugins/sample_task_plugin"],
       "@kbn/sample-task-plugin/*": ["x-pack/test/plugin_api_integration/plugins/sample_task_plugin/*"],
       "@kbn/sample-task-plugin-update-by-query": ["x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget"],

x-pack/platform/packages/shared/kbn-sample-parser/README.md

@@ -0,0 +1,12 @@
+# @kbn/sample-log-parser
+
+This library downloads, parses and re-generates data from [loghub](https://github.com/logpai/loghub), a collection of sample log data sets.
+
+## CLI
+
+Run `AZURE_OPENAI_ENDPOINT=... AZURE_OPENAI_API_KEY=... node x-pack/scripts/loghub_parser.js` to generate and validate Loghub parsers. Every parser exports functions that extract and replace timestamps in log messages from Loghub systems. A parser is considered valid if the extracted timestamp is the same as the replaced timestamp. If a parser does not exist or is not valid,
+the LLM is used to re-generate a new one.
+
+## SampleParserClient
+
+`SampleParserClient` reads the parsers and the sample datasets, and returns generators that will replay the loghub sample sets with updated timestamps. If the indexing rate is higher than a certain maximum (currently 100rpm), it will replay the sample set at a lower speed.

x-pack/platform/packages/shared/kbn-sample-parser/cli/index.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { Command } from 'commander';
+import pLimit from 'p-limit';
+import { partition } from 'lodash';
+import { ensureLoghubRepo } from '../src/ensure_loghub_repo';
+import { readLoghubSystemFiles } from '../src/read_loghub_system_files';
+import { ensureValidParser } from '../src/ensure_valid_parser';
+import { createOpenAIClient } from '../src/create_openai_client';
+import { ensureValidQueries } from '../src/ensure_valid_queries';
+
+async function run({ log }: { log: ToolingLog }) {
+  await ensureLoghubRepo({ log });
+
+  const systems = await readLoghubSystemFiles({ log });
+  const limiter = pLimit(5);
+
+  const openAIClient = createOpenAIClient();
+
+  const results = await Promise.all(
+    systems.map(async (system) => {
+      return limiter(async () =>
+        Promise.all([
+          ensureValidParser({
+            openAIClient,
+            log,
+            system,
+          }),
+          ensureValidQueries({
+            openAIClient,
+            system,
+            log,
+          }),
+        ])
+          .then(() => {
+            return {
+              name: system.name,
+              error: null,
+            };
+          })
+          .catch((error) => {
+            return {
+              name: system.name,
+              error,
+            };
+          })
+      );
+    })
+  );
+
+  const [valid, invalid] = partition(results, (result) => !result.error);
+  if (invalid.length === 0) {
+    log.info(`Ensured ${valid.length} parsers`);
+    return;
+  }
+
+  invalid.forEach((result) => {
+    log.error(`Failed generating a valid parser for ${result.name}`);
+    log.error(result.error);
+  });
+
+  throw new Error(`${invalid.length} out of ${results.length} parsers are invalid`);
+}
+
+export function cli() {
+  const program = new Command('bin/kibana-setup');
+
+  program
+    .name('loghub-parser')
+    .description(
+      'Generates code to extract and replace timestamps in loglines from Loghub datasets'
+    )
+    .option('-d, --debug', 'Debug logging', false)
+    .option('-v, --verbose', 'Verbose logging', false)
+    .option('-s, --silent', 'Prevent all logging', false)
+    .action(async () => {
+      const options = program.opts() as {
+        silent: boolean;
+        verbose: boolean;
+        debug: boolean;
+      };
+
+      const log = new ToolingLog({
+        level: options.silent
+          ? 'silent'
+          : options.debug
+          ? 'debug'
+          : options.verbose
+          ? 'verbose'
+          : 'info',
+        writeTo: process.stdout,
+      });
+
+      return run({ log });
+    })
+    .parse(process.argv);
+}

x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.test.ts

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// createLoghubGenerator.test.ts
+import { ToolingLog } from '@kbn/tooling-log';
+import { LoghubSystem } from '../src/read_loghub_system_files';
+import { LoghubParser } from '../src/types';
+import { createLoghubGenerator } from './create_loghub_generator';
+
+describe('createLoghubGenerator', () => {
+  let system: LoghubSystem;
+  let parser: LoghubParser;
+  let log: ToolingLog;
+
+  beforeEach(() => {
+    parser = {
+      getTimestamp: (line: string) => parseInt(line, 10),
+      replaceTimestamp: (line: string, timestamp: number) => line,
+    };
+  });
+
+  describe('full speed', () => {
+    beforeEach(() => {
+      system = {
+        name: 'TestSystem',
+        readme: '',
+        // 2rpm
+        logLines: ['0', '60000'],
+        templates: [],
+      };
+      log = new ToolingLog();
+    });
+
+    it('generates the first event at the start time', () => {
+      const generator = createLoghubGenerator({ system, parser, log, queries: [] });
+      const startTime = 100_000;
+      const docs = generator.next(startTime);
+
+      expect(docs).toHaveLength(1);
+
+      expect(docs[0]['@timestamp']).toBe(startTime);
+    });
+
+    it('generates the second event with the right offset', () => {
+      const generator = createLoghubGenerator({ system, parser, log, queries: [] });
+      const startTime = 100_000;
+      generator.next(startTime);
+
+      const docs = generator.next(startTime + 60000);
+      expect(docs).toHaveLength(1);
+      expect(docs[0]['@timestamp']).toBe(startTime + 60000);
+    });
+
+    it('returns no events if current time is before the next event', () => {
+      const generator = createLoghubGenerator({ system, parser, log, queries: [] });
+      const startTime = 100_000;
+      generator.next(startTime);
+
+      const docs = generator.next(startTime + 60000 - 1);
+      expect(docs).toHaveLength(0);
+    });
+  });
+
+  describe('throttled', () => {
+    beforeEach(() => {
+      system = {
+        name: 'TestSystem',
+        readme: '',
+        // 200rpm
+        logLines: ['0', '599'],
+        templates: [],
+      };
+      log = new ToolingLog();
+    });
+
+    it('applies speed throttle when log rate is too high', () => {
+      const generator = createLoghubGenerator({ system, parser, log, targetRpm: 100, queries: [] });
+
+      const startTime = 100_000;
+      const firstBatch = generator.next(startTime);
+
+      expect(firstBatch).toHaveLength(1);
+      // it starts at the usual time
+      expect(firstBatch[0]['@timestamp']).toBe(startTime);
+
+      // after that, the delta should be half of what is expected
+      const secondBatch = generator.next(startTime + 1200);
+      const expectedTimestamp = startTime + 1200;
+
+      expect(secondBatch.length).toBe(1);
+
+      expect(secondBatch[0]['@timestamp']).toBe(expectedTimestamp);
+    });
+  });
+});

x-pack/platform/packages/shared/kbn-sample-parser/client/create_loghub_generator.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { LoghubSystem } from '../src/read_loghub_system_files';
+import { LoghubParser } from '../src/types';
+import { StreamLogDocument, StreamLogGenerator } from './types';
+import { parseDataset } from './parse_dataset';
+import { LoghubQuery } from '../src/validate_queries';
+
+export function createLoghubGenerator({
+  system,
+  queries,
+  parser,
+  log,
+  targetRpm,
+}: {
+  system: LoghubSystem;
+  queries: LoghubQuery[];
+  parser: LoghubParser;
+  log: ToolingLog;
+  targetRpm?: number;
+}): StreamLogGenerator {
+  let index = 0;
+  let start = 0;
+
+  const { rpm: systemRpm, lines, min, range } = parseDataset({ system, parser });
+
+  const count = lines.length;
+
+  const speed = targetRpm === undefined ? 1 : targetRpm / systemRpm;
+
+  const filepath = `${system.name}.log`;
+
+  log.debug(
+    `Throughput for ${system.name} will be around ${Math.round(targetRpm ?? systemRpm)}rpm`
+  );
+
+  return {
+    name: system.name,
+    filepath,
+    queries,
+    next: (timestamp) => {
+      if (index === 0) {
+        start = timestamp;
+      }
+
+      const docs: StreamLogDocument[] = [];
+
+      while (true) {
+        const line = lines[index % count];
+
+        const rotations = Math.floor(index / count);
+
+        const rel = (line.timestamp - min) / range;
+
+        // add 1 ms per rotation to separate start and end events
+        const delta = (1 / speed) * range * (rel + rotations) + rotations;
+
+        // ES likes its timestamp to be an integer
+        const simulatedTimestamp = Math.floor(start + delta);
+
+        if (simulatedTimestamp > timestamp) {
+          break;
+        }
+
+        index++;
+
+        const next = {
+          '@timestamp': simulatedTimestamp,
+          message: parser.replaceTimestamp(line.message, simulatedTimestamp),
+          filepath,
+        };
+
+        docs.push(next);
+      }
+
+      return docs;
+    },
+  };
+}

x-pack/platform/packages/shared/kbn-sample-parser/client/index.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { sumBy } from 'lodash';
+import { StreamLogGenerator } from './types';
+import { readLoghubSystemFiles } from '../src/read_loghub_system_files';
+import { ensureLoghubRepo } from '../src/ensure_loghub_repo';
+import { validateParser } from '../src/validate_parser';
+import { getParser } from '../src/get_parser';
+import { createLoghubGenerator } from './create_loghub_generator';
+import { parseDataset } from './parse_dataset';
+import { validateQueries } from '../src/validate_queries';
+import { getQueries } from '../src/get_queries';
+
+export class SampleParserClient {
+  private readonly logger: ToolingLog;
+  constructor(options: { logger: ToolingLog }) {
+    this.logger = options.logger;
+  }
+
+  async getLogGenerators({
+    rpm = 10000,
+    distribution = 'uniform',
+  }: {
+    rpm?: number;
+    distribution?: 'relative' | 'uniform';
+  }): Promise<StreamLogGenerator[]> {
+    await ensureLoghubRepo({ log: this.logger });
+    const systems = await readLoghubSystemFiles({ log: this.logger });
+
+    const results = await Promise.all(
+      systems.map(async (system) => {
+        await Promise.all([validateParser(system), validateQueries(system)]).catch((error) => {
+          throw new AggregateError([error], `Parser for ${system.name} is not valid`);
+        });
+
+        const [parser, queries] = await Promise.all([getParser(system), getQueries(system)]);
+        const { rpm: systemRpm } = parseDataset({ system, parser });
+
+        return {
+          parser,
+          system,
+          rpm: systemRpm,
+          queries,
+        };
+      })
+    );
+
+    const totalRpm = sumBy(results, ({ rpm: systemRpm }) => systemRpm);
+
+    return await Promise.all(
+      results.map(({ system, parser, rpm: systemRpm, queries }) => {
+        let targetRpm: number;
+        if (distribution === 'relative') {
+          const share = systemRpm / totalRpm;
+          targetRpm = rpm === undefined ? Math.min(100, systemRpm) : share * rpm;
+        } else {
+          targetRpm = Math.round(rpm / results.length);
+        }
+
+        return createLoghubGenerator({
+          system,
+          parser,
+          log: this.logger,
+          targetRpm: Math.max(1, targetRpm),
+          queries,
+        });
+      })
+    );
+  }
+}

x-pack/platform/packages/shared/kbn-sample-parser/client/parse_dataset.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import { LoghubSystem } from '../src/read_loghub_system_files';
+import { LoghubParser } from '../src/types';
+
+export function parseDataset({ system, parser }: { parser: LoghubParser; system: LoghubSystem }) {
+  const parsedLogLines = system.logLines.map((line) => {
+    const timestamp = parser.getTimestamp(line);
+    return {
+      timestamp,
+      message: line,
+    };
+  });
+
+  const min = parsedLogLines[0].timestamp;
+
+  let minTimestamp = min;
+  let years = 0;
+
+  // add years for timestamps without years
+  parsedLogLines.forEach((logLine) => {
+    if (logLine.timestamp < minTimestamp) {
+      minTimestamp = logLine.timestamp;
+      years++;
+    }
+    if (years >= 0) {
+      logLine.timestamp = moment(logLine.timestamp).add(years, 'years').valueOf();
+    }
+  });
+
+  const max = parsedLogLines[parsedLogLines.length - 1].timestamp;
+
+  const count = parsedLogLines.length;
+
+  const range = max - min;
+
+  const rpm = count / (range / 1000 / 60);
+
+  return {
+    lines: parsedLogLines,
+    rpm,
+    range,
+    min,
+    max,
+  };
+}

x-pack/platform/packages/shared/kbn-sample-parser/client/types.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { LoghubQuery } from '../src/validate_queries';
+
+export interface StreamLogDocument {
+  [x: string]: unknown;
+  filepath: string;
+  message: string;
+  '@timestamp': number;
+}
+
+export interface StreamLogGenerator {
+  name: string;
+  filepath: string;
+  queries: LoghubQuery[];
+  next: (timestamp: number) => StreamLogDocument[];
+}

x-pack/platform/packages/shared/kbn-sample-parser/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { cli } from './cli';
+export { SampleParserClient } from './client';
+export { type LoghubQuery, createQueryMatcher, tokenize } from './src/validate_queries';

x-pack/platform/packages/shared/kbn-sample-parser/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/x-pack/platform/packages/shared/kbn-sample-parser'],
+};

x-pack/platform/packages/shared/kbn-sample-parser/kibana.jsonc

@@ -0,0 +1,8 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/sample-log-parser",
+  "owner": "@elastic/streams-program-team",
+  "group": "platform",
+  "visibility": "shared",
+  "devOnly": true
+}

x-pack/platform/packages/shared/kbn-sample-parser/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/sample-log-parser",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/parser.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const [_, month, day, hour, minute, second, millisecond] = match;
+    const dateString = `2023-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`;
+    return moment.utc(dateString).valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = moment.utc(timestamp).format('MM-DD HH:mm:ss.SSS');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Android/queries.json

@@ -0,0 +1,365 @@
+{
+  "queries": [
+    {
+      "id": "register_callback",
+      "title": "Register Callback",
+      "description": "Identifies log messages related to registering a callback.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "register callback",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unregister_callback",
+      "title": "Unregister Callback",
+      "description": "Identifies log messages related to unregistering a callback.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "unregister callback",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unregister_callback_null",
+      "title": "Unregister Callback Null",
+      "description": "Identifies log messages related to unregistering a callback for null.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "unregister callback null",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cleanup_application_record",
+      "title": "Clean Up Application Record",
+      "description": "Identifies log messages related to cleaning up an application record.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cleanUpApplicationRecord",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cleanup_application_record_locked_restart_false",
+      "title": "Clean Up Application Record Locked (Restart False)",
+      "description": "Identifies log messages related to cleaning up an application record with restart set to false.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cleanUpApplicationRecordLocked restart false",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cleanup_application_record_locked_reset_pid",
+      "title": "Clean Up Application Record Locked (Reset PID)",
+      "description": "Identifies log messages related to cleaning up an application record with reset PID.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cleanUpApplicationRecordLocked reset pid",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "new_process_app",
+      "title": "New Process App",
+      "description": "Identifies log messages related to the creation of a new process app.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "new Process app=ProcessRecord",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "process_has_died",
+      "title": "Process Has Died",
+      "description": "Identifies log messages related to a process that has died.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Process has died",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "start_proc_for_service",
+      "title": "Start Process for Service",
+      "description": "Identifies log messages related to starting a process for a service.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Start proc for service",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cannot_be_cast_to_token",
+      "title": "Cannot Be Cast to Token",
+      "description": "Identifies log messages related to casting errors to a Token.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cannot be cast to Token",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "must_execute_in_ui",
+      "title": "Must Execute in UI",
+      "description": "Identifies log messages related to execution requirements in the UI.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Must execute in UI",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "bad_activity_token",
+      "title": "Bad Activity Token",
+      "description": "Identifies log messages related to bad activity tokens.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Bad activity token",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unable_to_start_service_intent_not_found",
+      "title": "Unable to Start Service Intent (Not Found)",
+      "description": "Identifies log messages related to unable to start service intent not found.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Unable to start service Intent not found",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_app_window_token_going_to_hide",
+      "title": "Skipping App Window Token (Going to Hide)",
+      "description": "Identifies log messages related to skipping app window token going to hide.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping AppWindowToken going to hide",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_with_excluded_false_tr_intent_intent",
+      "title": "Skipping with Excluded False (tr.intent:Intent)",
+      "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping withExcluded false tr.intent:Intent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_with_excluded_false_tr_intent_intent_typ",
+      "title": "Skipping with Excluded False (tr.intent:Intent with Type)",
+      "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with type.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping withExcluded false tr.intent:Intent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_with_excluded_false_tr_intent_intent_flg",
+      "title": "Skipping with Excluded False (tr.intent:Intent with Flag)",
+      "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping withExcluded false tr.intent:Intent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_with_excluded_false_tr_intent_intent_flg_cmp",
+      "title": "Skipping with Excluded False (tr.intent:Intent with Flag and Component)",
+      "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag and component.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping withExcluded false tr.intent:Intent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "skipping_with_excluded_false_tr_intent_intent_flg_cmp_bnds",
+      "title": "Skipping with Excluded False (tr.intent:Intent with Flag, Component, and Bounds)",
+      "description": "Identifies log messages related to skipping with excluded false tr.intent:Intent with flag, component, and bounds.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Skipping withExcluded false tr.intent:Intent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/parser.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const timestampRegex = /\[(\w{3}) (\w{3}) (\d{1,2}) (\d{2}:\d{2}:\d{2}) (\d{4})\]/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const [_, dayOfWeek, month, day, time, year] = match;
+    const timestampString = `${dayOfWeek} ${month} ${day} ${time} ${year}`;
+    const date = moment.utc(timestampString, 'ddd MMM DD HH:mm:ss YYYY');
+    return date.valueOf();
+  }
+  throw new Error('Invalid log line format');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newDate = moment.utc(timestamp);
+  const newTimestampString = newDate.format('ddd MMM DD HH:mm:ss YYYY');
+  return logLine.replace(timestampRegex, `[${newTimestampString}]`);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Apache/queries.json

@@ -0,0 +1,118 @@
+{
+  "queries": [
+    {
+      "id": "jk2_init_found_child",
+      "title": "JK2 Init Found Child",
+      "description": "Returns logs where 'jk2_init() Found child' is found in the scoreboard slot.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "jk2_init() Found child scoreboard slot",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "worker_env_init_ok",
+      "title": "Worker Environment Init OK",
+      "description": "Returns logs where 'workerEnv.init() ok' is mentioned.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "workerEnv.init() ok",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mod_jk_child_error_state",
+      "title": "Mod JK Child WorkerEnv Error State",
+      "description": "Returns logs where 'mod_jk child workerEnv in error state' is found.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mod_jk child workerEnv error state",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "directory_index_forbidden",
+      "title": "Directory Index Forbidden",
+      "description": "Returns logs where 'Directory index forbidden by rule' is mentioned.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Directory index forbidden by rule",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "jk2_init_cant_find_child",
+      "title": "JK2 Init Can't Find Child",
+      "description": "Returns logs where 'jk2_init() Can't find child' is found in the scoreboard.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "jk2_init() Can't find child scoreboard",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mod_jk_child_init",
+      "title": "Mod JK Child Init",
+      "description": "Returns logs where 'mod_jk child init' is mentioned.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mod_jk child init",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/parser.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const timestampRegex = /\d{4}-\d{2}-\d{2}-\d{2}\.\d{2}\.\d{2}\.\d{6}/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const timestamp = match[0];
+    const momentObj = moment.utc(timestamp, 'YYYY-MM-DD-HH.mm.ss.SSSSSS');
+    return momentObj.valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const momentObj = moment.utc(timestamp);
+  const formattedTimestamp = momentObj.format('YYYY-MM-DD-HH.mm.ss.SSSSSS');
+  return logLine.replace(timestampRegex, formattedTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/BGL/queries.json

@@ -0,0 +1,821 @@
+{
+  "queries": [
+    {
+      "id": "e1",
+      "title": "DDR Errors Detected and Corrected with Rank, Symbol, and Seconds",
+      "description": "Returns log messages indicating DDR errors that were detected and corrected, including rank, symbol, and seconds information.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ddr error(s) detected AND corrected AND rank AND symbol AND seconds",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e2",
+      "title": "DDR Errors Detected and Corrected with Rank, Symbol, and Bit",
+      "description": "Returns log messages indicating DDR errors that were detected and corrected, including rank, symbol, and bit information.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ddr errors(s) detected AND corrected AND rank AND symbol AND bit",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e5",
+      "title": "L3 EDRAM Errors Detected and Corrected",
+      "description": "Returns log messages indicating L3 EDRAM errors that were detected and corrected.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "L3 EDRAM error(s) detected AND corrected",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e6",
+      "title": "L3 EDRAM Errors Detected and Corrected with Seconds",
+      "description": "Returns log messages indicating L3 EDRAM errors that were detected and corrected, including seconds information.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "L3 EDRAM error(s) detected AND corrected AND seconds",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e17",
+      "title": "Cannot Get Assembly Information for Node Card",
+      "description": "Returns log messages indicating an inability to get assembly information for a node card.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Can not get assembly information for node card",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e21",
+      "title": "Error Creating Node Map with Bad File Descriptor",
+      "description": "Returns log messages indicating an error in creating a node map due to a bad file descriptor.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error creating node map AND Bad file descriptor",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e22",
+      "title": "Error Creating Node Map with Block Device Required",
+      "description": "Returns log messages indicating an error in creating a node map due to a block device being required.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error creating node map AND Block device required",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e23",
+      "title": "Error Creating Node Map with No Child Processes",
+      "description": "Returns log messages indicating an error in creating a node map due to no child processes.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error creating node map AND No child processes",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e24",
+      "title": "Error Creating Node Map with Permission Denied",
+      "description": "Returns log messages indicating an error in creating a node map due to permission being denied.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error creating node map AND Permission denied",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e25",
+      "title": "Error Loading with Invalid or Missing Program Image and Exec Format Error",
+      "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and an exec format error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error loading AND invalid or missing program image AND Exec format error",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e26",
+      "title": "Error Loading with Invalid or Missing Program Image and Permission Denied",
+      "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and permission being denied.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error loading AND invalid or missing program image AND Permission denied",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e27",
+      "title": "Error Loading with Program Image Too Big",
+      "description": "Returns log messages indicating an error in loading due to the program image being too big.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error loading AND program image too big",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e28",
+      "title": "Error Loading with Invalid or Missing Program Image and No Such File or Directory",
+      "description": "Returns log messages indicating an error in loading due to an invalid or missing program image and no such file or directory.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error loading AND invalid or missing program image AND No such file or directory",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e29",
+      "title": "Error Reading Message Prefix with Link Severed",
+      "description": "Returns log messages indicating an error in reading the message prefix due to the link being severed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error reading message prefix AND Link has been severed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e30",
+      "title": "Error Reading Message Prefix with Connection Reset by Peer",
+      "description": "Returns log messages indicating an error in reading the message prefix due to the connection being reset by the peer.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error reading message prefix AND Connection reset by peer",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e31",
+      "title": "Error Reading Message Prefix with Connection Timed Out",
+      "description": "Returns log messages indicating an error in reading the message prefix due to the connection timing out.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error reading message prefix AND Connection timed out",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e32",
+      "title": "Error Reading Message Prefix with Link Severed",
+      "description": "Returns log messages indicating an error in reading the message prefix due to the link being severed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error reading message prefix AND Link has been severed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e33",
+      "title": "Failed to Read Message Prefix with Control Stream",
+      "description": "Returns log messages indicating a failure to read the message prefix due to the control stream.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "failed to read message prefix AND control stream",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e34",
+      "title": "Generated Core Files for Program",
+      "description": "Returns log messages indicating that core files were generated for a program.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "generated core files for program",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e40",
+      "title": "Poll Control Descriptors with Debugger Died",
+      "description": "Returns log messages indicating an issue with poll control descriptors due to the debugger dying.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pollControlDescriptors AND Detected the debugger died",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e44",
+      "title": "Critical Input Interrupt with Warning and Torus Y+ Wire",
+      "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Y+ wire.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "critical input interrupt AND warning AND torus y+ wire",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e45",
+      "title": "Critical Input Interrupt with Warning and Torus Z- Wire",
+      "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Z- wire.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "critical input interrupt AND warning AND torus z- wire",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e46",
+      "title": "Critical Input Interrupt with Warning, Torus Z+ Wire, and Suppressing Further Interrupts",
+      "description": "Returns log messages indicating a critical input interrupt with a warning related to the torus Z+ wire and suppressing further interrupts.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "critical input interrupt AND warning AND torus z+ wire AND suppressing further interrupts",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e47",
+      "title": "Critical Input Interrupt with Warning, Tree C1 Wire, and Suppressing Further Interrupts",
+      "description": "Returns log messages indicating a critical input interrupt with a warning related to the tree C1 wire and suppressing further interrupts.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "critical input interrupt AND warning AND tree C1 wire AND suppressing further interrupts",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e51",
+      "title": "Data Cache Search Parity Error Detected and Attempting to Correct",
+      "description": "Returns log messages indicating a data cache search parity error that was detected and an attempt to correct it.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "data cache search parity error detected AND attempting to correct",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e77",
+      "title": "Instruction Cache Parity Error Corrected",
+      "description": "Returns log messages indicating an instruction cache parity error that was corrected.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "instruction cache parity error corrected",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e80",
+      "title": "Lustre Mount Failed",
+      "description": "Returns log messages indicating a failed Lustre mount.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Lustre mount FAILED",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e81",
+      "title": "Lustre Mount Failed",
+      "description": "Returns log messages indicating a failed Lustre mount.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Lustre mount FAILED",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e82",
+      "title": "Machine Check DCR Read Timeout",
+      "description": "Returns log messages indicating a machine check DCR read timeout.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "MACHINE CHECK DCR read timeout",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e89",
+      "title": "NFS Mount Failed",
+      "description": "Returns log messages indicating a failed NFS mount.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "NFS Mount failed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e90",
+      "title": "Node Card Not Fully Functional",
+      "description": "Returns log messages indicating that a node card is not fully functional.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Node card is not fully functional",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e94",
+      "title": "NodeCard Not Fully Functional",
+      "description": "Returns log messages indicating that a NodeCard is not fully functional.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "NodeCard is not fully functional",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e95",
+      "title": "PrepareForService Shutting Down Node Card",
+      "description": "Returns log messages indicating that PrepareForService is shutting down a node card.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PrepareForService shutting down Node card",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e96",
+      "title": "PrepareForService Shutting Down NodeCard",
+      "description": "Returns log messages indicating that PrepareForService is shutting down a NodeCard.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PrepareForService shutting down NodeCard",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e107",
+      "title": "RTS Internal Error",
+      "description": "Returns log messages indicating an RTS internal error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rts internal error",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e108",
+      "title": "RTS Panic and Stopping Execution",
+      "description": "Returns log messages indicating an RTS panic and stopping execution.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rts panic AND stopping execution",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e109",
+      "title": "RTS Tree/Torus Link Training Failed",
+      "description": "Returns log messages indicating an RTS tree/torus link training failure.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rts tree/torus link training failed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e111",
+      "title": "RTS Kernel Terminated for Reason",
+      "description": "Returns log messages indicating that the RTS kernel was terminated for a reason.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rts kernel terminated for reason",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e113",
+      "title": "Shutdown Complete",
+      "description": "Returns log messages indicating that a shutdown is complete.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "shutdown complete",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e117",
+      "title": "Suppressing Further Interrupts of Same Type",
+      "description": "Returns log messages indicating that further interrupts of the same type are being suppressed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "suppressing further interrupts of same type",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e118",
+      "title": "Total DDR Errors Detected and Corrected",
+      "description": "Returns log messages indicating the total number of DDR errors that were detected and corrected.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "total of ddr error(s) detected AND corrected",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e119",
+      "title": "Total DDR Errors Detected and Corrected with Seconds",
+      "description": "Returns log messages indicating the total number of DDR errors that were detected and corrected, including seconds information.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "total of ddr error(s) detected AND corrected AND seconds",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "e120",
+      "title": "Wait State Enable",
+      "description": "Returns log messages indicating that the wait state is enabled.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "wait state enable",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/parser.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const timestampRegex = /(\d{6}) (\d{6}) (\d{1,3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const [_, datePart, timePart, millisPart] = match;
+    const formattedDate = `20${datePart} ${timePart}.${millisPart.padStart(3, '0')}`;
+    const momentDate = moment.utc(formattedDate, 'YYYYMMDD HHmmss.SSS');
+    return momentDate.valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const momentDate = moment.utc(timestamp);
+    const newTimestamp = momentDate.format('YYMMDD HHmmss SSS');
+    return logLine.replace(timestampRegex, newTimestamp);
+  }
+  throw new Error('Timestamp not found in log line');
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HDFS/queries.json

@@ -0,0 +1,232 @@
+{
+  "queries": [
+    {
+      "id": "starting_thread_transfer_block",
+      "title": "Starting thread to transfer block",
+      "description": "Matches log messages that indicate the start of a thread to transfer a block.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting thread to transfer block",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "exception_serving_block",
+      "title": "Got exception while serving block",
+      "description": "Matches log messages that indicate an exception occurred while serving a block.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Got exception while serving",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "ask_delete_block",
+      "title": "Ask to delete block",
+      "description": "Matches log messages that indicate a request to delete a block.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ask to delete blk",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "ask_replicate_block",
+      "title": "Ask to replicate block",
+      "description": "Matches log messages that indicate a request to replicate a block.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ask to replicate blk",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "blockmap_updated",
+      "title": "BlockMap updated",
+      "description": "Matches log messages that indicate the blockMap has been updated.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "blockMap updated",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "allocate_block",
+      "title": "Allocate block",
+      "description": "Matches log messages that indicate a block allocation.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "allocateBlock",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "block_added_invalidset",
+      "title": "Block is added to invalidSet",
+      "description": "Matches log messages that indicate a block has been added to the invalid set.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "is added to invalidSet",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "deleting_block",
+      "title": "Deleting block",
+      "description": "Matches log messages that indicate a block is being deleted.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Deleting block",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "packetresponder_terminating",
+      "title": "PacketResponder terminating",
+      "description": "Matches log messages that indicate a PacketResponder is terminating.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PacketResponder terminating",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "received_block",
+      "title": "Received block",
+      "description": "Matches log messages that indicate a block has been received.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Received block",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "receiving_block",
+      "title": "Receiving block",
+      "description": "Matches log messages that indicate a block is being received.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Receiving block",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "verification_succeeded_block",
+      "title": "Verification succeeded for block",
+      "description": "Matches log messages that indicate a block verification succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Verification succeeded for blk",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/parser.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const TIMESTAMP_REGEX = /\b\d{10}\b/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    return parseInt(match[0], 10) * 1000;
+  }
+  throw new Error('Timestamp not found');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = Math.floor(timestamp / 1000).toString();
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HPC/queries.json

@@ -0,0 +1,330 @@
+{
+  "queries": [
+    {
+      "id": "startup_boot_command",
+      "title": "Startup Message (boot command)",
+      "description": "Returns log messages that indicate a boot command was issued.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "boot command",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_closing",
+      "title": "Shutdown Message (closing)",
+      "description": "Returns log messages that indicate a shutdown process with the term 'closing'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "closing"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_starting",
+      "title": "Startup Message (starting)",
+      "description": "Returns log messages that indicate a startup process with the term 'starting'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "starting"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_halt_command",
+      "title": "Shutdown Message (halt command)",
+      "description": "Returns log messages that indicate a halt command was issued.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "halt command",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_boot_halt_asserted",
+      "title": "Error Message (boot command Error: HALT asserted)",
+      "description": "Returns log messages that indicate a boot command error with HALT asserted.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "boot.*Error: HALT asserted"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "critical_message",
+      "title": "Critical Message (critical)",
+      "description": "Returns log messages that contain the term 'critical'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "critical"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_command_aborted",
+      "title": "Error Message (Command has been aborted)",
+      "description": "Returns log messages that indicate a command has been aborted.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "Command has been aborted"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_failed_subcommands",
+      "title": "Error Message (Failed subcommands)",
+      "description": "Returns log messages that indicate failed subcommands.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "Failed subcommands"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "link_error",
+      "title": "Link Error Message (Link error)",
+      "description": "Returns log messages that indicate a link error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "Link error"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "link_error_broadcast_tree",
+      "title": "Link Error Message (Link error on broadcast tree Interconnect)",
+      "description": "Returns log messages that indicate a link error on the broadcast tree interconnect.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Link error on broadcast tree Interconnect.*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_message",
+      "title": "Warning Message (warning)",
+      "description": "Returns log messages that contain the term 'warning'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "warning"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "temperature_warning",
+      "title": "Temperature Warning Message (Temperature exceeds warning threshold)",
+      "description": "Returns log messages that indicate temperature exceeds the warning threshold.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Temperature.*exceeds warning threshold"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_server_filesystem_panic",
+      "title": "Error Message (ServerFileSystem domain panic)",
+      "description": "Returns log messages that indicate a ServerFileSystem domain panic.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "ServerFileSystem.*domain panic"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_server_filesystem_full",
+      "title": "Error Message (ServerFileSystem domain is full)",
+      "description": "Returns log messages that indicate a ServerFileSystem domain is full.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "ServerFileSystem.*domain.*is full"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_not_responding",
+      "title": "Error Message (not responding)",
+      "description": "Returns log messages that indicate a system or service is not responding.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "not responding"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_not_responding_hyphen",
+      "title": "Error Message (not-responding)",
+      "description": "Returns log messages that indicate a system or service is not-responding.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "not-responding"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "power_control_problem",
+      "title": "Power/Control Problem Message (power/control problem)",
+      "description": "Returns log messages that indicate a power or control problem.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": "power/control problem"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "psu_failure",
+      "title": "PSU Failure Message (psu failure)",
+      "description": "Returns log messages that indicate a PSU failure.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "psu failure.*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_risboot_timeout",
+      "title": "Error Message (risBoot command Error: Timed out)",
+      "description": "Returns log messages that indicate a risBoot command error due to timeout.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "risBoot.*Error: Timed out"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "network_connection_failure",
+      "title": "Network Connection Failure Message (detected a failed network connection)",
+      "description": "Returns log messages that indicate a failed network connection.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "detected a failed network connection"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/parser.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const timestamp = match[1];
+    return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss,SSS').valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss,SSS');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Hadoop/queries.json

@@ -0,0 +1,308 @@
+{
+  "queries": [
+    {
+      "id": "failures_on_node",
+      "title": "Failures on node MININT",
+      "description": "Finds log messages indicating failures on node MININT.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "failures on node MININT",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "added_attempt_failed_maps",
+      "title": "Added attempt to list of failed maps",
+      "description": "Finds log messages indicating an attempt was added to the list of failed maps.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Added attempt to list of failed maps",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "taskattempt_fail_container_cleanup_to_fail_task_cleanup",
+      "title": "TaskAttempt Transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP",
+      "description": "Finds log messages indicating a TaskAttempt transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "TaskAttempt Transitioned from FAIL_CONTAINER_CLEANUP to FAIL_TASK_CLEANUP",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "taskattempt_fail_task_cleanup_to_failed",
+      "title": "TaskAttempt Transitioned from FAIL_TASK_CLEANUP to FAILED",
+      "description": "Finds log messages indicating a TaskAttempt transitioned from FAIL_TASK_CLEANUP to FAILED.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "TaskAttempt Transitioned from FAIL_TASK_CLEANUP to FAILED",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "taskattempt_running_to_fail_container_cleanup",
+      "title": "TaskAttempt Transitioned from RUNNING to FAIL_CONTAINER_CLEANUP",
+      "description": "Finds log messages indicating a TaskAttempt transitioned from RUNNING to FAIL_CONTAINER_CLEANUP.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "TaskAttempt Transitioned from RUNNING to FAIL_CONTAINER_CLEANUP",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "datastreamer_exception",
+      "title": "DataStreamer Exception",
+      "description": "Finds log messages indicating a DataStreamer Exception.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "DataStreamer Exception",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "dfsoutputstream_responseprocessor_exception",
+      "title": "DFSOutputStream ResponseProcessor exception for block",
+      "description": "Finds log messages indicating a DFSOutputStream ResponseProcessor exception for a block.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "DFSOutputStream ResponseProcessor exception for block",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "diagnostics_report_no_route_to_host",
+      "title": "Diagnostics report from attempt: No Route to Host",
+      "description": "Finds log messages indicating a diagnostics report from an attempt with a No Route to Host error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Diagnostics report from attempt Error java.net.NoRouteToHostException No Route to Host",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_contacting_rm",
+      "title": "ERROR IN CONTACTING RM",
+      "description": "Finds log messages indicating an error in contacting the Resource Manager (RM).",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ERROR IN CONTACTING RM",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_recovery_bad_datanode",
+      "title": "Error Recovery for block in pipeline: bad datanode",
+      "description": "Finds log messages indicating an error recovery for a block in the pipeline due to a bad datanode.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error Recovery for block in pipeline bad datanode",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_writing_history_event",
+      "title": "Error writing History Event",
+      "description": "Finds log messages indicating an error writing a History Event.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error writing History Event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "failed_to_renew_lease",
+      "title": "Failed to renew lease for DFSClient_NONMAPREDUCE",
+      "description": "Finds log messages indicating a failure to renew a lease for DFSClient_NONMAPREDUCE.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Failed to renew lease for DFSClient_NONMAPREDUCE",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "killing_attempt",
+      "title": "KILLING attempt",
+      "description": "Finds log messages indicating an attempt is being killed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "KILLING attempt",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "task_cleanup_failed",
+      "title": "Task cleanup failed for attempt",
+      "description": "Finds log messages indicating a task cleanup failed for an attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Task cleanup failed for attempt",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "task_exited_no_route_to_host",
+      "title": "Task attempt exited: No Route to Host",
+      "description": "Finds log messages indicating a task attempt exited due to a No Route to Host error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Task attempt exited java.net.NoRouteToHostException No Route to Host",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "thread_threw_exception",
+      "title": "Thread threw an Exception",
+      "description": "Finds log messages indicating a thread threw an exception.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Thread Thread threw an Exception",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/invalid_parser.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /^(\d{4})(\d{1,2})(\d{1,2})-(\d{1,2}):(\d{1,2}):(\d{1,2}):(\d{1,3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const [_, year, month, day, hour, minute, second, millisecond] = match;
+    const timestampStr = `${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`;
+    const timestamp = moment.utc(timestampStr, 'YYYY-MM-DDTHH:mm:ss.SSSZ').valueOf();
+    return timestamp;
+  }
+  throw new Error('Timestamp not found');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestampStr = moment.utc(timestamp).format('YYYYMMDD-HH:mm:ss:SSS');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestampStr);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/parser.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /(\d{4}\d{2}\d{2}-\d{1,2}:\d{1,2}:\d{1,2}:\d{1,3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    return moment.utc(match[0], 'YYYYMMDD-HH:mm:ss:SSS').valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = moment.utc(timestamp).format('YYYYMMDD-HH:mm:ss:SSS');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/HealthApp/queries.json

@@ -0,0 +1,226 @@
+{
+  "queries": [
+    {
+      "id": "init_environment",
+      "title": "Init Environment",
+      "description": "Returns log messages related to the initialization of the environment.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "initEnviroument",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "init_data_privacy",
+      "title": "Init Data Privacy",
+      "description": "Returns log messages indicating that data privacy has been initialized and is set to true.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "initDataPrivacy the dataPrivacy is true",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "init_user_privacy",
+      "title": "Init User Privacy",
+      "description": "Returns log messages indicating that user privacy has been initialized and is set to true.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "initUserPrivacy the userPrivacy is true",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "start_timer_auto_sync",
+      "title": "Start Timer AutoSync",
+      "description": "Returns log messages indicating that the auto-sync timer has started.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "startTimer start autoSync",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "bulk_save_fail_error_code",
+      "title": "Bulk Save Fail Error Code",
+      "description": "Returns log messages indicating a failure in bulk saving health data with an error code.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "insertHiHealthData\\(\\) bulkSaveDetailHiHealthData fail errorCode = .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "save_one_detail_data_fail",
+      "title": "Save One Detail Data Fail",
+      "description": "Returns log messages indicating a failure in saving one detail health data with specific data and type.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "saveHealthDetailData\\(\\) saveOneDetailData fail hiHealthData = .*,type = .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "upload_statics_to_db_failed",
+      "title": "Upload Statics to DB Failed",
+      "description": "Returns log messages indicating a failure in uploading statics to the database with a message set to true.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "uploadStaticsToDB failed message=true",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "send_sync_failed_broadcast",
+      "title": "Send Sync Failed Broadcast",
+      "description": "Returns log messages indicating that a sync failed broadcast was sent.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "sendSyncFailedBroadcast",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "screen_off",
+      "title": "Screen Off",
+      "description": "Returns log messages indicating that the screen off action was received.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "onReceive action: android.intent.action.SCREEN_OFF",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "screen_on",
+      "title": "Screen On",
+      "description": "Returns log messages indicating that the screen on action was received.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "onReceive action: android.intent.action.SCREEN_ON",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "process_handle_screen_on",
+      "title": "Process Handle Screen On",
+      "description": "Returns log messages indicating that the process handled a screen on broadcast action.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "processHandleBroadcastAction action:android.intent.action.SCREEN_ON",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "screen_status_unknown",
+      "title": "Screen Status Unknown",
+      "description": "Returns log messages indicating that the screen status is unknown but assumed to be on.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "screen status unknown,think screen on",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/parser.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const [_, month, day, hour, minute, second] = match;
+    const dateString = `${month} ${day} ${hour}:${minute}:${second} UTC`;
+    const date = moment.utc(dateString, 'MMM DD HH:mm:ss');
+    return date.valueOf();
+  }
+  throw new Error('Timestamp not found');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newDate = moment.utc(timestamp).format('MMM DD HH:mm:ss');
+  return logLine.replace(TIMESTAMP_REGEX, newDate);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Linux/queries.json

@@ -0,0 +1,369 @@
+{
+  "queries": [
+    {
+      "id": "cupsd_startup",
+      "title": "CUPS Daemon Startup",
+      "description": "Extracts log messages indicating that the CUPS daemon startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cupsd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cupsd_shutdown",
+      "title": "CUPS Daemon Shutdown",
+      "description": "Extracts log messages indicating that the CUPS daemon shutdown succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "cupsd shutdown succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_client_address",
+      "title": "Warning: Client Address Issue",
+      "description": "Extracts log messages indicating a warning related to client address connection reset by peer.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "warning: can't get client address: Connection reset by peer",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "auth_failed_permission_denied",
+      "title": "Error: Authentication Failed (Permission Denied)",
+      "description": "Extracts log messages indicating authentication failure due to permission denied in replay cache code.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Authentication failed from .* \\(.*\\): Permission denied in replay cache code"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "auth_failed_connection_abort",
+      "title": "Error: Authentication Failed (Connection Abort)",
+      "description": "Extracts log messages indicating authentication failure due to software caused connection abort.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Authentication failed from .* \\(.*\\): Software caused connection abort"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "couldnt_authenticate_user",
+      "title": "Error: Couldn't Authenticate User",
+      "description": "Extracts log messages indicating failure to authenticate a user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Couldn't authenticate user",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "failure_registering_capabilities",
+      "title": "Error: Failure Registering Capabilities",
+      "description": "Extracts log messages indicating failure in registering capabilities with the kernel.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Failure registering capabilities with the kernel",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "kerberos_auth_failed",
+      "title": "Error: Kerberos Authentication Failed",
+      "description": "Extracts log messages indicating Kerberos authentication failure.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Kerberos authentication failed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "anonymous_ftp_login",
+      "title": "Error: Anonymous FTP Login",
+      "description": "Extracts log messages indicating an anonymous FTP login attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "ANONYMOUS FTP LOGIN FROM .*,  \\(anonymous\\)"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "alert_exited_abnormally",
+      "title": "Error: Alert Exited Abnormally",
+      "description": "Extracts log messages indicating an alert that exited abnormally with code 1.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ALERT exited abnormally with [1]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "audit_netlink_socket",
+      "title": "Audit: Initializing Netlink Socket",
+      "description": "Extracts log messages indicating that the audit system is initializing a netlink socket.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "audit: initializing netlink socket (disabled)",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "audit_initialized",
+      "title": "Audit: Initialized",
+      "description": "Extracts log messages indicating that the audit system has been initialized.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "audit\\(.*\\..*\\:.*\\): initialized"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "klogd_startup",
+      "title": "Klogd Startup",
+      "description": "Extracts log messages indicating that the klogd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "klogd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "syslogd_startup",
+      "title": "Syslogd Startup",
+      "description": "Extracts log messages indicating that the syslogd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "syslogd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "rpc_idmapd_startup",
+      "title": "RPC Idmapd Startup",
+      "description": "Extracts log messages indicating that the rpc.idmapd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rpc.idmapd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "rpc_statd_startup",
+      "title": "RPC Statd Startup",
+      "description": "Extracts log messages indicating that the rpc.statd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rpc.statd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "sdpd_startup",
+      "title": "SDPD Startup",
+      "description": "Extracts log messages indicating that the sdpd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "sdpd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "hcid_startup",
+      "title": "HCID Startup",
+      "description": "Extracts log messages indicating that the hcid startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "hcid startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "imps2_autodetected",
+      "title": "IMPS2 Auto-detected",
+      "description": "Extracts log messages indicating that the IMPS2 auto-detected an IntelliMouse PS/2.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "imps2: Auto-detected intellimouse PS/.*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "crypto_api_init",
+      "title": "Initializing Cryptographic API",
+      "description": "Extracts log messages indicating that the cryptographic API is being initialized.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Initializing Cryptographic API",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/parser.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const [_, month, day, hour, minute, second] = match;
+    const year = new Date().getFullYear(); // Assume current year
+    const dateStr = `${year} ${month} ${day} ${hour}:${minute}:${second}`;
+    const date = moment.utc(dateStr, 'YYYY MMM DD HH:mm:ss');
+    return date.valueOf(); // Return epoch milliseconds
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const date = moment.utc(timestamp);
+  const newTimestamp = date.format('MMM DD HH:mm:ss');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Mac/queries.json

@@ -0,0 +1,457 @@
+{
+  "queries": [
+    {
+      "id": "startup_delay_hci_reset",
+      "title": "Startup Delay HCI Reset",
+      "description": "Returns log messages related to BroadcomBluetoothHostController SetupController delaying HCI reset.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "BroadcomBluetoothHostController SetupController Delay HCI Reset",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_completed_matched_device",
+      "title": "Startup Completed Matched Device",
+      "description": "Returns log messages related to BroadcomBluetoothHostControllerUSBTransport start completed and matched on device.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "BroadcomBluetoothHostControllerUSBTransport start Completed matched on Device",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_calling_register_service",
+      "title": "Startup Calling Register Service",
+      "description": "Returns log messages related to IOBluetoothFamily calling registerService during ProcessBluetoothTransportShowsUpActionWL.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "IOBluetoothFamily ProcessBluetoothTransportShowsUpActionWL calling registerService",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_connected_transport_successfully",
+      "title": "Startup Connected to Transport Successfully",
+      "description": "Returns log messages related to IOBluetoothFamily successfully connecting to the transport during ProcessBluetoothTransportShowsUpActionWL.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "IOBluetoothFamily ProcessBluetoothTransportShowsUpActionWL Connected to the transport successfully",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_completed_result_true",
+      "title": "Startup Completed Result TRUE",
+      "description": "Returns log messages related to IOBluetoothHostControllerUSBTransport start completed with result TRUE.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "IOBluetoothHostControllerUSBTransport start completed result TRUE",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "init_host_controller_published",
+      "title": "INIT Host Controller Published",
+      "description": "Returns log messages related to the initialization of the host controller being published.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "INIT Host controller is published",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_import_bailout",
+      "title": "Error Import Bailout",
+      "description": "Returns log messages related to ImportBailout error asking to exit for Diskarb.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "ImportBailout.Error Asked to exit for Diskarb",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_sdef_argument",
+      "title": "Warning SDEF Argument",
+      "description": "Returns log messages related to sdef warning for argument of command in Microsoft Word Suite.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "sdef warning for argument of command can continue previous list in suite Microsoft Word Suite is not a valid type name",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_dns_config_service",
+      "title": "Warning DNS Config Service",
+      "description": "Returns log messages related to DNS config service posix failing to read DnsConfig.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "WARNING dns_config_service_posix.cc Failed to read DnsConfig",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_bluetooth_hid_device_controller",
+      "title": "Error Bluetooth HID Device Controller",
+      "description": "Returns log messages related to BluetoothHIDDeviceController error of not finding the disconnected object.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "BluetoothHIDDeviceController ERROR Could not find the disconnected object",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_carddavplugin_offline",
+      "title": "Error CardDAVPlugin Offline",
+      "description": "Returns log messages related to CardDAVPlugin error indicating the internet connection appears to be offline.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "CardDAVPlugin-ERROR getPrincipalInfo Error Domain NSURLErrorDomain Code The Internet connection appears to be offline",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_carddavplugin_timeout",
+      "title": "Error CardDAVPlugin Timeout",
+      "description": "Returns log messages related to CardDAVPlugin error indicating the request timed out.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "CardDAVPlugin-ERROR getPrincipalInfo Error Domain NSURLErrorDomain Code The request timed out",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_apple_device_management_hid_event_service",
+      "title": "Error Apple Device Management HID Event Service",
+      "description": "Returns log messages related to AppleDeviceManagementHIDEventService error of not making a string from connection notification key.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "HID ATC Error AppleDeviceManagementHIDEventService start Could not make a string from out connection notification key",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_timed_out",
+      "title": "Error Timed Out",
+      "description": "Returns log messages related to errors indicating a timeout.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": ".*ERROR.*timed out after.*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_media_validator_unrecognized_codec",
+      "title": "Error Media Validator Unrecognized Codec",
+      "description": "Returns log messages related to MediaValidator unrecognized codec failing codec specific check.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "MediaValidator Unrecognized codec Failed codec specific check",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_media_validator_lookup_codec_support",
+      "title": "Error Media Validator Lookup Codec Support",
+      "description": "Returns log messages related to MediaValidator mv_LookupCodecSupport unrecognized codec.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "MediaValidator mv_LookupCodecSupport Unrecognized codec",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_daemon_connection_invalidated",
+      "title": "Error Daemon Connection Invalidated",
+      "description": "Returns log messages related to daemon connection being invalidated.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Daemon connection invalidated",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_db_no_such_table",
+      "title": "Error DB No Such Table",
+      "description": "Returns log messages related to database error indicating no such table.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "DB Error no such table",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_core_drag_remove_receive_handler",
+      "title": "Error Core Drag Remove Receive Handler",
+      "description": "Returns log messages related to error in CoreDragRemoveReceiveHandler.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error in CoreDragRemoveReceiveHandler",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_core_drag_remove_tracking_handler",
+      "title": "Error Core Drag Remove Tracking Handler",
+      "description": "Returns log messages related to error in CoreDragRemoveTrackingHandler.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error in CoreDragRemoveTrackingHandler",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_iconservicesagent",
+      "title": "Error IconServicesAgent",
+      "description": "Returns log messages related to error returned from iconservicesagent.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Error returned from iconservicesagent",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_service_exited_abnormal_code",
+      "title": "Error Service Exited Abnormal Code",
+      "description": "Returns log messages related to service exiting with abnormal code.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Service exited with abnormal code",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_hibernate_page_list_setall",
+      "title": "Warning Hibernate Page List SetAll",
+      "description": "Returns log messages related to hibernate_page_list_setall skipping xpmapped pages.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "WARNING hibernate_page_list_setall skipped xpmapped pages",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_type1_font_data",
+      "title": "Warning Type1 Font Data",
+      "description": "Returns log messages related to Type1 font data not being in the correct format required by the Adobe Type Font Format specification.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "WARNING Type1 font data isn't in the correct format required by the Adobe Type Font Format specification",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/parser.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const timestampRegex = /(\w{3})\s+(\d{1,2})\s+(\d{2}:\d{2}:\d{2})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const [_, month, day, time] = match;
+    const dateString = `${month} ${day} ${time}`;
+    const date = moment.utc(dateString, 'MMM DD HH:mm:ss');
+    return date.valueOf();
+  }
+  throw new Error('Invalid log line format');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newDate = moment.utc(timestamp);
+  const newTimestamp = newDate.format('MMM DD HH:mm:ss');
+  return logLine.replace(timestampRegex, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenSSH/queries.json

@@ -0,0 +1,517 @@
+{
+  "queries": [
+    {
+      "id": "accepted_password",
+      "title": "Accepted password for SSH",
+      "description": "Logs indicating a successful SSH login using a password.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Accepted password for from port ssh2",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "connection_closed_preauth",
+      "title": "Connection closed by preauth",
+      "description": "Logs indicating a connection closed by the client before authentication.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Connection closed by [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "no_identification_string",
+      "title": "No identification string received",
+      "description": "Logs indicating that the server did not receive an identification string from the client.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Did not receive identification string from",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "too_many_auth_failures_admin",
+      "title": "Too many authentication failures for admin",
+      "description": "Logs indicating too many authentication failures for the admin user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Disconnecting: Too many authentication failures for admin [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "too_many_auth_failures_root",
+      "title": "Too many authentication failures for root",
+      "description": "Logs indicating too many authentication failures for the root user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Disconnecting: Too many authentication failures for root [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "auth_fail_jsch_exception",
+      "title": "Auth fail JSchException",
+      "description": "Logs indicating an authentication failure with a JSchException.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error: Received disconnect from com.jcraft.jsch.JSchException: Auth fail [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "no_more_auth_methods",
+      "title": "No more user authentication methods",
+      "description": "Logs indicating no more user authentication methods are available.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error: Received disconnect from No more user authentication methods available. [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "failed_none_invalid_user",
+      "title": "Failed none for invalid user",
+      "description": "Logs indicating a failed authentication attempt for an invalid user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Failed none for invalid user from port ssh2",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "failed_password",
+      "title": "Failed password",
+      "description": "Logs indicating a failed password authentication attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Failed password for from port ssh2",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "failed_password_invalid_user",
+      "title": "Failed password for invalid user",
+      "description": "Logs indicating a failed password authentication attempt for an invalid user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Failed password for invalid user from port ssh2",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "write_failed_connection_reset",
+      "title": "Write failed: Connection reset by peer",
+      "description": "Logs indicating a connection reset by peer error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "fatal: Write failed: Connection reset by peer [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "invalid_user_preauth",
+      "title": "Invalid user preauth",
+      "description": "Logs indicating an invalid user authentication request before authentication.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "input_userauth_request: invalid user [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "invalid_user",
+      "title": "Invalid user",
+      "description": "Logs indicating an invalid user authentication attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Invalid user from",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "message_repeated",
+      "title": "Message repeated",
+      "description": "Logs indicating repeated messages of failed password attempts for root.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "message repeated times: [ Failed password for root from port ]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_auth_failure",
+      "title": "PAM authentication failure",
+      "description": "Logs indicating a PAM authentication failure.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PAM more authentication failure; logname= uid= euid= tty=ssh ruser= rhost=",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_auth_failures",
+      "title": "PAM authentication failures",
+      "description": "Logs indicating multiple PAM authentication failures.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PAM more authentication failures; logname= uid= euid= tty=ssh ruser= rhost=",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_auth_failures_root",
+      "title": "PAM authentication failures for root",
+      "description": "Logs indicating multiple PAM authentication failures for the root user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PAM more authentication failures; logname= uid= euid= tty=ssh ruser= rhost= user=root",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_ignoring_max_retries",
+      "title": "PAM ignoring max retries",
+      "description": "Logs indicating that the PAM service is ignoring max retries.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "PAM service(sshd) ignoring max retries;",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_unix_auth_failure",
+      "title": "PAM Unix authentication failure",
+      "description": "Logs indicating a PAM Unix authentication failure.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pam_unix(sshd:auth): authentication failure; logname= uid= euid= tty=ssh ruser= rhost=",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_unix_auth_failure_user",
+      "title": "PAM Unix authentication failure for user",
+      "description": "Logs indicating a PAM Unix authentication failure for a specific user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pam_unix(sshd:auth): authentication failure; logname= uid= euid= tty=ssh ruser= rhost= user=",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_unix_check_pass",
+      "title": "PAM Unix check pass",
+      "description": "Logs indicating a PAM Unix check pass for an unknown user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pam_unix(sshd:auth): check pass; user unknown",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_unix_session_closed",
+      "title": "PAM Unix session closed",
+      "description": "Logs indicating a PAM Unix session closed for a user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pam_unix(sshd:session): session closed for user",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "pam_unix_session_opened",
+      "title": "PAM Unix session opened",
+      "description": "Logs indicating a PAM Unix session opened for a user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "pam_unix(sshd:session): session opened for user by (uid=)",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "received_disconnect_bye_bye",
+      "title": "Received disconnect: Bye Bye",
+      "description": "Logs indicating a received disconnect with the message 'Bye Bye'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Received disconnect from Bye Bye [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "received_disconnect_user_request",
+      "title": "Received disconnect: User request",
+      "description": "Logs indicating a received disconnect due to user request.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Received disconnect from Closed due to user request. [preauth]",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "received_disconnect_disconnected_by_user",
+      "title": "Received disconnect: Disconnected by user",
+      "description": "Logs indicating a received disconnect by the user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Received disconnect from disconnected by user",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "reverse_mapping_failed",
+      "title": "Reverse mapping failed",
+      "description": "Logs indicating a failed reverse mapping check, suggesting a possible break-in attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "reverse mapping checking getaddrinfo for failed - POSSIBLE BREAK-IN ATTEMPT!",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/parser.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const timestamp = match[1];
+    return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss.SSS').valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss.SSS');
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/OpenStack/queries.json

@@ -0,0 +1,207 @@
+{
+  "queries": [
+    {
+      "id": "terminate_instance",
+      "title": "E11: Terminating instance",
+      "description": "Returns logs where an instance is being terminated.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Terminating instance",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "vm_paused",
+      "title": "E20: VM Paused (Lifecycle Event)",
+      "description": "Returns logs where a VM has been paused.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "VM Paused Lifecycle Event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "vm_resumed",
+      "title": "E21: VM Resumed (Lifecycle Event)",
+      "description": "Returns logs where a VM has been resumed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "VM Resumed Lifecycle Event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "vm_started",
+      "title": "E22: VM Started (Lifecycle Event)",
+      "description": "Returns logs where a VM has been started.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "VM Started Lifecycle Event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "vm_stopped",
+      "title": "E23: VM Stopped (Lifecycle Event)",
+      "description": "Returns logs where a VM has been stopped.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "VM Stopped Lifecycle Event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "disk_limit_not_specified",
+      "title": "E6: disk limit not specified, defaulting to unlimited",
+      "description": "Returns logs where the disk limit was not specified and defaulted to unlimited.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "disk limit not specified defaulting to unlimited",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "sync_power_state_pending_task",
+      "title": "E7: During sync_power_state the instance has a pending task (spawning). Skip.",
+      "description": "Returns logs where an instance has a pending task during sync_power_state.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "During sync_power_state the instance has a pending task spawning Skip",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "vcpu_limit_not_specified",
+      "title": "E19: vcpu limit not specified, defaulting to unlimited",
+      "description": "Returns logs where the vcpu limit was not specified and defaulted to unlimited.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "vcpu limit not specified defaulting to unlimited",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "http_exception_no_instances",
+      "title": "E33: HTTP exception thrown: No instances found for any event",
+      "description": "Returns logs where an HTTP exception was thrown due to no instances found for any event.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "HTTP exception thrown No instances found for any event",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "instance_sync_host_mismatch",
+      "title": "E40: The instance sync for host '<*>' did not match. Re-created its InstanceList.",
+      "description": "Returns logs where the instance sync for a host did not match and its InstanceList was re-created.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "The instance sync for host .* did not match Re-created its InstanceList"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "sync_instance_power_states_mismatch",
+      "title": "E43: While synchronizing instance power states, found <*> instances in the database and <*> instances on the hypervisor.",
+      "description": "Returns logs where there was a mismatch in the number of instances found in the database and on the hypervisor during instance power state synchronization.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "While synchronizing instance power states found .* instances in the database and .* instances on the hypervisor"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/parser.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /\[(\d{1,2})\.(\d{1,2}) (\d{1,2}):(\d{2}):(\d{2})\]/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const [_, month, day, hour, minute, second] = match;
+    const dateString = `${new Date().getFullYear()}-${month}-${day} ${hour}:${minute}:${second}`;
+    const date = moment.utc(dateString, 'YYYY-MM-DD HH:mm:ss');
+    return date.valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newDate = moment.utc(timestamp);
+  const newTimestamp = `[${newDate.format('MM.DD HH:mm:ss')}]`;
+  return logLine.replace(TIMESTAMP_REGEX, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Proxifier/queries.json

@@ -0,0 +1,153 @@
+{
+  "queries": [
+    {
+      "id": "open_through_proxy_socks5",
+      "title": "Open through proxy SOCKS5",
+      "description": "Returns log messages where the connection is opened through a SOCKS5 proxy.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "open through proxy SOCKS5",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "open_through_proxy_https",
+      "title": "Open through proxy HTTPS",
+      "description": "Returns log messages where the connection is opened through an HTTPS proxy.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "open through proxy HTTPS",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_could_not_connect_to_proxy_resolve_error",
+      "title": "Error: Could not connect to proxy - resolve error",
+      "description": "Returns log messages where there is an error connecting to a proxy due to a resolution error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error Could not connect to proxy Could not resolve error",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_could_not_connect_to_proxy_connection_attempt_failed",
+      "title": "Error: Could not connect to proxy - connection attempt failed",
+      "description": "Returns log messages where there is an error connecting to a proxy due to a failed connection attempt.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error Could not connect to proxy connection attempt failed with error",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_could_not_connect_through_proxy_status_code",
+      "title": "Error: Could not connect through proxy - status code",
+      "description": "Returns log messages where there is an error connecting through a proxy due to a status code issue.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error Could not connect through proxy Proxy server cannot establish a connection with the target status code",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_could_not_connect_through_proxy_closed_connection",
+      "title": "Error: Could not connect through proxy - closed connection",
+      "description": "Returns log messages where there is an error connecting through a proxy due to the proxy closing the connection unexpectedly.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error Could not connect through proxy Proxy closed the connection unexpectedly",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_connection_request_canceled",
+      "title": "Error: Connection request canceled",
+      "description": "Returns log messages where a connection request was canceled before completion.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "error A connection request was canceled before the completion",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "close_bytes_sent_received_lifetime",
+      "title": "Close, bytes sent/received, lifetime",
+      "description": "Returns log messages where a connection is closed, showing bytes sent, bytes received, and the connection lifetime.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": ".* close, .* bytes.*sent, .* bytes.*received, lifetime .*"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/parser.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX = /^(\d{2}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const timestampString = match[1];
+    return moment.utc(timestampString, 'YY/MM/DD HH:mm:ss').valueOf();
+  }
+  throw new Error('Timestamp not found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const formattedTimestamp = moment.utc(timestamp).format('YY/MM/DD HH:mm:ss');
+  return logLine.replace(TIMESTAMP_REGEX, formattedTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Spark/queries.json

@@ -0,0 +1,280 @@
+{
+  "queries": [
+    {
+      "id": "memory_store_started",
+      "title": "MemoryStore Started",
+      "description": "Identifies logs where MemoryStore started with a specified capacity.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "MemoryStore started capacity",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "registered_signal_handlers",
+      "title": "Registered Signal Handlers",
+      "description": "Identifies logs where signal handlers for TERM, HUP, and INT were registered.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Registered signal handlers TERM HUP INT",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "remoting_started",
+      "title": "Remoting Started",
+      "description": "Identifies logs where remoting started and is listening on specified addresses.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Remoting started; listening on addresses :\\[akka\\.tcp://.*\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "slf4jlogger_started",
+      "title": "Slf4jLogger Started",
+      "description": "Identifies logs where Slf4jLogger started.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Slf4jLogger started",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "starting_executor",
+      "title": "Starting Executor",
+      "description": "Identifies logs where an executor is starting on a specified host.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting executor ID host",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "starting_remoting",
+      "title": "Starting Remoting",
+      "description": "Identifies logs where remoting is starting.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting remoting",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "successfully_registered_with_driver",
+      "title": "Successfully Registered with Driver",
+      "description": "Identifies logs where the system successfully registered with the driver.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Successfully registered driver",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "started_netty_block_transfer_service",
+      "title": "Started Netty Block Transfer Service",
+      "description": "Identifies logs where the Netty Block Transfer Service started on a specified port.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Successfully started service 'org\\.apache\\.spark\\.network\\.netty\\.NettyBlockTransferService' on port .*\\."
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "started_spark_executor_actor_system",
+      "title": "Started Spark Executor Actor System",
+      "description": "Identifies logs where the Spark Executor Actor System started on a specified port.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Successfully started service 'sparkExecutorActorSystem' on port .*\\."
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "partition_rdd_not_found",
+      "title": "Partition RDD Not Found",
+      "description": "Identifies logs where a partition RDD was not found and is being computed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Partition rdd not found computing",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mapred_job_id_deprecated",
+      "title": "Mapred Job ID Deprecated",
+      "description": "Identifies logs where the mapred.job.id is deprecated and suggests using mapreduce.job.id instead.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mapred.job.id deprecated mapreduce.job.id",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mapred_task_id_deprecated",
+      "title": "Mapred Task ID Deprecated",
+      "description": "Identifies logs where the mapred.task.id is deprecated and suggests using mapreduce.task.attempt.id instead.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mapred.task.id deprecated mapreduce.task.attempt.id",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mapred_task_is_map_deprecated",
+      "title": "Mapred Task Is Map Deprecated",
+      "description": "Identifies logs where the mapred.task.is.map is deprecated and suggests using mapreduce.task.ismap instead.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mapred.task.is.map deprecated mapreduce.task.ismap",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mapred_task_partition_deprecated",
+      "title": "Mapred Task Partition Deprecated",
+      "description": "Identifies logs where the mapred.task.partition is deprecated and suggests using mapreduce.task.partition instead.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mapred.task.partition deprecated mapreduce.task.partition",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "mapred_tip_id_deprecated",
+      "title": "Mapred Tip ID Deprecated",
+      "description": "Identifies logs where the mapred.tip.id is deprecated and suggests using mapreduce.task.id instead.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "mapred.tip.id deprecated mapreduce.task.id",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/parser.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const TIMESTAMP_REGEX = /^- (\d+) (\d{4}\.\d{2}\.\d{2} .*)/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const timestamp = parseInt(match[1], 10);
+    return timestamp * 1000; // Convert to milliseconds
+  }
+  throw new Error('Timestamp not found');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const match = logLine.match(TIMESTAMP_REGEX);
+  if (match) {
+    const newTimestamp = Math.floor(timestamp / 1000); // Convert to seconds
+    return `- ${newTimestamp} ${match[2]}`;
+  }
+  throw new Error('Timestamp not found');
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Thunderbird/queries.json

@@ -0,0 +1,360 @@
+{
+  "queries": [
+    {
+      "id": "gmetad_startup",
+      "title": "Gmetad Startup Succeeded",
+      "description": "Identifies logs where gmetad startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "gmetad startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "portmap_startup",
+      "title": "Portmap Startup Succeeded",
+      "description": "Identifies logs where portmap startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "portmap startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "rpc_statd_startup",
+      "title": "RPC.Statd Startup Succeeded",
+      "description": "Identifies logs where rpc.statd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "rpc.statd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "syslog_ng_startup",
+      "title": "Syslog-NG Startup Succeeded",
+      "description": "Identifies logs where syslog-ng startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "syslog-ng startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "xinetd_startup",
+      "title": "Xinetd Startup Succeeded",
+      "description": "Identifies logs where xinetd startup succeeded.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "xinetd startup succeeded",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "bind_port_failed",
+      "title": "Bind to Port Failed",
+      "description": "Identifies logs where binding to a port failed due to address already in use.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "error: Bind to port .* on .* failed: Address already in use."
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "local_disconnected",
+      "title": "Local Disconnected",
+      "description": "Identifies logs where a local connection was disconnected.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Local disconnected: Connection closed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "connection_lost",
+      "title": "Connection Lost",
+      "description": "Identifies logs where a connection was lost.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "connection lost: 'Connection closed.'",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cannot_open_file",
+      "title": "Cannot Open File",
+      "description": "Identifies logs where a file could not be opened for writing.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Cannot open file /dev/logsurfer for writing (No such file or directory)",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "resolve_data_source_failed",
+      "title": "Failed to Resolve Data Source",
+      "description": "Identifies logs where resolving a data source name failed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Warning: we failed to resolve data source name .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "resolve_trusted_host_failed",
+      "title": "Failed to Resolve Trusted Host",
+      "description": "Identifies logs where resolving a trusted host name failed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Warning: we failed to resolve trusted host name .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "wait_for_ready_failed",
+      "title": "Wait for Ready Failed",
+      "description": "Identifies logs where waiting for ready state before probe failed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Wait for ready failed before probe !",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "cache_data_failed",
+      "title": "Cache Data Request Failed",
+      "description": "Identifies logs where asking for cache data failed.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "asking for cache data failed",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "probe_failed",
+      "title": "Probe Failed",
+      "description": "Identifies logs where probing vesafb0 failed with an error.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "probe of vesafb0 failed with error .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "dhcpdiscover_no_free_leases",
+      "title": "DHCPDISCOVER No Free Leases",
+      "description": "Identifies logs where a DHCPDISCOVER request found no free leases.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "DHCPDISCOVER from .* via eth1: network A_net: no free leases"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "dhcprequest_unknown_lease",
+      "title": "DHCPREQUEST Unknown Lease",
+      "description": "Identifies logs where a DHCPREQUEST was made for an unknown lease.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "DHCPREQUEST for .* \\(.*\\) from .* via eth1: unknown lease .*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unqualified_host_name",
+      "title": "Unqualified Host Name",
+      "description": "Identifies logs where an unqualified host name was unknown, leading to a retry.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "My unqualified host name \\(.*\\) unknown; sleeping for retry"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unable_to_qualify_domain",
+      "title": "Unable to Qualify Domain",
+      "description": "Identifies logs where the system was unable to qualify its own domain name.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "unable to qualify my own domain name \\(.*\\) -- using short name"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "session_closed_root",
+      "title": "Session Closed for Root",
+      "description": "Identifies logs where a session was closed for the root user.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "session closed for user root",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "session_opened_root",
+      "title": "Session Opened for Root",
+      "description": "Identifies logs where a session was opened for the root user by uid=0.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "session opened for user root by (uid=0)",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/parser.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const TIMESTAMP_REGEX_1 = /\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/;
+const TIMESTAMP_REGEX_2 = /\d{4}\/\d{1,2}\/\d{1,2}:\d{2}:\d{2}:\d{2}\.\d{3}/;
+
+export function getTimestamp(logLine: string): number {
+  const match1 = logLine.match(TIMESTAMP_REGEX_1);
+  if (match1) {
+    return moment.utc(match1[0], 'YYYY-MM-DD HH:mm:ss').valueOf();
+  }
+
+  const match2 = logLine.match(TIMESTAMP_REGEX_2);
+  if (match2) {
+    return moment.utc(match2[0], 'YYYY/M/D:HH:mm:ss.SSS').valueOf();
+  }
+
+  throw new Error('No valid timestamp found in log line');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const match1 = logLine.match(TIMESTAMP_REGEX_1);
+  if (match1) {
+    const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss');
+    return logLine.replace(TIMESTAMP_REGEX_1, newTimestamp);
+  }
+
+  const match2 = logLine.match(TIMESTAMP_REGEX_2);
+  if (match2) {
+    const newTimestamp = moment.utc(timestamp).format('YYYY/M/D:HH:mm:ss.SSS');
+    return logLine.replace(TIMESTAMP_REGEX_2, newTimestamp);
+  }
+
+  throw new Error('No valid timestamp found in log line');
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Windows/queries.json

@@ -0,0 +1,268 @@
+{
+  "queries": [
+    {
+      "id": "startup_trustedinstaller_main_loop",
+      "title": "Startup TrustedInstaller Main Loop",
+      "description": "Returns logs where the TrustedInstaller main loop is starting.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting TrustedInstaller main loop",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_trustedinstaller_finalization",
+      "title": "Startup TrustedInstaller Finalization",
+      "description": "Returns logs where the TrustedInstaller finalization is starting.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting TrustedInstaller finalization",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_trustedinstaller_initialization",
+      "title": "Startup TrustedInstaller Initialization",
+      "description": "Returns logs where the TrustedInstaller initialization is starting.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting TrustedInstaller initialization",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "trustedinstaller_service_start",
+      "title": "TrustedInstaller Service Start",
+      "description": "Returns logs where the TrustedInstaller service starts successfully.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "TrustedInstaller service starts successfully",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_trustedinstaller_main_loop",
+      "title": "Shutdown TrustedInstaller Main Loop",
+      "description": "Returns logs where the TrustedInstaller main loop is ending.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Ending TrustedInstaller main loop",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_trustedinstaller_finalization",
+      "title": "Shutdown TrustedInstaller Finalization",
+      "description": "Returns logs where the TrustedInstaller finalization is ending.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Ending TrustedInstaller finalization",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_trustedinstaller_initialization",
+      "title": "Shutdown TrustedInstaller Initialization",
+      "description": "Returns logs where the TrustedInstaller initialization is ending.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Ending TrustedInstaller initialization",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_manifest_invalid_item",
+      "title": "Error Manifest Invalid Item",
+      "description": "Returns logs where there is an error related to an invalid item in the manifest.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Expecting attribute name \\[HRESULT = .* - CBS_E_MANIFEST_INVALID_ITEM\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_backup_log_cab",
+      "title": "Error Backup Log Cab",
+      "description": "Returns logs where there is an error creating a backup log cab.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Failed to create backup log cab\\. \\[HRESULT = .* - ERROR_INVALID_FUNCTION\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_next_element",
+      "title": "Error Next Element",
+      "description": "Returns logs where there is an error getting the next element.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Failed to get next element \\[HRESULT = .* - CBS_E_MANIFEST_INVALID_ITEM\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_open_package",
+      "title": "Error Open Package",
+      "description": "Returns logs where there is an error opening a package internally.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Failed to internally open package\\. \\[HRESULT = .* - CBS_E_INVALID_PACKAGE\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_sqm_sample_upload",
+      "title": "Error SQM Sample Upload",
+      "description": "Returns logs where there is an error starting a standard sample upload.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "SQM: Failed to start standard sample upload\\. \\[HRESULT = .* - E_FAIL\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_sqm_upload_file_pattern",
+      "title": "Error SQM Upload File Pattern",
+      "description": "Returns logs where there is an error starting an upload with a specific file pattern.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "SQM: Failed to start upload with file pattern: .* flags: .* \\[HRESULT = .* - E_FAIL\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_sqm_unsent_reports",
+      "title": "Warning SQM Unsent Reports",
+      "description": "Returns logs where there is a warning about failing to upload all unsent reports.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "SQM: Warning: Failed to upload all unsent reports\\. \\[HRESULT = .* - E_FAIL\\]"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "warning_unrecognized_package_attribute",
+      "title": "Warning Unrecognized Package Attribute",
+      "description": "Returns logs where there is a warning about an unrecognized packageExtended attribute.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Warning Unrecognized packageExtended attribute",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/parser.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+const timestampRegex = /(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}),(\d{3})/;
+
+export function getTimestamp(logLine: string): number {
+  const match = logLine.match(timestampRegex);
+  if (match) {
+    const [_, year, month, day, hour, minute, second, millisecond] = match;
+    const timestamp = `${year}-${month}-${day} ${hour}:${minute}:${second}.${millisecond}`;
+    return moment.utc(timestamp, 'YYYY-MM-DD HH:mm:ss.SSS').valueOf();
+  }
+  throw new Error('Invalid log line format');
+}
+
+export function replaceTimestamp(logLine: string, timestamp: number): string {
+  const newTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss,SSS');
+  return logLine.replace(timestampRegex, newTimestamp);
+}

x-pack/platform/packages/shared/kbn-sample-parser/parsers/Zookeeper/queries.json

@@ -0,0 +1,201 @@
+{
+  "queries": [
+    {
+      "id": "shutdown_message",
+      "title": "Shutdown Message",
+      "description": "Identifies log messages indicating a shutdown event with the keyword 'GOODBYE'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "GOODBYE",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "error_message",
+      "title": "Error Message",
+      "description": "Identifies log messages indicating an error related to opening a channel at an election address.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Cannot open channel to .* at election address /.*:.*"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "exception_message",
+      "title": "Exception Message",
+      "description": "Identifies log messages indicating an 'end of stream' exception.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "caught end of stream exception",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "connection_broken_message",
+      "title": "Connection Broken Message",
+      "description": "Identifies log messages indicating a broken connection for a specific ID.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Connection broken for id .*, my id = .*, error ="
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "ioexception_message",
+      "title": "IOException Message",
+      "description": "Identifies log messages indicating a session close due to a java.io.IOException with the message 'ZooKeeperServer not running'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Exception causing close of session .* due to java.io.IOException: ZooKeeperServer not running"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "session_expiry_message",
+      "title": "Session Expiry Message",
+      "description": "Identifies log messages indicating a session expiry due to a timeout.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "regexp": {
+                "message": "Expiring session .*, timeout of .*ms exceeded"
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_message",
+      "title": "Startup Message",
+      "description": "Identifies log messages indicating a startup event with the keyword 'starting up'.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "starting up",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "shutdown_complete_message",
+      "title": "Shutdown Complete Message",
+      "description": "Identifies log messages indicating the completion of a shutdown process.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "shutdown of request processor complete",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "startup_quorum_peer_message",
+      "title": "Startup Quorum Peer Message",
+      "description": "Identifies log messages indicating the startup of a quorum peer.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Starting quorum peer",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unexpected_exception_shutdown_message",
+      "title": "Unexpected Exception Causing Shutdown Message",
+      "description": "Identifies log messages indicating an unexpected exception causing a shutdown while the socket is still open.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Unexpected exception causing shutdown while sock still open",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    {
+      "id": "unexpected_exception_message",
+      "title": "Unexpected Exception Message",
+      "description": "Identifies log messages indicating an unexpected exception.",
+      "query": {
+        "bool": {
+          "filter": [
+            {
+              "match": {
+                "message": {
+                  "query": "Unexpected Exception",
+                  "operator": "AND"
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/constants.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import Path from 'path';
+import { REPO_ROOT } from '@kbn/repo-info';
+
+export const LOGHUB_DIR = Path.join(REPO_ROOT, '../loghub');
+export const LOGHUB_REPO = 'https://github.com/logpai/loghub.git';
+export const LOGHUB_PARSER_DIR = Path.join(__dirname, '../parsers');

x-pack/platform/packages/shared/kbn-sample-parser/src/create_openai_client.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AzureOpenAI } from 'openai';
+
+export interface OpenAIClient extends AzureOpenAI {
+  model: string;
+}
+
+export function createOpenAIClient(): OpenAIClient {
+  const apiKey = process.env.AZURE_OPENAI_API_KEY;
+  const endpoint = process.env.AZURE_OPENAI_ENDPOINT;
+  const deployment = process.env.AZURE_OPENAI_DEPLOYMENT;
+  const apiVersion = process.env.AZURE_OPENAI_API_VERSION ?? '2025-01-01-preview';
+
+  const openAIClient = new AzureOpenAI({
+    apiKey,
+    endpoint,
+    deployment,
+    apiVersion,
+  }) as OpenAIClient;
+  const model = process.env.AZURE_OPENAI_MODEL || `gpt-4o`;
+
+  // chat complete methods require a model parameter,
+  // so we expose it on the client in order to ~fully
+  // encapsulate config
+  openAIClient.model = model;
+
+  return openAIClient;
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_loghub_repo.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { ToolingLog } from '@kbn/tooling-log';
+import { promises as Fs } from 'fs';
+import simpleGit from 'simple-git';
+import { LOGHUB_DIR, LOGHUB_REPO } from './constants';
+
+export async function ensureLoghubRepo({ log }: { log: ToolingLog }) {
+  const dirExists = await Fs.stat(LOGHUB_DIR)
+    .then((stat) => stat.isDirectory())
+    .catch(() => false);
+
+  if (!dirExists) {
+    log.info(`Directory "${LOGHUB_DIR}" does not exist. Cloning repository...`);
+    await simpleGit().clone(LOGHUB_REPO, LOGHUB_DIR, ['--depth', '1']);
+  }
+
+  const repoGit = simpleGit(LOGHUB_DIR);
+
+  log.debug(`Fetching from logai/loghub`);
+
+  await repoGit.fetch();
+
+  const defaultBranch =
+    (await repoGit.revparse(['--abbrev-ref', 'origin/HEAD'])).replace('origin/', '') || 'master';
+
+  const currentBranch = (await repoGit.revparse(['--abbrev-ref', 'HEAD'])) || defaultBranch;
+
+  if (currentBranch !== defaultBranch) {
+    log.info(`Checking out ${defaultBranch}`);
+
+    await repoGit.checkout(defaultBranch);
+  }
+
+  const status = await repoGit.status();
+  if (status.behind && status.behind > 0) {
+    log.info(`Local branch is behind by ${status.behind} commit(s); pulling changes.`);
+    await repoGit.pull('origin', defaultBranch);
+  } else {
+    log.debug(`Local branch is up-to-date; no pull needed.`);
+  }
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_parser.ts

@@ -0,0 +1,176 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import pRetry from 'p-retry';
+import { OpenAIClient } from './create_openai_client';
+import { type LoghubSystem } from './read_loghub_system_files';
+import { getParserFilename, writeFileRecursively } from './utils';
+import { validateParser } from './validate_parser';
+
+async function generateParser({
+  openAIClient,
+  system,
+  error,
+  log,
+}: {
+  openAIClient: OpenAIClient;
+  system: LoghubSystem;
+  error?: Error;
+  log: ToolingLog;
+}): Promise<string> {
+  log.info(`Attempting to generate a parser for ${system.name}`);
+
+  const systemPrompt = `You are given a system's documentation and log files.
+      
+      Your goal is to write a TypeScript files that exports two functions:
+      
+      - \`getTimestamp ( logLine:string ):number\`: extract the timestamp
+      from the logline and return it as epoch milliseconds
+      - \`replaceTimestamp ( logLine:string, timestamp:number ):string\`:
+      replace the timestamp with the new timestamp, in the format that is
+      used in the log line.
+      
+      Generally, you will want to generate
+      a regular expression that can be used to A) extract the values
+      to parse the date, B) replace the original timestamp with the
+      formatted injected timestamp.
+
+      You can use \`moment\`, but not any other libraries.
+
+      Some notes:
+      - in some cases, leading 0s are stripped. Take this into account
+      when generating a regex (e.g. use \d{2,3} instead of \d{2}).
+            `;
+
+  const reasoningResponse = await openAIClient.chat.completions.create({
+    model: openAIClient.model,
+    messages: [
+      {
+        role: 'system',
+        content: systemPrompt,
+      },
+      {
+        role: 'user',
+        content: `Analyze the log lines and reason about the date format
+            in the log lines. Find the piece of data that is likely the
+            timestamp. Reason about regular expressions, timezones, date
+            formatting/parsing from the format in the log line to and from
+            epoch ms. You don't have to write the actual code yet, that
+            happens in a follow-up, but you can write snippets. If an error
+            occurred, reason about how the error should be fixed.
+
+            ${
+              error
+                ? `# Error
+            The following error occurred on a previous attempt: ${error.message}`
+                : ''
+            }
+            
+            ## README.md
+
+            ${system.readme ?? 'Empty'}
+
+            ## Log file
+
+            ${system.logLines.slice(0, 500).join('\n')}
+            `,
+      },
+    ],
+    temperature: 0.2,
+  });
+
+  const analysis = reasoningResponse.choices[0].message.content;
+
+  const output = await openAIClient.chat.completions.create({
+    model: openAIClient.model,
+    messages: [
+      {
+        role: 'system',
+        content: systemPrompt,
+      },
+      {
+        role: 'user',
+        content: `Based on previous reasoning, output the
+            typescript file in the following format. Make sure
+            any dates are parsed as UTC if the time zone is not
+            specified, regardless of where the script runs:
+
+            export function getTimestamp ( logLine:string ):number {
+              // function implementation here
+            }
+
+            export function replaceTimestamp ( logLine:string, timestamp:number ) {
+              // function implementation here
+            }
+
+            DO NOT output anything else, including any markdown backticks,
+            just the file contents. The result of your output will be written
+            to disk directly. If you use \`moment\`, add the following import
+            statement to the top of the file:
+
+            \`\`\`import moment from "moment";\`\`\`
+
+            If you use a regex that is shared, add it to the top of the file
+            as a constant.
+
+            # Reasoning
+            
+            ${analysis}
+            
+            ## README.md
+
+            ${system.readme ?? 'Empty'}
+
+            ## Log file
+
+            ${system.logLines.slice(0, 500).join('\n')}
+            `,
+      },
+    ],
+    temperature: 0.2,
+  });
+
+  const file = output.choices[0].message.content;
+
+  if (!file) {
+    throw new Error(`No content received from the LLM`);
+  }
+
+  return file;
+}
+
+export async function ensureValidParser({
+  openAIClient,
+  system,
+  log,
+}: {
+  openAIClient: OpenAIClient;
+  system: LoghubSystem;
+  log: ToolingLog;
+}) {
+  let error: Error | undefined;
+
+  const isValid = await validateParser(system)
+    .then(() => true)
+    .catch(() => false);
+
+  if (isValid) {
+    return;
+  }
+
+  await pRetry(
+    async () => {
+      const file = await generateParser({ system, error, log, openAIClient });
+
+      await writeFileRecursively(getParserFilename(system), file);
+
+      await validateParser(system);
+    },
+    { retries: 5 }
+  );
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/ensure_valid_queries.ts

@@ -0,0 +1,201 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import pRetry from 'p-retry';
+import { OpenAIClient } from './create_openai_client';
+import { type LoghubSystem } from './read_loghub_system_files';
+import { getQueriesFilename, writeFileRecursively } from './utils';
+import { queryFileSchema, validateQueries } from './validate_queries';
+
+async function generateQueries({
+  openAIClient,
+  system,
+  error,
+  log,
+}: {
+  openAIClient: OpenAIClient;
+  system: LoghubSystem;
+  error?: Error;
+  log: ToolingLog;
+}): Promise<string> {
+  log.info(`Attempting to generate queries for ${system.name}`);
+
+  const systemPrompt = `You are an expert in Elasticsearch DSL, log patterns
+      and log analytics.`;
+
+  const reasoningResponse = await openAIClient.chat.completions.create({
+    model: openAIClient.model,
+    messages: [
+      {
+        role: 'system',
+        content: systemPrompt,
+      },
+      {
+        role: 'user',
+        content: `Given the following log message dataset and templates,
+          generate queries using ES DSL. These queries should return log
+          messages that are "interesting", such as startup/shutdown related
+          messages, or warning/error/fatal messages. For each query,
+          use a single \`match\` or \`regexp\` query. Keep the following
+          things in mind:
+
+          - Each query should return a single template. Do not group templates
+          together.
+          - Use the \`message\` field.
+          - In the templates, \`<*>\` refers to a _variable_ in the actual log
+          message. DO NOT include this in your query. Instead, just skip over
+          the word in your match query, or group it as a variable in your
+          regexp query.
+          - The default operator for a \`match\` query is \`OR\`. Keep this in
+          mind, most likely you'll want \`AND\`. Use capitalized letters.
+
+            ${
+              error
+                ? `# Error
+            The following error occurred on a previous attempt: ${error.message}`
+                : ''
+            }
+          # Readme
+
+          ${system.readme}
+
+          # Templates
+
+          ${system.templates.join('\n')}
+            `,
+      },
+    ],
+    temperature: 0.2,
+  });
+
+  const analysis = reasoningResponse.choices[0].message.content;
+
+  log.verbose(`Analysis for ${system.name}:
+    
+    ${analysis}`);
+
+  const output = await openAIClient.chat.completions.create({
+    model: openAIClient.model,
+    messages: [
+      {
+        role: 'system',
+        content: systemPrompt,
+      },
+      {
+        role: 'user',
+        content: `Based on previous reasoning, output the
+        queries in a structured format.
+        
+        # Analysis
+        ${analysis}`,
+      },
+    ],
+    tools: [
+      {
+        function: {
+          name: 'output',
+          description:
+            'Output the queries in structured data. Make sure you output the query DSL in `queries[n].query`',
+          strict: true,
+          parameters: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              queries: {
+                type: 'array',
+                items: {
+                  type: 'object',
+                  additionalProperties: false,
+                  properties: {
+                    id: {
+                      type: 'string',
+                      description:
+                        'The id of the query. Use [a-z_]+. Make sure the id is descriptive of the query, but keep it short.',
+                    },
+                    query: {
+                      type: 'string',
+                      description:
+                        'The Elasticsearch Query DSL for the query, as serialized JSON. Wrap in { bool: { filter: [...] } }',
+                    },
+                    title: {
+                      type: 'string',
+                      description: 'A short title for the query',
+                    },
+                    description: {
+                      type: 'string',
+                      description: 'A human-readable description of what the query returns',
+                    },
+                  },
+                  required: ['id', 'query', 'title', 'description'],
+                },
+              },
+            },
+            required: ['queries'],
+          },
+        },
+        type: 'function',
+      },
+    ],
+    tool_choice: {
+      function: {
+        name: 'output',
+      },
+      type: 'function',
+    },
+    temperature: 0.2,
+  });
+
+  const message = output.choices[0].message;
+
+  const args = message.tool_calls?.[0]?.function.arguments;
+
+  if (!args) {
+    throw new Error(`Expected tool call, received message: ${message.content}`);
+  }
+
+  const queries = queryFileSchema.parse(JSON.parse(args));
+
+  return JSON.stringify(queries, null, 2);
+}
+
+export async function ensureValidQueries({
+  openAIClient,
+  system,
+  log,
+}: {
+  openAIClient: OpenAIClient;
+  system: LoghubSystem;
+  log: ToolingLog;
+}) {
+  let error: Error | undefined;
+
+  const isValid = await validateQueries(system)
+    .then(() => true)
+    .catch(() => false);
+
+  if (isValid) {
+    return;
+  }
+
+  await pRetry(
+    async () => {
+      const file = await generateQueries({ system, error, log, openAIClient });
+
+      await writeFileRecursively(getQueriesFilename(system), file);
+
+      await validateQueries(system);
+    },
+    {
+      retries: 5,
+      onFailedAttempt(err) {
+        log.debug(`Error generating queries for ${system.name}`);
+        log.debug(err);
+      },
+    }
+  );
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/get_parser.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type LoghubSystem } from './read_loghub_system_files';
+import { LoghubParser } from './types';
+import { getParserFilename } from './utils';
+
+export async function getParser(system: LoghubSystem): Promise<LoghubParser> {
+  const fileName = getParserFilename(system);
+
+  return (await import(fileName)) as LoghubParser;
+}

x-pack/platform/packages/shared/kbn-sample-parser/src/get_queries.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { type LoghubSystem } from './read_loghub_system_files';
+import { getQueriesFilename } from './utils';
+import { LoghubQuery } from './validate_queries';
+
+export async function getQueries(system: LoghubSystem): Promise<LoghubQuery[]> {
+  const fileName = getQueriesFilename(system);
+
+  const { queries } = (await import(fileName)) as { queries: LoghubQuery[] };
+  return queries;
+}