github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Log Explorer] Add support for log generation in synthtrace (#170107)

Browse source 
Closes - https://github.com/elastic/kibana/issues/170133

## Summary

This PR adds support for generating logs using the Synthtrace Client

Changes include

1. Changes to Synthtrace package to support new Logs Client and Log
Class for helper methods
2. [Stateful Tests] - Change to our FTR Context config to inject the new
the Log Synthtrace Client
3. [Serverless Tests] - Injected Synthtrace as a service for serverless
tests.
4. A sample test added to `app.ts` to demonstrate how Synthtrace can be
used to generate Log data in both Stateful and Serverless
5. Add support to generate logs via CLI. 2 scenarios added -
`simple_logs.ts` and `logs_and_metrics.ts`

```
# Live Data
node scripts/synthtrace simple_logs.ts --clean --live

# Static Data
node scripts/synthtrace simple_logs.ts --clean
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tiago Costa <tiago.costa@elastic.co>
Co-authored-by: Yngrid Coello <yngrdyn@gmail.com>
This commit is contained in:
Achyut Jhunjhunwala 2023-11-16 14:00:33 +01:00 • committed by GitHub
parent 0916894657
commit db5176b17a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1047 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.github/CODEOWNERS vendored
View file

@ -39,8 +39,8 @@ packages/analytics/shippers/gainsight @elastic/kibana-core
packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam
x-pack/plugins/apm_data_access @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team
x-pack/plugins/apm @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team
packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team
packages/kbn-apm-utils @elastic/obs-ux-infra_services-team
test/plugin_functional/plugins/app_link_test @elastic/kibana-core
x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core

1
packages/kbn-apm-synthtrace-client/index.ts
View file

@ -33,3 +33,4 @@ export { dedot } from './src/lib/utils/dedot';
export { generateLongId, generateShortId } from './src/lib/utils/generate_id';
export { appendHash, hashKeysOf } from './src/lib/utils/hash';
export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } from './src/types';
export { log, type LogDocument } from './src/lib/logs';

2
packages/kbn-apm-synthtrace-client/kibana.jsonc
View file

@ -2,5 +2,5 @@
"type": "shared-common",
"id": "@kbn/apm-synthtrace-client",
"devOnly": true,
"owner": "@elastic/obs-ux-infra_services-team"
"owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
}

77
packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts Normal file
View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { Fields } from '../entity';
import { Serializable } from '../serializable';
export type LogDocument = Fields &
Partial<{
'input.type': string;
'log.file.path'?: string;
'service.name'?: string;
'data_stream.namespace': string;
'data_stream.type': string;
'data_stream.dataset': string;
message?: string;
'event.dataset': string;
'log.level'?: string;
'host.name'?: string;
'trace.id'?: string;
'agent.name'?: string;
'orchestrator.cluster.name'?: string;
'orchestrator.cluster.id'?: string;
'orchestrator.resource.id'?: string;
'cloud.provider'?: string;
'cloud.region'?: string;
'cloud.availability_zone'?: string;
'cloud.project.id'?: string;
'cloud.instance.id'?: string;
}>;
class Log extends Serializable<LogDocument> {
service(name: string) {
this.fields['service.name'] = name;
return this;
}
namespace(value: string) {
this.fields['data_stream.namespace'] = value;
return this;
}
dataset(value: string) {
this.fields['data_stream.dataset'] = value;
this.fields['event.dataset'] = value;
return this;
}
logLevel(level: string) {
this.fields['log.level'] = level;
return this;
}
message(message: string) {
this.fields.message = message;
return this;
}
}
function create(): Log {
return new Log({
'input.type': 'logs',
'data_stream.namespace': 'default',
'data_stream.type': 'logs',
'data_stream.dataset': 'synth',
'event.dataset': 'synth',
'host.name': 'synth-host',
});
}
export const log = {
create,
};

27
packages/kbn-apm-synthtrace/README.md
View file

@ -21,6 +21,7 @@ This library can currently be used in two ways:
- `Instance`: a single instance of a monitored service. E.g., the workload for a monitored service might be spread across multiple containers. An `Instance` object contains fields like `service.node.name` and `container.id`.
- `Timerange`: an object that will return an array of timestamps based on an interval and a rate. These timestamps can be used to generate events/metricsets.
- `Transaction`, `Span`, `APMError` and `Metricset`: events/metricsets that occur on an instance. For more background, see the [explanation of the APM data model](https://www.elastic.co/guide/en/apm/get-started/7.15/apm-data-model.html)
- `Log`: An instance of Log generating Service which supports additional helpers to customise fields like `messages`, `logLevel`
#### Example
@ -109,12 +110,22 @@ node scripts/synthtrace simple_trace.ts --target=http://admin:changeme@localhost
The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
### Understanding Scenario Files
Scenario files accept 3 arguments, 2 of them optional and 1 mandatory
| Arguments | Type | Description |
|-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------|
| `generate` | mandatory | This is the main function responsible for returning the events which will be indexed |
| `bootstrap` | optional | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with |
| `setClient` | optional | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned |
The following options are supported:
### Connection options
| Option | Type | Default | Description |
| ------------------- | -------- | :------ | ------------------------------------------------------------------------------------------ |
|---------------------|----------|:--------|--------------------------------------------------------------------------------------------|
| `--target` | [string] | | Elasticsearch target |
| `--kibana` | [string] | | Kibana target, used to bootstrap datastreams/mappings/templates/settings |
| `--versionOverride` | [string] | | String to be used for `observer.version`. Defauls to the version of the installed package. |
@ -129,12 +140,11 @@ Note:
### Scenario options
| Option | Type | Default | Description |
| ---------------- | --------- | :------ | ------------------------------------ |
|------------------|-----------|:--------|--------------------------------------|
| `--from` | [date] | `now()` | The start of the time window |
| `--to` | [date] | | The end of the time window |
| `--live` | [boolean] | | Generate and index data continuously |
| `--scenarioOpts` | | | Raw options specific to the scenario |
Note:
- The default `--to` is `15m`.
@ -143,11 +153,12 @@ Note:
### Setup options
| Option | Type | Default | Description |
| ------------ | --------- | :------ | --------------------------------------- |
| `--clean` | [boolean] | `false` | Clean APM data before indexing new data |
| `--workers` | [number] | | Amount of Node.js worker threads |
| `--logLevel` | [enum] | `info` | Log level |
| Option | Type | Default | Description |
|--------------|-----------|:--------|-------------------------------------------------------------------------|
| `--clean` | [boolean] | `false` | Clean APM data before indexing new data |
| `--workers` | [number] | | Amount of Node.js worker threads |
| `--logLevel` | [enum] | `info` | Log level |
| `--type` | [string] | `apm` | Type of data to be generated, `log` must be passed when generating logs |
## Testing

2
packages/kbn-apm-synthtrace/index.ts
View file

@ -17,6 +17,8 @@ export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_
export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_synthtrace_es_client';
export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client';
export {
addObserverVersionTransform,
deleteSummaryFieldTransform,

2
packages/kbn-apm-synthtrace/kibana.jsonc
View file

@ -2,5 +2,5 @@
"type": "shared-server",
"id": "@kbn/apm-synthtrace",
"devOnly": true,
"owner": "@elastic/obs-ux-infra_services-team"
"owner": ["@elastic/obs-ux-infra_services-team", "@elastic/obs-ux-logs-team"]
}

17
packages/kbn-apm-synthtrace/src/cli/scenario.ts
View file

@ -6,17 +6,24 @@
* Side Public License, v 1.
*/
import { SynthtraceGenerator, Timerange } from '@kbn/apm-synthtrace-client';
import { Readable } from 'stream';
import { ApmSynthtraceEsClient } from '../lib/apm/client/apm_synthtrace_es_client';
import { Timerange } from '@kbn/apm-synthtrace-client';
import { Logger } from '../lib/utils/create_logger';
import { RunOptions } from './utils/parse_run_cli_flags';
import { ApmSynthtraceEsClient, LogsSynthtraceEsClient } from '../..';
import { ScenarioReturnType } from '../lib/utils/with_client';
type Generate<TFields> = (options: {
range: Timerange;
}) => SynthtraceGenerator<TFields> | Array<SynthtraceGenerator<TFields>> | Readable;
clients: {
apmEsClient: ApmSynthtraceEsClient;
logsEsClient: LogsSynthtraceEsClient;
};
}) => ScenarioReturnType<TFields> | Array<ScenarioReturnType<TFields>>;
export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
bootstrap?: (options: { apmEsClient: ApmSynthtraceEsClient }) => Promise<void>;
bootstrap?: (options: {
apmEsClient: ApmSynthtraceEsClient;
logsEsClient: LogsSynthtraceEsClient;
}) => Promise<void>;
generate: Generate<TFields>;
}>;

13
packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts
View file

@ -7,7 +7,8 @@
*/
import { createLogger } from '../../lib/utils/create_logger';
import { getEsClient } from './get_es_client';
import { getApmEsClient } from './get_apm_es_client';
import { getLogsEsClient } from './get_logs_es_client';
import { getKibanaClient } from './get_kibana_client';
import { getServiceUrls } from './get_service_urls';
import { RunOptions } from './parse_run_cli_flags';
@ -26,22 +27,30 @@ export async function bootstrap(runOptions: RunOptions) {
const version = runOptions.versionOverride || latestPackageVersion;
const apmEsClient = getEsClient({
const apmEsClient = getApmEsClient({
target: esUrl,
logger,
concurrency: runOptions.concurrency,
version,
});
const logsEsClient = getLogsEsClient({
target: esUrl,
logger,
concurrency: runOptions.concurrency,
});
await kibanaClient.installApmPackage(latestPackageVersion);
if (runOptions.clean) {
await apmEsClient.clean();
await logsEsClient.clean();
}
return {
logger,
apmEsClient,
logsEsClient,
version,
kibanaUrl,
esUrl,

4
packages/kbn-apm-synthtrace/src/cli/utils/get_es_client.ts → packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts
View file

@ -7,11 +7,11 @@
*/
import { Client } from '@elastic/elasticsearch';
import { ApmSynthtraceEsClient } from '../../lib/apm/client/apm_synthtrace_es_client';
import { ApmSynthtraceEsClient } from '../../..';
import { Logger } from '../../lib/utils/create_logger';
import { RunOptions } from './parse_run_cli_flags';
export function getEsClient({
export function getApmEsClient({
target,
logger,
version,

31
packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { Client } from '@elastic/elasticsearch';
import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client';
import { Logger } from '../../lib/utils/create_logger';
import { RunOptions } from './parse_run_cli_flags';
export function getLogsEsClient({
target,
logger,
concurrency,
}: Pick<RunOptions, 'concurrency'> & {
target: string;
logger: Logger;
}) {
const client = new Client({
node: target,
});
return new LogsSynthtraceEsClient({
client,
logger,
concurrency,
});
}

80
packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts
View file

@ -14,6 +14,7 @@ import { awaitStream } from '../../lib/utils/wait_until_stream_finished';
import { bootstrap } from './bootstrap';
import { getScenario } from './get_scenario';
import { RunOptions } from './parse_run_cli_flags';
import { SynthtraceEsClient } from '../../lib/utils/with_client';
export async function startLiveDataUpload({
runOptions,
@ -24,7 +25,7 @@ export async function startLiveDataUpload({
}) {
const file = runOptions.file;
const { logger, apmEsClient } = await bootstrap(runOptions);
const { logger, apmEsClient, logsEsClient } = await bootstrap(runOptions);
const scenario = await getScenario({ file, logger });
const { generate } = await scenario({ ...runOptions, logger });
@ -32,22 +33,22 @@ export async function startLiveDataUpload({
const bucketSizeInMs = 1000 * 60;
let requestedUntil = start;
const stream = new PassThrough({
objectMode: true,
});
let currentStreams: PassThrough[] = [];
const cachedStreams: WeakMap<SynthtraceEsClient, PassThrough> = new WeakMap();
apmEsClient.index(stream);
process.on('SIGINT', () => closeStreams());
process.on('SIGTERM', () => closeStreams());
process.on('SIGQUIT', () => closeStreams());
function closeStream() {
stream.end(() => {
process.exit(0);
function closeStreams() {
currentStreams.forEach((stream) => {
stream.end(() => {
process.exit(0);
});
});
currentStreams = []; // Reset the stream array
}
process.on('SIGINT', closeStream);
process.on('SIGTERM', closeStream);
process.on('SIGQUIT', closeStream);
async function uploadNextBatch() {
const now = Date.now();
@ -59,22 +60,52 @@ export async function startLiveDataUpload({
`Requesting ${new Date(bucketFrom).toISOString()} to ${new Date(bucketTo).toISOString()}`
);
const next = logger.perf('execute_scenario', () =>
generate({ range: timerange(bucketFrom.getTime(), bucketTo.getTime()) })
);
const generatorsAndClients = generate({
range: timerange(bucketFrom.getTime(), bucketTo.getTime()),
clients: { logsEsClient, apmEsClient },
});
const concatenatedStream = castArray(next)
.reverse()
.reduce<Writable>((prev, current) => {
const currentStream = isGeneratorObject(current) ? Readable.from(current) : current;
return currentStream.pipe(prev);
}, new PassThrough({ objectMode: true }));
const generatorsAndClientsArray = castArray(generatorsAndClients);
concatenatedStream.pipe(stream, { end: false });
const streams = generatorsAndClientsArray.map(({ client }) => {
let stream: PassThrough;
await awaitStream(concatenatedStream);
if (cachedStreams.has(client)) {
stream = cachedStreams.get(client)!;
} else {
stream = new PassThrough({ objectMode: true });
cachedStreams.set(client, stream);
client.index(stream);
}
await apmEsClient.refresh();
return stream;
});
currentStreams = streams;
const promises = generatorsAndClientsArray.map(({ generator }, i) => {
const concatenatedStream = castArray(generator)
.reverse()
.reduce<Writable>((prev, current) => {
const currentStream = isGeneratorObject(current) ? Readable.from(current) : current;
return currentStream.pipe(prev);
}, new PassThrough({ objectMode: true }));
concatenatedStream.pipe(streams[i], { end: false });
return awaitStream(concatenatedStream);
});
await Promise.all(promises);
logger.info('Indexing completed');
const refreshPromise = generatorsAndClientsArray.map(async ({ client }) => {
await client.refresh();
});
await Promise.all(refreshPromise);
logger.info('Refreshing completed');
requestedUntil = bucketTo;
}
@ -85,6 +116,7 @@ export async function startLiveDataUpload({
await delay(bucketSizeInMs);
} while (true);
}
async function delay(ms: number) {
return await new Promise((resolve) => setTimeout(resolve, ms));
}

28
packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts
View file

@ -7,12 +7,14 @@
*/
import { parentPort, workerData } from 'worker_threads';
import pidusage from 'pidusage';
import { castArray } from 'lodash';
import { memoryUsage } from 'process';
import { timerange } from '@kbn/apm-synthtrace-client';
import { getEsClient } from './get_es_client';
import { getApmEsClient } from './get_apm_es_client';
import { getScenario } from './get_scenario';
import { loggerProxy } from './logger_proxy';
import { RunOptions } from './parse_run_cli_flags';
import { getLogsEsClient } from './get_logs_es_client';
export interface WorkerData {
bucketFrom: Date;
@ -27,13 +29,19 @@ const { bucketFrom, bucketTo, runOptions, esUrl, version } = workerData as Worke
async function start() {
const logger = loggerProxy;
const apmEsClient = getEsClient({
const apmEsClient = getApmEsClient({
concurrency: runOptions.concurrency,
target: esUrl,
logger,
version,
});
const logsEsClient = getLogsEsClient({
concurrency: runOptions.concurrency,
target: esUrl,
logger,
});
const file = runOptions.file;
const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger }));
@ -43,15 +51,17 @@ async function start() {
const { generate, bootstrap } = await scenario({ ...runOptions, logger });
if (bootstrap) {
await bootstrap({ apmEsClient });
await bootstrap({ apmEsClient, logsEsClient });
}
logger.debug('Generating scenario');
const generators = logger.perf('generate_scenario', () =>
generate({ range: timerange(bucketFrom, bucketTo) })
const generatorsAndClients = logger.perf('generate_scenario', () =>
generate({ range: timerange(bucketFrom, bucketTo), clients: { logsEsClient, apmEsClient } })
);
const generatorsAndClientsArray = castArray(generatorsAndClients);
logger.debug('Indexing scenario');
function mb(value: number): string {
@ -65,8 +75,12 @@ async function start() {
}, 5000);
await logger.perf('index_scenario', async () => {
await apmEsClient.index(generators);
await apmEsClient.refresh();
const promises = generatorsAndClientsArray.map(async ({ client, generator }) => {
await client.index(generator);
await client.refresh();
});
await Promise.all(promises);
});
}

61
packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { Client } from '@elastic/elasticsearch';
import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client';
import { pipeline, Readable, Transform } from 'stream';
import { LogDocument } from '@kbn/apm-synthtrace-client/src/lib/logs';
import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client';
import { getSerializeTransform } from '../shared/get_serialize_transform';
import { Logger } from '../utils/create_logger';
export type LogsSynthtraceEsClientOptions = Omit<SynthtraceEsClientOptions, 'pipeline'>;
export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
constructor(options: { client: Client; logger: Logger } & LogsSynthtraceEsClientOptions) {
super({
...options,
pipeline: logsPipeline(),
});
this.dataStreams = ['logs-*-*'];
}
}
function logsPipeline() {
return (base: Readable) => {
return pipeline(
base,
getSerializeTransform<LogDocument>(),
getRoutingTransform(),
(err: unknown) => {
if (err) {
throw err;
}
}
);
};
}
function getRoutingTransform() {
return new Transform({
objectMode: true,
transform(document: ESDocumentWithOperation<LogDocument>, encoding, callback) {
if (
'data_stream.type' in document &&
'data_stream.dataset' in document &&
'data_stream.namespace' in document
) {
document._index = `${document['data_stream.type']}-${document['data_stream.dataset']}-${document['data_stream.namespace']}`;
} else {
throw new Error('Cannot determine index for event');
}
callback(null, document);
},
});
}

10
packages/kbn-apm-synthtrace/src/lib/shared/get_serialize_transform.ts
View file

@ -9,13 +9,13 @@
import { ApmFields, Serializable } from '@kbn/apm-synthtrace-client';
import { Transform } from 'stream';
export function getSerializeTransform() {
const buffer: ApmFields[] = [];
export function getSerializeTransform<TFields = ApmFields>() {
const buffer: TFields[] = [];
let cb: (() => void) | undefined;
function push(stream: Transform, events: ApmFields[], callback?: () => void) {
let event: ApmFields | undefined;
function push(stream: Transform, events: TFields[], callback?: () => void) {
let event: TFields | undefined;
while ((event = events.shift())) {
if (!stream.push(event)) {
buffer.push(...events);
@ -37,7 +37,7 @@ export function getSerializeTransform() {
push(this, nextEvents, nextCallback);
}
},
write(chunk: Serializable<ApmFields>, encoding, callback) {
write(chunk: Serializable<TFields>, encoding, callback) {
push(this, chunk.serialize(), callback);
},
});

29
packages/kbn-apm-synthtrace/src/lib/utils/with_client.ts Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { SynthtraceGenerator } from '@kbn/apm-synthtrace-client';
import { Readable } from 'stream';
import { ApmSynthtraceEsClient, LogsSynthtraceEsClient } from '../../..';
export type SynthtraceEsClient = ApmSynthtraceEsClient | LogsSynthtraceEsClient;
export type SynthGenerator<TFields> =
| SynthtraceGenerator<TFields>
| Array<SynthtraceGenerator<TFields>>
| Readable;
export const withClient = <TFields>(
client: SynthtraceEsClient,
generator: SynthGenerator<TFields>
) => {
return {
client,
generator,
};
};
export type ScenarioReturnType<TFields> = ReturnType<typeof withClient<TFields>>;

24
packages/kbn-apm-synthtrace/src/scenarios/agent_config.ts
View file

@ -7,21 +7,25 @@
*/
import { observer, AgentConfigFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { withClient } from '../lib/utils/with_client';
const scenario: Scenario<AgentConfigFields> = async ({ logger }) => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const agentConfig = observer().agentConfig();
return range
.interval('30s')
.rate(1)
.generator((timestamp) => {
const events = logger.perf('generating_agent_config_events', () => {
return agentConfig.etag('test-etag').timestamp(timestamp);
});
return events;
});
return withClient(
apmEsClient,
range
.interval('30s')
.rate(1)
.generator((timestamp) => {
const events = logger.perf('generating_agent_config_events', () => {
return agentConfig.etag('test-etag').timestamp(timestamp);
});
return events;
})
);
},
};
};

5
packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts
View file

@ -10,12 +10,13 @@ import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const timestamps = range.ratePerMinute(180);
const cloudFields: ApmFields = {
@ -91,7 +92,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
];
});
return awsLambdaEvents;
return withClient(apmEsClient, awsLambdaEvents);
},
};
};

5
packages/kbn-apm-synthtrace/src/scenarios/azure_functions.ts
View file

@ -10,12 +10,13 @@ import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const timestamps = range.ratePerMinute(180);
const cloudFields: ApmFields = {
@ -61,7 +62,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
];
});
return awsLambdaEvents;
return withClient(apmEsClient, awsLambdaEvents);
},
};
};

8
packages/kbn-apm-synthtrace/src/scenarios/cloud_services_icons.ts
View file

@ -9,6 +9,7 @@
import { apm, ApmFields, Instance } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -16,7 +17,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const { numServices = 3 } = scenarioOpts || {};
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const transactionName = 'Azure-AWS-Transaction';
const successfulTimestamps = range.ratePerMinute(60);
@ -176,7 +177,10 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
return successfulTraceEvents;
};
return logger.perf('generating_apm_events', () => instances.flatMap(instanceSpans));
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () => instances.flatMap(instanceSpans))
);
},
};
};

62
packages/kbn-apm-synthtrace/src/scenarios/continuous_rollups.ts
View file

@ -11,6 +11,7 @@ import { merge, range as lodashRange } from 'lodash';
import { Scenario } from '../cli/scenario';
import { ComponentTemplateName } from '../lib/apm/client/apm_synthtrace_es_client';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENTS = ['production', 'development'].map((env) =>
getSynthtraceEnvironment(__filename, env)
@ -40,7 +41,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
}
);
},
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const TRANSACTION_TYPES = ['request', 'custom'];
const MIN_DURATION = 10;
@ -64,36 +65,39 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const transactionGroupRange = lodashRange(0, numTxGroups);
return range
.interval('1m')
.rate(1)
.generator((timestamp, timestampIndex) => {
return logger.perf(
'generate_events_for_timestamp ' + new Date(timestamp).toISOString(),
() => {
const events = instances.flatMap((instance) =>
transactionGroupRange.flatMap((groupId, groupIndex) =>
OUTCOMES.map((outcome) => {
const duration = Math.round(
(timestampIndex % MAX_BUCKETS) * BUCKET_SIZE + MIN_DURATION
);
return withClient(
apmEsClient,
range
.interval('1m')
.rate(1)
.generator((timestamp, timestampIndex) => {
return logger.perf(
'generate_events_for_timestamp ' + new Date(timestamp).toISOString(),
() => {
const events = instances.flatMap((instance) =>
transactionGroupRange.flatMap((groupId, groupIndex) =>
OUTCOMES.map((outcome) => {
const duration = Math.round(
(timestampIndex % MAX_BUCKETS) * BUCKET_SIZE + MIN_DURATION
);
return instance
.transaction(
`transaction-${groupId}`,
TRANSACTION_TYPES[groupIndex % TRANSACTION_TYPES.length]
)
.timestamp(timestamp)
.duration(duration)
.outcome(outcome);
})
)
);
return instance
.transaction(
`transaction-${groupId}`,
TRANSACTION_TYPES[groupIndex % TRANSACTION_TYPES.length]
)
.timestamp(timestamp)
.duration(duration)
.outcome(outcome);
})
)
);
return events;
}
);
});
return events;
}
);
})
);
},
};
};

5
packages/kbn-apm-synthtrace/src/scenarios/distributed_trace.ts
View file

@ -11,12 +11,13 @@ import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const transactionName = '240rpm/75% 1000ms';
const successfulTimestamps = range.interval('1s').rate(3);
@ -86,7 +87,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
);
});
return traces;
return withClient(apmEsClient, traces);
},
};
};

5
packages/kbn-apm-synthtrace/src/scenarios/distributed_trace_long.ts
View file

@ -12,12 +12,13 @@ import { apm, ApmFields, DistributedTrace } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const ratePerMinute = 1;
const traceDuration = 1100;
const rootTransactionName = `${ratePerMinute}rpm / ${traceDuration}ms`;
@ -122,7 +123,7 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
}).getTransaction();
});
return traces;
return withClient(apmEsClient, traces);
},
};
};

14
packages/kbn-apm-synthtrace/src/scenarios/high_throughput.ts
View file

@ -10,6 +10,7 @@ import { random } from 'lodash';
import { apm, Instance, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -18,7 +19,7 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
const services = ['web', 'order-processing', 'api-backend'];
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const successfulTimestamps = range.interval('1s');
const instances = services.map((service, index) =>
@ -95,10 +96,13 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
return successfulTraceEvents;
};
return logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }, index) => instanceSpans(instance, url, index))
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }, index) => instanceSpans(instance, url, index))
)
);
},
};

161
packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts Normal file
View file

@ -0,0 +1,161 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import {
LogDocument,
log,
generateShortId,
generateLongId,
apm,
Instance,
} from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { withClient } from '../lib/utils/with_client';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<LogDocument> = async (runOptions) => {
return {
generate: ({ range, clients: { logsEsClient, apmEsClient } }) => {
const { numServices = 3 } = runOptions.scenarioOpts || {};
const { logger } = runOptions;
// Logs Data logic
const MESSAGE_LOG_LEVELS = [
{ message: 'A simple log', level: 'info' },
{ message: 'Yet another debug log', level: 'debug' },
{ message: 'Error with certificate: "ca_trusted_fingerprint"', level: 'error' },
];
const CLOUD_PROVIDERS = ['gcp', 'aws', 'azure'];
const CLOUD_REGION = ['eu-central-1', 'us-east-1', 'area-51'];
const CLUSTER = [
{ clusterId: generateShortId(), clusterName: 'synth-cluster-1' },
{ clusterId: generateShortId(), clusterName: 'synth-cluster-2' },
{ clusterId: generateShortId(), clusterName: 'synth-cluster-3' },
];
const SERVICE_NAMES = Array(3)
.fill(null)
.map((_, idx) => `synth-service-${idx}`);
const logs = range
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(20)
.fill(0)
.map(() => {
const index = Math.floor(Math.random() * 3);
return log
.create()
.message(MESSAGE_LOG_LEVELS[index].message)
.logLevel(MESSAGE_LOG_LEVELS[index].level)
.service(SERVICE_NAMES[index])
.defaults({
'trace.id': generateShortId(),
'agent.name': 'synth-agent',
'orchestrator.cluster.name': CLUSTER[index].clusterName,
'orchestrator.cluster.id': CLUSTER[index].clusterId,
'orchestrator.resource.id': generateShortId(),
'cloud.provider': CLOUD_PROVIDERS[Math.floor(Math.random() * 3)],
'cloud.region': CLOUD_REGION[index],
'cloud.availability_zone': `${CLOUD_REGION[index]}a`,
'cloud.project.id': generateShortId(),
'cloud.instance.id': generateShortId(),
'log.file.path': `/logs/${generateLongId()}/error.txt`,
})
.timestamp(timestamp);
});
});
// APM Simple Trace
const transactionName = '240rpm/75% 1000ms';
const successfulTimestamps = range.interval('1m').rate(180);
const failedTimestamps = range.interval('1m').rate(180);
const instances = [...Array(numServices).keys()].map((index) =>
apm
.service({ name: SERVICE_NAMES[index], environment: ENVIRONMENT, agentName: 'go' })
.instance('instance')
);
const instanceSpans = (instance: Instance) => {
const successfulTraceEvents = successfulTimestamps.generator((timestamp) =>
instance
.transaction({ transactionName })
.timestamp(timestamp)
.duration(1000)
.success()
.children(
instance
.span({
spanName: 'GET apm-*/_search',
spanType: 'db',
spanSubtype: 'elasticsearch',
})
.duration(1000)
.success()
.destination('elasticsearch')
.timestamp(timestamp),
instance
.span({ spanName: 'custom_operation', spanType: 'custom' })
.duration(100)
.success()
.timestamp(timestamp)
)
);
const failedTraceEvents = failedTimestamps.generator((timestamp) =>
instance
.transaction({ transactionName })
.timestamp(timestamp)
.duration(1000)
.failure()
.errors(
instance
.error({ message: '[ResponseError] index_not_found_exception' })
.timestamp(timestamp + 50)
)
);
const metricsets = range
.interval('30s')
.rate(1)
.generator((timestamp) =>
instance
.appMetrics({
'system.memory.actual.free': 800,
'system.memory.total': 1000,
'system.cpu.total.norm.pct': 0.6,
'system.process.cpu.total.norm.pct': 0.7,
})
.timestamp(timestamp)
);
return [successfulTraceEvents, failedTraceEvents, metricsets];
};
return [
withClient(
logsEsClient,
logger.perf('generating_logs', () => logs)
),
withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances.flatMap((instance) => instanceSpans(instance))
)
),
];
},
};
};
export default scenario;

14
packages/kbn-apm-synthtrace/src/scenarios/low_throughput.ts
View file

@ -10,6 +10,7 @@ import { ApmFields, Instance, apm } from '@kbn/apm-synthtrace-client';
import { random } from 'lodash';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -18,7 +19,7 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
const services = ['web', 'order-processing', 'api-backend'];
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const successfulTimestamps = range.ratePerMinute(60);
const instances = services.map((serviceName, index) =>
@ -78,10 +79,13 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
return successfulTraceEvents;
};
return logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }, index) => instanceSpans(instance, url, index))
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }, index) => instanceSpans(instance, url, index))
)
);
},
};

14
packages/kbn-apm-synthtrace/src/scenarios/many_services.ts
View file

@ -10,6 +10,7 @@ import { ApmFields, apm, Instance } from '@kbn/apm-synthtrace-client';
import { flatten, random } from 'lodash';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -25,7 +26,7 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
};
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const successfulTimestamps = range.ratePerMinute(180);
const instances = flatten(
@ -95,10 +96,13 @@ const scenario: Scenario<ApmFields> = async ({ logger }) => {
return successfulTraceEvents;
};
return logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }) => instanceSpans(instance, url))
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) => urls.map((url) => ({ instance, url })))
.map(({ instance, url }) => instanceSpans(instance, url))
)
);
},
};

22
packages/kbn-apm-synthtrace/src/scenarios/many_transactions_per_service.ts
View file

@ -8,6 +8,7 @@
import { ApmFields, apm, Instance } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -17,7 +18,7 @@ const scenario: Scenario<ApmFields> = async (runOptions) => {
const numTransactions = 100;
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const urls = ['GET /order', 'POST /basket', 'DELETE /basket', 'GET /products'];
const successfulTimestamps = range.ratePerMinute(180);
@ -68,14 +69,17 @@ const scenario: Scenario<ApmFields> = async (runOptions) => {
return [successfulTraceEvents, failedTraceEvents, metricsets];
};
return logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) =>
transactionNames.map((transactionName) => ({ instance, transactionName }))
)
.flatMap(({ instance, transactionName }, index) =>
instanceSpans(instance, transactionName)
)
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances
.flatMap((instance) =>
transactionNames.map((transactionName) => ({ instance, transactionName }))
)
.flatMap(({ instance, transactionName }, index) =>
instanceSpans(instance, transactionName)
)
)
);
},
};

7
packages/kbn-apm-synthtrace/src/scenarios/mobile.ts
View file

@ -15,6 +15,7 @@ import type {
} from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -328,7 +329,7 @@ const scenario: Scenario<ApmFields> = async ({ scenarioOpts, logger }) => {
const { numDevices = 10 } = scenarioOpts || {};
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const androidDevices = [...Array(numDevices).keys()].map((index) => {
const deviceMetadata = ANDROID_DEVICES[randomInt(ANDROID_DEVICES.length)];
const geoNetwork = GEO_AND_NETWORK[randomInt(GEO_AND_NETWORK.length)];
@ -444,13 +445,13 @@ const scenario: Scenario<ApmFields> = async ({ scenarioOpts, logger }) => {
);
};
return [
return withClient(apmEsClient, [
...androidDevices.flatMap((device) => [
sessionTransactions(device),
appLaunchMetrics(device),
]),
...iOSDevices.map((device) => sessionTransactions(device)),
];
]);
},
};
};

69
packages/kbn-apm-synthtrace/src/scenarios/other_bucket_group.ts
View file

@ -9,6 +9,7 @@ import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { range as lodashRange } from 'lodash';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENTS = ['production', 'development'].map((env) =>
getSynthtraceEnvironment(__filename, env)
@ -18,7 +19,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const { services: numServices = 10, txGroups: numTxGroups = 10 } = scenarioOpts ?? {};
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const TRANSACTION_TYPES = ['request'];
const MIN_DURATION = 10;
@ -46,41 +47,47 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
'_other',
];
return range
.interval('1m')
.rate(1)
.generator((timestamp, timestampIndex) => {
return logger.perf(
'generate_events_for_timestamp ' + new Date(timestamp).toISOString(),
() => {
const events = instances.flatMap((instance) =>
transactionGroupRange.flatMap((groupId, groupIndex) => {
const duration = Math.round(
(timestampIndex % MAX_BUCKETS) * BUCKET_SIZE + MIN_DURATION
);
return withClient(
apmEsClient,
range
.interval('1m')
.rate(1)
.generator((timestamp, timestampIndex) => {
return logger.perf(
'generate_events_for_timestamp ' + new Date(timestamp).toISOString(),
() => {
const events = instances.flatMap((instance) =>
transactionGroupRange.flatMap((groupId, groupIndex) => {
const duration = Math.round(
(timestampIndex % MAX_BUCKETS) * BUCKET_SIZE + MIN_DURATION
);
if (groupId === '_other') {
return instance
.transaction(groupId)
.timestamp(timestamp)
.duration(duration)
.defaults({
'transaction.aggregation.overflow_count': 10,
});
}
if (groupId === '_other') {
return instance
.transaction(groupId)
.transaction(
groupId,
TRANSACTION_TYPES[groupIndex % TRANSACTION_TYPES.length]
)
.timestamp(timestamp)
.duration(duration)
.defaults({
'transaction.aggregation.overflow_count': 10,
});
}
.success();
})
);
return instance
.transaction(groupId, TRANSACTION_TYPES[groupIndex % TRANSACTION_TYPES.length])
.timestamp(timestamp)
.duration(duration)
.success();
})
);
return events;
}
);
});
return events;
}
);
})
);
},
};
};

116
packages/kbn-apm-synthtrace/src/scenarios/service_map.ts
View file

@ -10,67 +10,71 @@ import { ApmFields, serviceMap } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const environment = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
return {
generate: ({ range }) => {
return range
.interval('1s')
.rate(3)
.generator(
serviceMap({
services: [
{ 'frontend-rum': 'rum-js' },
{ 'frontend-node': 'nodejs' },
{ advertService: 'java' },
{ checkoutService: 'go' },
{ cartService: 'dotnet' },
{ paymentService: 'nodejs' },
{ productCatalogService: 'go' },
],
environment,
definePaths([rum, node, adv, chk, cart, pay, prod]) {
return [
[
[rum, 'fetchAd'],
[node, 'GET /nodejs/adTag'],
[adv, 'APIRestController#getAd'],
['elasticsearch', 'GET ad-*/_search'],
],
[
[rum, 'AddToCart'],
[node, 'POST /nodejs/addToCart'],
[cart, 'POST /dotnet/reserveProduct'],
['redis', 'DECR inventory:i012345:stock'],
],
[
[rum, 'Checkout'],
[node, 'POST /nodejs/placeOrder'],
[chk, 'POST /go/placeOrder'],
[pay, 'POST /nodejs/processPayment'],
],
[
[chk, 'POST /go/clearCart'],
[cart, 'PUT /dotnet/cart/c12345/reset'],
['redis', 'INCR inventory:i012345:stock'],
],
[
[rum, 'ProductDashboard'],
[node, 'GET /nodejs/products'],
[prod, 'GET /go/product-catalog'],
['elasticsearch', 'GET product-*/_search'],
],
[
[chk, 'PUT /go/update-inventory'],
[prod, 'PUT /go/product/i012345'],
],
[pay],
];
},
})
);
generate: ({ range, clients: { apmEsClient } }) => {
return withClient(
apmEsClient,
range
.interval('1s')
.rate(3)
.generator(
serviceMap({
services: [
{ 'frontend-rum': 'rum-js' },
{ 'frontend-node': 'nodejs' },
{ advertService: 'java' },
{ checkoutService: 'go' },
{ cartService: 'dotnet' },
{ paymentService: 'nodejs' },
{ productCatalogService: 'go' },
],
environment,
definePaths([rum, node, adv, chk, cart, pay, prod]) {
return [
[
[rum, 'fetchAd'],
[node, 'GET /nodejs/adTag'],
[adv, 'APIRestController#getAd'],
['elasticsearch', 'GET ad-*/_search'],
],
[
[rum, 'AddToCart'],
[node, 'POST /nodejs/addToCart'],
[cart, 'POST /dotnet/reserveProduct'],
['redis', 'DECR inventory:i012345:stock'],
],
[
[rum, 'Checkout'],
[node, 'POST /nodejs/placeOrder'],
[chk, 'POST /go/placeOrder'],
[pay, 'POST /nodejs/processPayment'],
],
[
[chk, 'POST /go/clearCart'],
[cart, 'PUT /dotnet/cart/c12345/reset'],
['redis', 'INCR inventory:i012345:stock'],
],
[
[rum, 'ProductDashboard'],
[node, 'GET /nodejs/products'],
[prod, 'GET /go/product-catalog'],
['elasticsearch', 'GET product-*/_search'],
],
[
[chk, 'PUT /go/update-inventory'],
[prod, 'PUT /go/product/i012345'],
],
[pay],
];
},
})
)
);
},
};
};

61
packages/kbn-apm-synthtrace/src/scenarios/service_map_oom.ts
View file

@ -10,18 +10,18 @@ import { ApmFields, httpExitSpan } from '@kbn/apm-synthtrace-client';
import { service } from '@kbn/apm-synthtrace-client/src/lib/apm/service';
import { Transaction } from '@kbn/apm-synthtrace-client/src/lib/apm/transaction';
import { Scenario } from '../cli/scenario';
import { RunOptions } from '../cli/utils/parse_run_cli_flags';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const environment = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
const scenario: Scenario<ApmFields> = async () => {
const numServices = 500;
const tracesPerMinute = 10;
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const services = new Array(numServices)
.fill(undefined)
.map((_, idx) => {
@ -29,34 +29,37 @@ const scenario: Scenario<ApmFields> = async (runOptions: RunOptions) => {
})
.reverse();
return range.ratePerMinute(tracesPerMinute).generator((timestamp) => {
const rootTransaction = services.reduce((prev, currentService) => {
const tx = currentService
.transaction(`GET /my/function`, 'request')
.timestamp(timestamp)
.duration(1000)
.children(
...(prev
? [
currentService
.span(
httpExitSpan({
spanName: `exit-span-${currentService.fields['service.name']}`,
destinationUrl: `http://address-to-exit-span-${currentService.fields['service.name']}`,
})
)
.timestamp(timestamp)
.duration(1000)
.children(prev),
]
: [])
);
return withClient(
apmEsClient,
range.ratePerMinute(tracesPerMinute).generator((timestamp) => {
const rootTransaction = services.reduce((prev, currentService) => {
const tx = currentService
.transaction(`GET /my/function`, 'request')
.timestamp(timestamp)
.duration(1000)
.children(
...(prev
? [
currentService
.span(
httpExitSpan({
spanName: `exit-span-${currentService.fields['service.name']}`,
destinationUrl: `http://address-to-exit-span-${currentService.fields['service.name']}`,
})
)
.timestamp(timestamp)
.duration(1000)
.children(prev),
]
: [])
);
return tx;
}, undefined as Transaction | undefined);
return tx;
}, undefined as Transaction | undefined);
return rootTransaction!;
});
return rootTransaction!;
})
);
},
};
};

9
packages/kbn-apm-synthtrace/src/scenarios/services_without_transactions.ts
View file

@ -8,12 +8,13 @@
import { apm, ApmFields } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
const scenario: Scenario<ApmFields> = async () => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const withTx = apm
.service('service-with-transactions', ENVIRONMENT, 'java')
.instance('instance');
@ -26,7 +27,7 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
.service('service-with-app-metrics-only', ENVIRONMENT, 'java')
.instance('instance');
return range
const data = range
.interval('1m')
.rate(1)
.generator((timestamp) => {
@ -45,6 +46,8 @@ const scenario: Scenario<ApmFields> = async ({ logger, scenarioOpts }) => {
.timestamp(timestamp),
];
});
return withClient(apmEsClient, data);
},
};
};

74
packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { LogDocument, log, generateShortId, generateLongId } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { withClient } from '../lib/utils/with_client';
const scenario: Scenario<LogDocument> = async (runOptions) => {
return {
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;
// Logs Data logic
const MESSAGE_LOG_LEVELS = [
{ message: 'A simple log', level: 'info' },
{ message: 'Yet another debug log', level: 'debug' },
{ message: 'Error with certificate: "ca_trusted_fingerprint"', level: 'error' },
];
const CLOUD_PROVIDERS = ['gcp', 'aws', 'azure'];
const CLOUD_REGION = ['eu-central-1', 'us-east-1', 'area-51'];
const CLUSTER = [
{ clusterId: generateShortId(), clusterName: 'synth-cluster-1' },
{ clusterId: generateShortId(), clusterName: 'synth-cluster-2' },
{ clusterId: generateShortId(), clusterName: 'synth-cluster-3' },
];
const SERVICE_NAMES = Array(3)
.fill(null)
.map((_, idx) => `synth-service-${idx}`);
const logs = range
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(20)
.fill(0)
.map(() => {
const index = Math.floor(Math.random() * 3);
return log
.create()
.message(MESSAGE_LOG_LEVELS[index].message)
.logLevel(MESSAGE_LOG_LEVELS[index].level)
.service(SERVICE_NAMES[index])
.defaults({
'trace.id': generateShortId(),
'agent.name': 'synth-agent',
'orchestrator.cluster.name': CLUSTER[index].clusterName,
'orchestrator.cluster.id': CLUSTER[index].clusterId,
'orchestrator.resource.id': generateShortId(),
'cloud.provider': CLOUD_PROVIDERS[Math.floor(Math.random() * 3)],
'cloud.region': CLOUD_REGION[index],
'cloud.availability_zone': `${CLOUD_REGION[index]}a`,
'cloud.project.id': generateShortId(),
'cloud.instance.id': generateShortId(),
'log.file.path': `/logs/${generateLongId()}/error.txt`,
})
.timestamp(timestamp);
});
});
return withClient(
logsEsClient,
logger.perf('generating_logs', () => logs)
);
},
};
};
export default scenario;

10
packages/kbn-apm-synthtrace/src/scenarios/simple_trace.ts
View file

@ -8,6 +8,7 @@
import { ApmFields, apm, Instance } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -16,7 +17,7 @@ const scenario: Scenario<ApmFields> = async (runOptions) => {
const { numServices = 3 } = runOptions.scenarioOpts || {};
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const transactionName = '240rpm/75% 1000ms';
const successfulTimestamps = range.interval('1m').rate(180);
@ -83,8 +84,11 @@ const scenario: Scenario<ApmFields> = async (runOptions) => {
return [successfulTraceEvents, failedTraceEvents, metricsets];
};
return logger.perf('generating_apm_events', () =>
instances.flatMap((instance) => instanceSpans(instance))
return withClient(
apmEsClient,
logger.perf('generating_apm_events', () =>
instances.flatMap((instance) => instanceSpans(instance))
)
);
},
};

8
packages/kbn-apm-synthtrace/src/scenarios/span_links.ts
View file

@ -11,6 +11,7 @@ import { Readable } from 'stream';
import { apm, ApmFields, generateLongId, generateShortId } from '@kbn/apm-synthtrace-client';
import { Scenario } from '../cli/scenario';
import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment';
import { withClient } from '../lib/utils/with_client';
const ENVIRONMENT = getSynthtraceEnvironment(__filename);
@ -32,7 +33,7 @@ function getSpanLinksFromEvents(events: ApmFields[]) {
const scenario: Scenario<ApmFields> = async () => {
return {
generate: ({ range }) => {
generate: ({ range, clients: { apmEsClient } }) => {
const producerInternalOnlyInstance = apm
.service({ name: 'producer-internal-only', environment: ENVIRONMENT, agentName: 'go' })
@ -111,8 +112,9 @@ const scenario: Scenario<ApmFields> = async () => {
);
});
return Readable.from(
Array.from(producerInternalOnlyEvents).concat(Array.from(consumerEvents))
return withClient(
apmEsClient,
Readable.from(Array.from(producerInternalOnlyEvents).concat(Array.from(consumerEvents)))
);
},
};

32
x-pack/test/functional/apps/observability_log_explorer/app.ts
View file

@ -5,11 +5,14 @@
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { log, timerange } from '@kbn/apm-synthtrace-client';
import { FtrProviderContext } from './config';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'navigationalSearch', 'observabilityLogExplorer']);
const testSubjects = getService('testSubjects');
const synthtrace = getService('logSynthtraceEsClient');
const dataGrid = getService('dataGrid');
describe('Application', () => {
it('is shown in the global search', async () => {
@ -24,5 +27,32 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.observabilityLogExplorer.navigateTo();
await testSubjects.existOrFail('observability-nav-observability-log-explorer-explorer');
});
it('should load logs', async () => {
const from = '2023-08-03T10:24:14.035Z';
const to = '2023-08-03T10:24:14.091Z';
const COUNT = 5;
await synthtrace.index(generateLogsData({ from, to, count: COUNT }));
await PageObjects.observabilityLogExplorer.navigateTo();
const docCount = await dataGrid.getDocCount();
expect(docCount).to.be(COUNT);
await synthtrace.clean();
});
});
}
function generateLogsData({ from, to, count = 1 }: { from: string; to: string; count: number }) {
const range = timerange(from, to);
return range
.interval('1m')
.rate(1)
.generator((timestamp) =>
Array(count)
.fill(0)
.map(() => {
return log.create().message('A sample log').timestamp(timestamp);
})
);
}

2
x-pack/test/functional/apps/observability_log_explorer/columns_selection.ts
View file

@ -6,7 +6,7 @@
*/
import expect from '@kbn/expect';
import rison from '@kbn/rison';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
const defaultLogColumns = ['@timestamp', 'service.name', 'host.name', 'message'];

29
x-pack/test/functional/apps/observability_log_explorer/config.ts
View file

@ -5,13 +5,38 @@
* 2.0.
*/
import { FtrConfigProviderContext } from '@kbn/test';
import { FtrConfigProviderContext, GenericFtrProviderContext } from '@kbn/test';
import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { FtrProviderContext as InheritedFtrProviderContext } from '../../ftr_provider_context';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
export default async function createTestConfig({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../../config.base.js'));
const services = functionalConfig.get('services');
const pageObjects = functionalConfig.get('pageObjects');
return {
...functionalConfig.getAll(),
testFiles: [require.resolve('.')],
services: {
...services,
logSynthtraceEsClient: (context: InheritedFtrProviderContext) => {
return new LogsSynthtraceEsClient({
client: context.getService('es'),
logger: createLogger(LogLevel.info),
refreshAfterIndex: true,
});
},
},
pageObjects,
};
}
export type CreateTestConfig = Awaited<ReturnType<typeof createTestConfig>>;
export type ObsLogExplorerServices = CreateTestConfig['services'];
export type ObsLogExplorerPageObject = CreateTestConfig['pageObjects'];
export type FtrProviderContext = GenericFtrProviderContext<
ObsLogExplorerServices,
ObsLogExplorerPageObject
>;

2
x-pack/test/functional/apps/observability_log_explorer/dataset_selection_state.ts
View file

@ -6,7 +6,7 @@
*/
import expect from '@kbn/expect';
import rison from '@kbn/rison';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

145
x-pack/test/functional/apps/observability_log_explorer/dataset_selector.ts
View file

@ -5,7 +5,8 @@
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
import { WebElementWrapper } from '../../../../../test/functional/services/lib/web_element_wrapper';
const initialPackageMap = {
apache: 'Apache HTTP Server',
@ -101,7 +102,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
it('should display an empty prompt for no integrations', async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(menuEntries.length).to.be(0);
await PageObjects.observabilityLogExplorer.assertListStatusEmptyPromptExistsWithTitle(
@ -161,7 +164,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const uncategorizedEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(uncategorizedEntries.length).to.be(0);
@ -313,7 +318,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const [panelTitleNode, integrationDatasetEntries] =
await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -335,7 +340,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Apache HTTP Server');
});
@ -345,7 +352,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be('access');
expect(await menuEntries[1].getVisibleText()).to.be('error');
@ -356,7 +365,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be('error');
expect(await menuEntries[1].getVisibleText()).to.be('access');
@ -367,7 +378,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be('access');
expect(await menuEntries[1].getVisibleText()).to.be('error');
@ -383,7 +396,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Apache HTTP Server');
});
@ -391,7 +406,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be('access');
expect(await menuEntries[1].getVisibleText()).to.be('error');
@ -402,7 +419,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(menuEntries.length).to.be(1);
expect(await menuEntries[0].getVisibleText()).to.be('error');
@ -418,7 +437,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Apache HTTP Server');
});
@ -426,7 +447,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be('access');
menuEntries[0].click();
@ -452,14 +475,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.observabilityLogExplorer.openDatasetSelector();
await PageObjects.observabilityLogExplorer
.getUncategorizedTab()
.then((tab) => tab.click());
.then((tab: WebElementWrapper) => tab.click());
});
it('should display a list of available datasets', async () => {
await retry.try(async () => {
const [panelTitleNode, menuEntries] = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -477,7 +500,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Uncategorized');
});
@ -487,7 +512,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedUncategorized[0]);
expect(await menuEntries[1].getVisibleText()).to.be(expectedUncategorized[1]);
@ -499,7 +526,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedUncategorized[2]);
expect(await menuEntries[1].getVisibleText()).to.be(expectedUncategorized[1]);
@ -511,7 +540,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedUncategorized[0]);
expect(await menuEntries[1].getVisibleText()).to.be(expectedUncategorized[1]);
@ -523,7 +554,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Uncategorized');
});
@ -531,7 +564,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedUncategorized[0]);
expect(await menuEntries[1].getVisibleText()).to.be(expectedUncategorized[1]);
@ -543,7 +578,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(menuEntries.length).to.be(1);
expect(await menuEntries[0].getVisibleText()).to.be('retail');
@ -554,7 +591,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(await panelTitleNode.getVisibleText()).to.be('Uncategorized');
});
@ -562,7 +601,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getUncategorizedContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedUncategorized[0]);
menuEntries[0].click();
@ -585,14 +626,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
beforeEach(async () => {
await browser.refresh();
await PageObjects.observabilityLogExplorer.openDatasetSelector();
await PageObjects.observabilityLogExplorer.getDataViewsTab().then((tab) => tab.click());
await PageObjects.observabilityLogExplorer
.getDataViewsTab()
.then((tab: WebElementWrapper) => tab.click());
});
it('should display a list of available data views', async () => {
await retry.try(async () => {
const [panelTitleNode, menuEntries] = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -614,7 +657,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(
await PageObjects.observabilityLogExplorer.getDataViewsContextMenuTitle(
@ -628,7 +673,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(sortedExpectedDataViews[2]);
expect(await menuEntries[1].getVisibleText()).to.be(sortedExpectedDataViews[1]);
@ -640,7 +687,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(sortedExpectedDataViews[0]);
expect(await menuEntries[1].getVisibleText()).to.be(sortedExpectedDataViews[1]);
@ -652,7 +701,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(
await PageObjects.observabilityLogExplorer.getDataViewsContextMenuTitle(
@ -664,7 +715,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[0].getVisibleText()).to.be(expectedDataViews[0]);
expect(await menuEntries[1].getVisibleText()).to.be(expectedDataViews[1]);
@ -676,7 +729,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(menuEntries.length).to.be(2);
expect(await menuEntries[0].getVisibleText()).to.be('logs-*');
@ -688,7 +743,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
expect(
await PageObjects.observabilityLogExplorer.getDataViewsContextMenuTitle(
@ -700,7 +757,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getDataViewsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(await menuEntries[2].getVisibleText()).to.be(expectedDataViews[2]);
menuEntries[2].click();
@ -733,7 +792,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const [panelTitleNode, menuEntries] = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -751,7 +810,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const [panelTitleNode, menuEntries] = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -803,7 +862,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const [panelTitleNode, menuEntries] = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) =>
.then((menu: WebElementWrapper) =>
Promise.all([
PageObjects.observabilityLogExplorer.getPanelTitle(menu),
PageObjects.observabilityLogExplorer.getPanelEntries(menu),
@ -820,7 +879,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
expect(menuEntries.length).to.be(1);
expect(await menuEntries[0].getVisibleText()).to.be('error');
@ -829,7 +890,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
// Navigate back to integrations
const panelTitleNode = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelTitle(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelTitle(menu)
);
panelTitleNode.click();
await retry.try(async () => {
@ -846,7 +909,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.try(async () => {
const menuEntries = await PageObjects.observabilityLogExplorer
.getIntegrationsContextMenu()
.then((menu) => PageObjects.observabilityLogExplorer.getPanelEntries(menu));
.then((menu: WebElementWrapper) =>
PageObjects.observabilityLogExplorer.getPanelEntries(menu)
);
const searchValue = await PageObjects.observabilityLogExplorer.getSearchFieldValue();
expect(searchValue).to.eql('err');

2
x-pack/test/functional/apps/observability_log_explorer/filter_controls.ts
View file

@ -4,7 +4,7 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');

2
x-pack/test/functional/apps/observability_log_explorer/header_menu.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const browser = getService('browser');

2
x-pack/test/functional/apps/observability_log_explorer/index.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from './config';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Observability Log Explorer', function () {

4
x-pack/test_serverless/functional/services/index.ts
View file

@ -16,6 +16,7 @@ import { SvlCommonScreenshotsProvider } from './svl_common_screenshots';
import { SvlCasesServiceProvider } from '../../api_integration/services/svl_cases';
import { MachineLearningProvider } from './ml';
import { SvlReportingServiceProvider } from './svl_reporting';
import { LogsSynthtraceProvider } from './log';
export const services = {
// deployment agnostic FTR services
@ -31,4 +32,7 @@ export const services = {
svlCases: SvlCasesServiceProvider,
svlMl: MachineLearningProvider,
svlReportingApi: SvlReportingServiceProvider,
// log services
svlLogsSynthtraceClient: LogsSynthtraceProvider,
};

17
x-pack/test_serverless/functional/services/log/index.ts Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { FtrProviderContext } from '../../ftr_provider_context';
export function LogsSynthtraceProvider(context: FtrProviderContext) {
return new LogsSynthtraceEsClient({
client: context.getService('es'),
logger: createLogger(LogLevel.info),
refreshAfterIndex: true,
});
}

34
x-pack/test_serverless/functional/test_suites/observability/observability_log_explorer/app.ts
View file

@ -5,15 +5,19 @@
* 2.0.
*/
import expect from '@kbn/expect';
import { log, timerange } from '@kbn/apm-synthtrace-client';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getPageObjects }: FtrProviderContext) {
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const PageObjects = getPageObjects([
'observabilityLogExplorer',
'svlCommonNavigation',
'svlCommonPage',
]);
const synthtrace = getService('svlLogsSynthtraceClient');
const dataGrid = getService('dataGrid');
describe('Application', () => {
before(async () => {
await PageObjects.svlCommonPage.login();
@ -25,6 +29,7 @@ export default function ({ getPageObjects }: FtrProviderContext) {
it('is shown in the global search', async () => {
await PageObjects.observabilityLogExplorer.navigateTo();
await PageObjects.svlCommonNavigation.search.showSearch();
await PageObjects.svlCommonNavigation.search.searchFor('log explorer');
@ -33,5 +38,32 @@ export default function ({ getPageObjects }: FtrProviderContext) {
await PageObjects.svlCommonNavigation.search.hideSearch();
});
it('should load logs', async () => {
const from = '2023-08-03T10:24:14.035Z';
const to = '2023-08-03T10:24:14.091Z';
const COUNT = 5;
await synthtrace.index(generateLogsData({ from, to, count: COUNT }));
await PageObjects.observabilityLogExplorer.navigateTo();
const docCount = await dataGrid.getDocCount();
expect(docCount).to.be(COUNT);
await synthtrace.clean();
});
});
}
function generateLogsData({ from, to, count = 1 }: { from: string; to: string; count: number }) {
const range = timerange(from, to);
return range
.interval('1m')
.rate(1)
.generator((timestamp) =>
Array(count)
.fill(0)
.map(() => {
return log.create().message('A sample log').timestamp(timestamp);
})
);
}

2
x-pack/test_serverless/tsconfig.json
View file

@ -63,6 +63,8 @@
"@kbn/cloud-security-posture-plugin",
"@kbn/reporting-plugin",
"@kbn/management-settings-ids",
"@kbn/apm-synthtrace",
"@kbn/apm-synthtrace-client",
"@kbn/reporting-export-types-csv-common",
]
}