github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.18] [Synthtrace] Introducing teardown for scenarios (#209739) (#209798)

Browse source 
# Backport

This will backport the following commits from `main` to `8.18`:
- [[Synthtrace] Introducing teardown for scenarios
(#209739)](https://github.com/elastic/kibana/pull/209739)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Yngrid
Coello","email":"yngrid.coello@elastic.co"},"sourceCommit":{"committedDate":"2025-02-05T14:57:50Z","message":"[Synthtrace]
Introducing teardown for scenarios (#209739)\n\n### Background\r\n\r\nIn
some scenarios we need to perform some setup at bootstrap time,
this\r\nsetup could affect following scenarios.\r\nTake for
example\r\n[failed_logs](https://github.com/elastic/kibana/blob/main/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts)\r\nscenario
where we create a pipeline that will do some checks in\r\n`log.level`
property, if we try to run an scenario after that one we\r\nwill enter
into some issues.\r\n\r\n### Changes\r\n\r\nThis PR aims to introduce a
`teardown` setup for scenarios where we\r\ncould undo the changes done
at `bootstrap`
time.","sha":"c56d7ea24a898cbf6943719af1ef37c03250e2ee","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-infra_services","v8.18.0","v9.1.0","v8.19.0"],"title":"[Synthtrace]
Introducing teardown for
scenarios","number":209739,"url":"https://github.com/elastic/kibana/pull/209739","mergeCommit":{"message":"[Synthtrace]
Introducing teardown for scenarios (#209739)\n\n### Background\r\n\r\nIn
some scenarios we need to perform some setup at bootstrap time,
this\r\nsetup could affect following scenarios.\r\nTake for
example\r\n[failed_logs](https://github.com/elastic/kibana/blob/main/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts)\r\nscenario
where we create a pipeline that will do some checks in\r\n`log.level`
property, if we try to run an scenario after that one we\r\nwill enter
into some issues.\r\n\r\n### Changes\r\n\r\nThis PR aims to introduce a
`teardown` setup for scenarios where we\r\ncould undo the changes done
at `bootstrap`
time.","sha":"c56d7ea24a898cbf6943719af1ef37c03250e2ee"}},"sourceBranch":"main","suggestedTargetBranches":["8.18","8.x"],"targetPullRequestStates":[{"branch":"8.18","label":"v8.18.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/209739","number":209739,"mergeCommit":{"message":"[Synthtrace]
Introducing teardown for scenarios (#209739)\n\n### Background\r\n\r\nIn
some scenarios we need to perform some setup at bootstrap time,
this\r\nsetup could affect following scenarios.\r\nTake for
example\r\n[failed_logs](https://github.com/elastic/kibana/blob/main/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts)\r\nscenario
where we create a pipeline that will do some checks in\r\n`log.level`
property, if we try to run an scenario after that one we\r\nwill enter
into some issues.\r\n\r\n### Changes\r\n\r\nThis PR aims to introduce a
`teardown` setup for scenarios where we\r\ncould undo the changes done
at `bootstrap`
time.","sha":"c56d7ea24a898cbf6943719af1ef37c03250e2ee"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

---------

Co-authored-by: Yngrid Coello <yngrid.coello@elastic.co>
This commit is contained in:
Kibana Machine 2025-02-06 04:24:26 +11:00 • committed by GitHub
parent 3d95ce58b4
commit 349a5b7817
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 81 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/kbn-apm-synthtrace/README.md
View file

@ -119,7 +119,7 @@ Scenario files accept 3 arguments, 2 of them optional and 1 mandatory
|-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------|
| `generate` | mandatory | This is the main function responsible for returning the events which will be indexed |
| `bootstrap` | optional | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with |
| `setClient` | optional | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned |
| `teardown` | optional | In case some setup needs to be done, after all data is generated, this function provides access to all available ES Clients to play with |
The following options are supported:

1
packages/kbn-apm-synthtrace/src/cli/scenario.ts
View file

@ -40,4 +40,5 @@ type Generate<TFields> = (options: {
export type Scenario<TFields> = (options: RunOptions & { logger: Logger }) => Promise<{
bootstrap?: (options: EsClients & KibanaClients) => Promise<void>;
generate: Generate<TFields>;
teardown?: (options: EsClients & KibanaClients) => Promise<void>;
}>;

29
packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts
View file

@ -37,7 +37,11 @@ export async function startLiveDataUpload({
} = await bootstrap(runOptions);
const scenario = await getScenario({ file, logger });
const { generate, bootstrap: scenarioBootsrap } = await scenario({ ...runOptions, logger });
const {
generate,
bootstrap: scenarioBootsrap,
teardown: scenarioTearDown,
} = await scenario({ ...runOptions, logger });
if (scenarioBootsrap) {
await scenarioBootsrap({
@ -57,11 +61,26 @@ export async function startLiveDataUpload({
// @ts-expect-error upgrade typescript v4.9.5
const cachedStreams: WeakMap<SynthtraceEsClient, PassThrough> = new WeakMap();
process.on('SIGINT', () => closeStreams());
process.on('SIGTERM', () => closeStreams());
process.on('SIGQUIT', () => closeStreams());
process.on('SIGINT', () => closeStreamsAndTeardown());
process.on('SIGTERM', () => closeStreamsAndTeardown());
process.on('SIGQUIT', () => closeStreamsAndTeardown());
async function closeStreamsAndTeardown() {
if (scenarioTearDown) {
try {
await scenarioTearDown({
apmEsClient,
logsEsClient,
infraEsClient,
syntheticsEsClient,
entitiesEsClient,
entitiesKibanaClient,
});
} catch (error) {
logger.error('Error during scenario teardown', error);
}
}
function closeStreams() {
currentStreams.forEach((stream) => {
stream.end(() => {
process.exit(0);

13
packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts
View file

@ -77,7 +77,7 @@ async function start() {
logger.info(`Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()}`);
const { generate, bootstrap } = await scenario({ ...runOptions, logger });
const { generate, bootstrap, teardown } = await scenario({ ...runOptions, logger });
if (bootstrap) {
await bootstrap({
@ -133,6 +133,17 @@ async function start() {
await Promise.all(promises);
});
if (teardown) {
await teardown({
apmEsClient,
logsEsClient,
infraEsClient,
syntheticsEsClient,
entitiesEsClient,
entitiesKibanaClient,
});
}
}
parentPort!.on('message', (message) => {

20
packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts
View file

@ -52,6 +52,15 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
}
}
async deleteIndexTemplate(name: IndexTemplateName) {
try {
await this.client.indices.deleteIndexTemplate({ name });
this.logger.info(`Index template successfully deleted: ${name}`);
} catch (err) {
this.logger.error(`Index template deletion failed: ${name} - ${err.message}`);
}
}
async createComponentTemplate({
name,
mappings,
@ -150,6 +159,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
}
}
async deleteCustomPipeline(id = LogsCustom) {
try {
this.client.ingest.deletePipeline({
id,
});
this.logger.info(`Custom pipeline deleted: ${id}`);
} catch (err) {
this.logger.error(`Custom pipeline deletion failed: ${id} - ${err.message}`);
}
}
getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) {
return logsPipeline({ includeSerialization });
}

3
packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts
View file

@ -36,6 +36,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
bootstrap: async ({ logsEsClient }) => {
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;

5
packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts
View file

@ -75,6 +75,11 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
},
});
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteComponentTemplate(LogsCustom);
await logsEsClient.deleteCustomPipeline();
if (isLogsDb) await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;

3
packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts
View file

@ -38,6 +38,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
bootstrap: async ({ logsEsClient }) => {
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient, apmEsClient } }) => {
const { numServices = 3 } = runOptions.scenarioOpts || {};
const { logger } = runOptions;

3
packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts
View file

@ -52,6 +52,9 @@ const scenario: Scenario<LogDocument | InfraDocument | ApmFields> = async (runOp
bootstrap: async ({ logsEsClient }) => {
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient, infraEsClient, apmEsClient } }) => {
const {
numSpaces,

3
packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts
View file

@ -70,6 +70,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
bootstrap: async ({ logsEsClient }) => {
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;

3
packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts
View file

@ -78,6 +78,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
await logsEsClient.createIndex('cloud-logs-synth.2-default');
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;

3
packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts
View file

@ -20,6 +20,9 @@ const scenario: Scenario<LogDocument> = async (runOptions) => {
bootstrap: async ({ logsEsClient }) => {
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
},
teardown: async ({ logsEsClient }) => {
await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb);
},
generate: ({ range, clients: { logsEsClient } }) => {
const { logger } = runOptions;