chore(slo): delete by queries (#217108)

## Summary Add safer parameters when deleting by queries, and make all delete by queries async (wait = false). In some cases, I've parallelized the calls
2025-04-24 09:48:58 -04:00 · 2025-04-03 18:36:35 -04:00 · 2025-04-03 18:36:35 -04:00 · 3a63089dc7
commit 3a63089dc7
parent e21739b657
12 changed files with 93 additions and 29 deletions
--- a/x-pack/solutions/observability/plugins/slo/server/services/snapshots/delete_slo.test.ts.snap
+++ b/x-pack/solutions/observability/plugins/slo/server/services/snapshots/delete_slo.test.ts.snap
@ -122,17 +122,20 @@ exports[`DeleteSLO happy path removes all resources associatde to the slo 7`] =
  "calls": Array [
    Array [
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.sli-v3*",
        "query": Object {
          "match": Object {
            "slo.id": "irrelevant",
          },
        },
+        "slices": "auto",
        "wait_for_completion": false,
      },
    ],
    Array [
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.summary-v3*",
        "query": Object {
          "match": Object {
@ -140,6 +143,8 @@ exports[`DeleteSLO happy path removes all resources associatde to the slo 7`] =
          },
        },
        "refresh": true,
+        "slices": "auto",
+        "wait_for_completion": false,
      },
    ],
  ],
--- a/x-pack/solutions/observability/plugins/slo/server/services/snapshots/reset_slo.test.ts.snap
+++ b/x-pack/solutions/observability/plugins/slo/server/services/snapshots/reset_slo.test.ts.snap
@ -69,6 +69,7 @@ exports[`ResetSLO happy path resets all associated resources 5`] = `
  "calls": Array [
    Array [
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.sli-v3*",
        "query": Object {
          "bool": Object {
@ -82,10 +83,12 @@ exports[`ResetSLO happy path resets all associated resources 5`] = `
          },
        },
        "refresh": true,
+        "slices": "auto",
      },
    ],
    Array [
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.summary-v3*",
        "query": Object {
          "bool": Object {
@ -99,6 +102,7 @@ exports[`ResetSLO happy path resets all associated resources 5`] = `
          },
        },
        "refresh": true,
+        "slices": "auto",
      },
    ],
  ],
--- a/x-pack/solutions/observability/plugins/slo/server/services/delete_slo.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/delete_slo.ts
@ -18,6 +18,7 @@ import {
 import { retryTransientEsErrors } from '../utils/retry';
 import { SLORepository } from './slo_repository';
 import { TransformManager } from './transform_manager';
+import { SLODefinition } from '../domain/models';

 export class DeleteSLO {
  constructor(
@ -32,38 +33,49 @@ export class DeleteSLO {
  public async execute(sloId: string): Promise<void> {
    const slo = await this.repository.findById(sloId);

-    const summaryTransformId = getSLOSummaryTransformId(slo.id, slo.revision);
-    await this.summaryTransformManager.stop(summaryTransformId);
-    await this.summaryTransformManager.uninstall(summaryTransformId);
+    await Promise.all([this.deleteSummaryTransform(slo), this.deleteRollupTransform(slo)]);

+    await Promise.all([
+      retryTransientEsErrors(() =>
+        this.scopedClusterClient.asSecondaryAuthUser.ingest.deletePipeline(
+          { id: getSLOPipelineId(slo.id, slo.revision) },
+          { ignore: [404] }
+        )
+      ),
+      retryTransientEsErrors(() =>
+        this.scopedClusterClient.asSecondaryAuthUser.ingest.deletePipeline(
+          { id: getSLOSummaryPipelineId(slo.id, slo.revision) },
+          { ignore: [404] }
+        )
+      ),
+    ]);
+
+    await Promise.all([
+      this.deleteRollupData(slo.id),
+      this.deleteSummaryData(slo.id),
+      this.deleteAssociatedRules(slo.id),
+      this.repository.deleteById(slo.id),
+    ]);
+  }
+
+  private async deleteRollupTransform(slo: SLODefinition) {
    const rollupTransformId = getSLOTransformId(slo.id, slo.revision);
    await this.transformManager.stop(rollupTransformId);
    await this.transformManager.uninstall(rollupTransformId);
+  }

-    await retryTransientEsErrors(() =>
-      this.scopedClusterClient.asSecondaryAuthUser.ingest.deletePipeline(
-        { id: getSLOPipelineId(slo.id, slo.revision) },
-        { ignore: [404] }
-      )
-    );
-
-    await retryTransientEsErrors(() =>
-      this.scopedClusterClient.asSecondaryAuthUser.ingest.deletePipeline(
-        { id: getSLOSummaryPipelineId(slo.id, slo.revision) },
-        { ignore: [404] }
-      )
-    );
-
-    await this.deleteRollupData(slo.id);
-    await this.deleteSummaryData(slo.id);
-    await this.deleteAssociatedRules(slo.id);
-    await this.repository.deleteById(slo.id);
+  private async deleteSummaryTransform(slo: SLODefinition) {
+    const summaryTransformId = getSLOSummaryTransformId(slo.id, slo.revision);
+    await this.summaryTransformManager.stop(summaryTransformId);
+    await this.summaryTransformManager.uninstall(summaryTransformId);
  }

  private async deleteRollupData(sloId: string): Promise<void> {
    await this.esClient.deleteByQuery({
      index: SLI_DESTINATION_INDEX_PATTERN,
      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        match: {
          'slo.id': sloId,
@ -76,6 +88,9 @@ export class DeleteSLO {
    await this.esClient.deleteByQuery({
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      refresh: true,
+      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        match: {
          'slo.id': sloId,
--- a/x-pack/solutions/observability/plugins/slo/server/services/delete_slo_instances.test.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/delete_slo_instances.test.ts
@ -43,6 +43,7 @@ describe('DeleteSLOInstances', () => {
    expect(mockEsClient.deleteByQuery).toHaveBeenCalledTimes(2);
    expect(mockEsClient.deleteByQuery.mock.calls[0][0]).toMatchInlineSnapshot(`
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.sli-v3*",
        "query": Object {
          "bool": Object {
@ -98,11 +99,13 @@ describe('DeleteSLOInstances', () => {
            ],
          },
        },
+        "slices": "auto",
        "wait_for_completion": false,
      }
    `);
    expect(mockEsClient.deleteByQuery.mock.calls[1][0]).toMatchInlineSnapshot(`
      Object {
+        "conflicts": "proceed",
        "index": ".slo-observability.summary-v3*",
        "query": Object {
          "bool": Object {
@ -159,6 +162,8 @@ describe('DeleteSLOInstances', () => {
          },
        },
        "refresh": true,
+        "slices": "auto",
+        "wait_for_completion": false,
      }
    `);
  });
--- a/x-pack/solutions/observability/plugins/slo/server/services/delete_slo_instances.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/delete_slo_instances.ts
@ -24,8 +24,7 @@ export class DeleteSLOInstances {
      throw new IllegalArgumentError("Cannot delete an SLO instance '*'");
    }

-    await this.deleteRollupData(params.list);
-    await this.deleteSummaryData(params.list);
+    await Promise.all([this.deleteRollupData(params.list), this.deleteSummaryData(params.list)]);
  }

  // Delete rollup data when excluding rollup data is not explicitly requested
@ -33,6 +32,8 @@ export class DeleteSLOInstances {
    await this.esClient.deleteByQuery({
      index: SLI_DESTINATION_INDEX_PATTERN,
      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          should: list
@ -54,6 +55,9 @@ export class DeleteSLOInstances {
    await this.esClient.deleteByQuery({
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      refresh: true,
+      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          should: list.map((item) => ({
--- a/x-pack/solutions/observability/plugins/slo/server/services/reset_slo.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/reset_slo.ts
@ -126,6 +126,8 @@ export class ResetSLO {
    await this.esClient.deleteByQuery({
      index: SLI_DESTINATION_INDEX_PATTERN,
      refresh: true,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          filter: [{ term: { 'slo.id': sloId } }],
@ -144,6 +146,8 @@ export class ResetSLO {
    await this.esClient.deleteByQuery({
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      refresh: true,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          filter: [{ term: { 'slo.id': sloId } }],
--- a/x-pack/solutions/observability/plugins/slo/server/services/summary_search_client/snapshots/summary_search_client.test.ts.snap
+++ b/x-pack/solutions/observability/plugins/slo/server/services/summary_search_client/snapshots/summary_search_client.test.ts.snap
@ -3,6 +3,7 @@
 exports[`Summary Search Client returns the summary documents without duplicate temporary summary documents 1`] = `
 Array [
  Object {
+    "conflicts": "proceed",
    "index": ".slo-observability.summary-v3*",
    "query": Object {
      "bool": Object {
@ -25,6 +26,7 @@ Array [
        ],
      },
    },
+    "slices": "auto",
    "wait_for_completion": false,
  },
 ]
--- a/x-pack/solutions/observability/plugins/slo/server/services/summary_search_client/summary_search_client.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/summary_search_client/summary_search_client.ts
@ -176,6 +176,8 @@ export class DefaultSummarySearchClient implements SummarySearchClient {
    await this.esClient.deleteByQuery({
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          filter: [{ terms: { 'slo.id': summarySloIds } }, { term: { isTempDoc: true } }],
--- a/x-pack/solutions/observability/plugins/slo/server/services/tasks/orphan_summary_cleanup_task.test.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/tasks/orphan_summary_cleanup_task.test.ts
@ -69,6 +69,8 @@ describe('SloSummaryCleanupTask', () => {
          ],
        },
      },
+      conflicts: 'proceed',
+      slices: 'auto',
      wait_for_completion: false,
    });
  });
@ -99,6 +101,8 @@ describe('SloSummaryCleanupTask', () => {
    expect(task.fetchSloSummariesIds).toHaveBeenCalledTimes(1);
    expect(esClient.deleteByQuery).toHaveBeenCalledTimes(1);
    expect(esClient.deleteByQuery).toHaveBeenNthCalledWith(1, {
+      conflicts: 'proceed',
+      slices: 'auto',
      wait_for_completion: false,
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      query: {
@ -159,6 +163,9 @@ describe('SloSummaryCleanupTask', () => {

    expect(esClient.deleteByQuery).toHaveBeenNthCalledWith(1, {
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
+      conflicts: 'proceed',
+      slices: 'auto',
+      wait_for_completion: false,
      query: {
        bool: {
          should: getDeleteQueryFilter([
@ -169,12 +176,13 @@ describe('SloSummaryCleanupTask', () => {
          ]),
        },
      },
-      wait_for_completion: false,
    });

    expect(esClient.deleteByQuery).toHaveBeenLastCalledWith({
-      wait_for_completion: false,
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
+      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          should: getDeleteQueryFilter([
@ -231,6 +239,8 @@ describe('SloSummaryCleanupTask', () => {
    expect(task.fetchSloSummariesIds).toHaveBeenCalledTimes(2);
    expect(esClient.deleteByQuery).toHaveBeenCalledTimes(2);
    expect(esClient.deleteByQuery).toHaveBeenNthCalledWith(1, {
+      conflicts: 'proceed',
+      slices: 'auto',
      wait_for_completion: false,
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      query: {
@ -245,6 +255,8 @@ describe('SloSummaryCleanupTask', () => {
      },
    });
    expect(esClient.deleteByQuery).toHaveBeenLastCalledWith({
+      conflicts: 'proceed',
+      slices: 'auto',
      wait_for_completion: false,
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      query: {
@ -299,6 +311,8 @@ describe('SloSummaryCleanupTask', () => {
    expect(esClient.deleteByQuery).toHaveBeenCalledTimes(2);

    expect(esClient.deleteByQuery).toHaveBeenNthCalledWith(1, {
+      conflicts: 'proceed',
+      slices: 'auto',
      wait_for_completion: false,
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      query: {
--- a/x-pack/solutions/observability/plugins/slo/server/services/tasks/orphan_summary_cleanup_task.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/tasks/orphan_summary_cleanup_task.ts
@ -105,8 +105,10 @@ export class SloOrphanSummaryCleanupTask {
          );

          await this.esClient.deleteByQuery({
-            wait_for_completion: false,
            index: SUMMARY_DESTINATION_INDEX_PATTERN,
+            wait_for_completion: false,
+            conflicts: 'proceed',
+            slices: 'auto',
            query: {
              bool: {
                should: getDeleteQueryFilter(sloSummaryIdsToDelete.sort()),
--- a/x-pack/solutions/observability/plugins/slo/server/services/transform_manager.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/transform_manager.ts
@ -151,7 +151,7 @@ export class DefaultTransformManager implements TransformManager {
    }
  }

-  async scheduleNowTransform(transformId: TransformId) {
+  private async scheduleNowTransform(transformId: TransformId) {
    this.scopedClusterClient.asSecondaryAuthUser.transform
      .scheduleNowTransform({ transform_id: transformId })
      .then(() => {
--- a/x-pack/solutions/observability/plugins/slo/server/services/update_slo.ts
+++ b/x-pack/solutions/observability/plugins/slo/server/services/update_slo.ts
@ -232,14 +232,18 @@ export class UpdateSLO {
      // Worst case we keep rolling up data for the previous revision number.
    }

-    await this.deleteRollupData(originalSlo.id, originalSlo.revision);
-    await this.deleteSummaryData(originalSlo.id, originalSlo.revision);
+    await Promise.all([
+      this.deleteRollupData(originalSlo.id, originalSlo.revision),
+      this.deleteSummaryData(originalSlo.id, originalSlo.revision),
+    ]);
  }

  private async deleteRollupData(sloId: string, sloRevision: number): Promise<void> {
    await this.esClient.deleteByQuery({
      index: SLI_DESTINATION_INDEX_PATTERN,
      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          filter: [{ term: { 'slo.id': sloId } }, { term: { 'slo.revision': sloRevision } }],
@ -252,6 +256,9 @@ export class UpdateSLO {
    await this.esClient.deleteByQuery({
      index: SUMMARY_DESTINATION_INDEX_PATTERN,
      refresh: true,
+      wait_for_completion: false,
+      conflicts: 'proceed',
+      slices: 'auto',
      query: {
        bool: {
          filter: [{ term: { 'slo.id': sloId } }, { term: { 'slo.revision': sloRevision } }],