[8.16] [Security Solution][Detection Engine] fixes ES|QL data tier filter from adv settings (#196390) (#196698)

# Backport This will backport the following commits from `main` to `8.16`: - [[Security Solution][Detection Engine] fixes ES|QL data tier filter from adv settings (#196390)](https://github.com/elastic/kibana/pull/196390)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com>
2025-04-24 09:48:58 -04:00 · 2024-10-18 04:45:22 +11:00 · 2024-10-18 04:45:22 +11:00 · ec90dae745
commit ec90dae745
parent 4105d351bf
4 changed files with 70 additions and 1 deletions
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts
@ -30,6 +30,7 @@ import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters';
 import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
 import type { CreateRuleOptions, RunOpts, SignalSource } from '../types';
 import { logEsqlRequest } from '../utils/logged_requests';
+import { getDataTierFilter } from '../utils/get_data_tier_filter';
 import * as i18n from '../translations';

 import {
@ -90,6 +91,10 @@ export const esqlExecutor = async ({
  return withSecuritySpan('esqlExecutor', async () => {
    const result = createSearchAfterReturnType();
    let size = tuple.maxSignals;
+    const dataTiersFilters = await getDataTierFilter({
+      uiSettingsClient: services.uiSettingsClient,
+    });
+
    try {
      while (
        result.createdSignalsCount <= tuple.maxSignals &&
@ -100,7 +105,7 @@ export const esqlExecutor = async ({
          from: tuple.from.toISOString(),
          to: tuple.to.toISOString(),
          size,
-          filters: [],
+          filters: dataTiersFilters,
          primaryTimestamp,
          secondaryTimestamp,
          exceptionFilter,
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts
@ -14,6 +14,7 @@ import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/comm
 import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring';

 import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils';
+import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '@kbn/security-solution-plugin/common/constants';
 import {
  getPreviewAlerts,
  previewRule,
@ -25,6 +26,7 @@ import {
  scheduleRuleRun,
  stopAllManualRuns,
  waitForBackfillExecuted,
+  setAdvancedSettings,
 } from '../../../../utils';
 import {
  deleteAllRules,
@ -1428,6 +1430,12 @@ export default ({ getService }: FtrProviderContext) => {
        await indexEnhancedDocuments({ documents: [doc1], interval, id });
      });

+      afterEach(async () => {
+        await setAdvancedSettings(supertest, {
+          [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: [],
+        });
+      });
+
      it('should not return requests property when not enabled', async () => {
        const { logs } = await previewRule({
          supertest,
@ -1463,6 +1471,35 @@ export default ({ getService }: FtrProviderContext) => {
          'POST /ecs_compliant/_search?ignore_unavailable=true'
        );
      });
+      it('should not return requests with any data tier filter', async () => {
+        const { logs } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          enableLoggedRequests: true,
+        });
+
+        const requests = logs[0].requests;
+
+        expect(requests![0].request).not.toContain('data_frozen');
+      });
+      it('should return requests with included data tiers filters from advanced settings', async () => {
+        await setAdvancedSettings(supertest, {
+          [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: ['data_frozen'],
+        });
+        const { logs } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          enableLoggedRequests: true,
+        });
+
+        const requests = logs[0].requests;
+
+        expect(requests![0].request).toMatch(
+          /"must_not":\s*\[\s*{\s*"terms":\s*{\s*"_tier":\s*\[\s*"data_frozen"\s*\]/
+        );
+      });
    });
  });
 };
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts
@ -24,3 +24,4 @@ export * from './get_stats';
 export * from './get_detection_metrics_from_body';
 export * from './get_stats_url';
 export * from './combine_to_ndjson';
+export * from './set_advanced_settings';
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts
@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ELASTIC_HTTP_VERSION_HEADER,
+  X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
+} from '@kbn/core-http-common';
+
+import type SuperTest from 'supertest';
+
+export const setAdvancedSettings = async (
+  supertest: SuperTest.Agent,
+  settings: Record<string, string[] | string | number | boolean>
+) => {
+  return supertest
+    .post('/internal/kibana/settings')
+    .set('kbn-xsrf', 'true')
+    .set(ELASTIC_HTTP_VERSION_HEADER, '1')
+    .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana')
+    .send({ changes: settings })
+    .expect(200);
+};