[Discover] Add selector syntax support to log source profile (#206937)

This adds support for the new selector syntax to the log source profile
heuristics. It will only match when index name expression exclusively
contains implicit or explicit `data` selectors.

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.test.ts

@@ -10,7 +10,7 @@
 import { createRegExpPatternFrom } from './create_regexp_pattern_from';
 
 describe('createRegExpPatternFrom should create a regular expression starting from a string that', () => {
-  const regExpPattern = createRegExpPatternFrom('logs');
+  const regExpPattern = createRegExpPatternFrom('logs', 'data');
 
   it('tests positive for single index patterns starting with the passed base pattern', () => {
     expect('logs*').toMatch(regExpPattern);
@@ -62,6 +62,52 @@ describe('createRegExpPatternFrom should create a regular expression starting fr
     expect('cluster1:logs-*,logs-*,').toMatch(regExpPattern);
   });
 
+  it('tests correctly for patterns with the data selector suffix', () => {
+    expect('logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data'));
+    expect('logs-*::data,').toMatch(createRegExpPatternFrom('logs', 'data'));
+    expect('cluster1:logs-*::data,logs-*::data').toMatch(createRegExpPatternFrom('logs', 'data'));
+
+    expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
+    expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
+    expect('cluster1:logs-*::data,logs-*::data').not.toMatch(
+      createRegExpPatternFrom('logs', 'failures')
+    );
+
+    expect('logs-*').not.toMatch(createRegExpPatternFrom('logs', '*'));
+    expect('logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*'));
+    expect('cluster1:logs-*::data,logs-*::data').not.toMatch(createRegExpPatternFrom('logs', '*'));
+  });
+
+  it('tests correctly for patterns with the failures selector suffix', () => {
+    expect('logs-*::failures').toMatch(createRegExpPatternFrom('logs', 'failures'));
+    expect('logs-*::failures,').toMatch(createRegExpPatternFrom('logs', 'failures'));
+    expect('cluster1:logs-*::failures,logs-*::failures').toMatch(
+      createRegExpPatternFrom('logs', 'failures')
+    );
+
+    expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', 'data'));
+    expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch(
+      createRegExpPatternFrom('logs', 'data')
+    );
+
+    expect('logs-*::failures').not.toMatch(createRegExpPatternFrom('logs', '*'));
+    expect('cluster1:logs-*::failures,logs-*::failures').not.toMatch(
+      createRegExpPatternFrom('logs', '*')
+    );
+  });
+
+  it('tests correctly for patterns with the wildcard selector suffix', () => {
+    expect('logs-*::*').toMatch(createRegExpPatternFrom('logs', '*'));
+    expect('logs-*::*,').toMatch(createRegExpPatternFrom('logs', '*'));
+    expect('cluster1:logs-*::*,logs-*::*').toMatch(createRegExpPatternFrom('logs', '*'));
+
+    expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data'));
+    expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'data'));
+
+    expect('logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
+    expect('cluster1:logs-*::*,logs-*::*').not.toMatch(createRegExpPatternFrom('logs', 'failures'));
+  });
+
   it('tests negative for patterns with spaces and unexpected commas', () => {
     expect('cluster1:logs-*,clust,er2:logs-*').not.toMatch(regExpPattern);
     expect('cluster1:logs-*,    cluster2:logs-*').not.toMatch(regExpPattern);

src/platform/packages/shared/kbn-data-view-utils/src/utils/create_regexp_pattern_from.ts

@@ -7,12 +7,31 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-export const createRegExpPatternFrom = (basePatterns: string | string[]) => {
-  const patterns = Array.isArray(basePatterns) ? basePatterns : [basePatterns];
-  // Create the base patterns union with strict boundaries
-  const basePatternGroup = `[^,\\s]*(\\b|_)(${patterns.join('|')})(\\b|_)([^,\\s]*)?`;
-  // Apply base patterns union for local and remote clusters
-  const localAndRemotePatternGroup = `((${basePatternGroup})|([^:,\\s]*:${basePatternGroup}))`;
-  // Handle trailing comma and multiple pattern concatenation
-  return new RegExp(`^${localAndRemotePatternGroup}(,${localAndRemotePatternGroup})*(,$|$)`, 'i');
+import { escapeRegExp } from 'lodash';
+
+type Selector = 'data' | 'failures' | '*';
+
+export const createRegExpPatternFrom = (basePatterns: string | string[], selector: Selector) => {
+  const normalizedBasePatterns = normalizeBasePatterns(basePatterns);
+
+  const indexNames = `(?:${normalizedBasePatterns.join('|')})`;
+  const selectorsSuffix = `(?:::(?:${escapeRegExp(selector)}))${
+    isDefaultSelector(selector) ? '?' : ''
+  }`;
+
+  return new RegExp(
+    `^(?:${optionalRemoteCluster}${optionalIndexNamePrefix}${indexNames}${optionalIndexNameSuffix}${selectorsSuffix},?)+$`,
+    'i'
+  );
 };
+
+const normalizeBasePatterns = (basePatterns: string | string[]): string[] =>
+  (Array.isArray(basePatterns) ? basePatterns : [basePatterns]).map(escapeRegExp);
+
+const isDefaultSelector = (selector: Selector): boolean => selector === 'data';
+
+const nameCharacters = '[^:,\\s]+';
+const segmentBoundary = '(?:\\b|_)';
+const optionalRemoteCluster = `(?:${nameCharacters}:)?`;
+const optionalIndexNamePrefix = `(?:${nameCharacters}${segmentBoundary})?`;
+const optionalIndexNameSuffix = `(?:${segmentBoundary}${nameCharacters})?`;

src/platform/packages/shared/kbn-discover-utils/src/data_types/logs/logs_context_service.ts

@@ -28,7 +28,8 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [
 ];
 
 export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom(
-  DEFAULT_ALLOWED_LOGS_BASE_PATTERNS
+  DEFAULT_ALLOWED_LOGS_BASE_PATTERNS,
+  'data'
 );
 
 export const createLogsContextService = async ({ logsDataAccess }: LogsContextServiceDeps) => {

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/profile.test.ts

@@ -28,9 +28,20 @@ const mockServices = createContextAwarenessMocks().profileProviderServices;
 
 describe('logsDataSourceProfileProvider', () => {
   const logsDataSourceProfileProvider = createLogsDataSourceProfileProvider(mockServices);
-  const VALID_INDEX_PATTERN = 'logs-nginx.access-*';
-  const MIXED_INDEX_PATTERN = 'logs-nginx.access-*,metrics-*';
-  const INVALID_INDEX_PATTERN = 'my_source-access-*';
+
+  const VALID_IMPLICIT_DATA_INDEX_PATTERN = 'logs-nginx.access-*';
+  const VALID_INDEX_PATTERNS: Array<[string, string]> = [
+    ['explicit data', 'logs-nginx.access-*::data'],
+    ['implicit data', VALID_IMPLICIT_DATA_INDEX_PATTERN],
+    ['mixed data selector qualification', 'logs-nginx.access-*::data,logs-nginx.error-*'],
+  ];
+  const INVALID_INDEX_PATTERNS: Array<[string, string]> = [
+    ['forbidden implicit data', 'my_source-access-*'],
+    ['mixed implicit data', 'logs-nginx.access-*,metrics-*'],
+    ['mixed explicit data', 'logs-nginx.access-*::data,metrics-*::data'],
+    ['mixed selector', 'logs-nginx.access-*,logs-nginx.access-*::failures'],
+  ];
+
   const ROOT_CONTEXT: ContextWithProfileId<RootContext> = {
     profileId: OBSERVABILITY_ROOT_PROFILE_ID,
     solutionType: SolutionType.Observability,
@@ -43,64 +54,62 @@ describe('logsDataSourceProfileProvider', () => {
     isMatch: false,
   };
 
-  it('should match ES|QL sources with an allowed index pattern in its query', () => {
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createEsqlDataSource(),
-        query: { esql: `from ${VALID_INDEX_PATTERN}` },
-      })
-    ).toEqual(RESOLUTION_MATCH);
-  });
+  it.each(VALID_INDEX_PATTERNS)(
+    'should match ES|QL sources with an allowed %s index pattern in its query',
+    (_, validIndexPattern) => {
+      expect(
+        logsDataSourceProfileProvider.resolve({
+          rootContext: ROOT_CONTEXT,
+          dataSource: createEsqlDataSource(),
+          query: { esql: `from ${validIndexPattern}` },
+        })
+      ).toEqual(RESOLUTION_MATCH);
+    }
+  );
 
-  it('should NOT match ES|QL sources with a mixed or not allowed index pattern in its query', () => {
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createEsqlDataSource(),
-        query: { esql: `from ${INVALID_INDEX_PATTERN}` },
-      })
-    ).toEqual(RESOLUTION_MISMATCH);
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createEsqlDataSource(),
-        query: { esql: `from ${MIXED_INDEX_PATTERN}` },
-      })
-    ).toEqual(RESOLUTION_MISMATCH);
-  });
+  it.each(INVALID_INDEX_PATTERNS)(
+    'should NOT match ES|QL sources with a %s index pattern in its query',
+    (_, invalidIndexPattern) => {
+      expect(
+        logsDataSourceProfileProvider.resolve({
+          rootContext: ROOT_CONTEXT,
+          dataSource: createEsqlDataSource(),
+          query: { esql: `from ${invalidIndexPattern}` },
+        })
+      ).toEqual(RESOLUTION_MISMATCH);
+    }
+  );
 
-  it('should match data view sources with an allowed index pattern', () => {
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createDataViewDataSource({ dataViewId: VALID_INDEX_PATTERN }),
-        dataView: createStubIndexPattern({ spec: { title: VALID_INDEX_PATTERN } }),
-      })
-    ).toEqual(RESOLUTION_MATCH);
-  });
+  it.each(VALID_INDEX_PATTERNS)(
+    'should match data view sources with an allowed %s index pattern',
+    (_, validIndexPattern) => {
+      expect(
+        logsDataSourceProfileProvider.resolve({
+          rootContext: ROOT_CONTEXT,
+          dataSource: createDataViewDataSource({ dataViewId: validIndexPattern }),
+          dataView: createStubIndexPattern({ spec: { title: validIndexPattern } }),
+        })
+      ).toEqual(RESOLUTION_MATCH);
+    }
+  );
 
-  it('should NOT match data view sources with a mixed or not allowed index pattern', () => {
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createDataViewDataSource({ dataViewId: INVALID_INDEX_PATTERN }),
-        dataView: createStubIndexPattern({ spec: { title: INVALID_INDEX_PATTERN } }),
-      })
-    ).toEqual(RESOLUTION_MISMATCH);
-    expect(
-      logsDataSourceProfileProvider.resolve({
-        rootContext: ROOT_CONTEXT,
-        dataSource: createDataViewDataSource({ dataViewId: MIXED_INDEX_PATTERN }),
-        dataView: createStubIndexPattern({ spec: { title: MIXED_INDEX_PATTERN } }),
-      })
-    ).toEqual(RESOLUTION_MISMATCH);
-  });
+  it.each(INVALID_INDEX_PATTERNS)(
+    'should NOT match data view sources with a %s index pattern',
+    (_, invalidIndexPattern) => {
+      expect(
+        logsDataSourceProfileProvider.resolve({
+          rootContext: ROOT_CONTEXT,
+          dataSource: createDataViewDataSource({ dataViewId: invalidIndexPattern }),
+          dataView: createStubIndexPattern({ spec: { title: invalidIndexPattern } }),
+        })
+      ).toEqual(RESOLUTION_MISMATCH);
+    }
+  );
 
   it('does NOT match data view sources when solution type is not Observability', () => {
     const params: Omit<DataSourceProfileProviderParams, 'rootContext'> = {
       dataSource: createEsqlDataSource(),
-      query: { esql: `from ${VALID_INDEX_PATTERN}` },
+      query: { esql: `from ${VALID_IMPLICIT_DATA_INDEX_PATTERN}` },
     };
     expect(logsDataSourceProfileProvider.resolve({ ...params, rootContext: ROOT_CONTEXT })).toEqual(
       RESOLUTION_MATCH
@@ -127,7 +136,7 @@ describe('logsDataSourceProfileProvider', () => {
 
   const dataViewWithLogLevel = createStubIndexPattern({
     spec: {
-      title: VALID_INDEX_PATTERN,
+      title: VALID_IMPLICIT_DATA_INDEX_PATTERN,
       fields: {
         'log.level': {
           name: 'log.level',
@@ -146,7 +155,7 @@ describe('logsDataSourceProfileProvider', () => {
 
   const dataViewWithoutLogLevel = createStubIndexPattern({
     spec: {
-      title: VALID_INDEX_PATTERN,
+      title: VALID_IMPLICIT_DATA_INDEX_PATTERN,
     },
   });
 

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/observability/logs_data_source_profile/sub_profiles/create_resolve.ts

@@ -14,7 +14,7 @@ import { OBSERVABILITY_ROOT_PROFILE_ID } from '../../consts';
 
 export const createResolve = (baseIndexPattern: string): DataSourceProfileProvider['resolve'] => {
   const testIndexPattern = testPatternAgainstAllowedList([
-    createRegExpPatternFrom(baseIndexPattern),
+    createRegExpPatternFrom(baseIndexPattern, 'data'),
   ]);
 
   return (params) => {

x-pack/solutions/observability/plugins/logs_explorer/common/data_views/models/data_view_descriptor.ts

@@ -11,7 +11,7 @@ import { DataViewSpecWithId } from '../../data_source_selection';
 import { DataViewDescriptorType } from '../types';
 
 const LOGS_ALLOWED_LIST = [
-  createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS),
+  createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS, 'data'),
   // Add more strings or regex patterns as needed
 ];
 
@@ -59,7 +59,7 @@ export class DataViewDescriptor {
 
   testAgainstAllowedList(allowedList: string[]) {
     return this.title
-      ? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList)])(this.title)
+      ? testPatternAgainstAllowedList([createRegExpPatternFrom(allowedList, 'data')])(this.title)
       : false;
   }