[Logs] Add Log Sources advanced setting and client / server access services (#186468)

## Summary

Implements part 1 of
https://github.com/elastic/observability-dev/issues/3498 (adds an
advanced setting and data access services for consumers).

## Reviewer notes

- Please see note comments inline with the code.

- The `limits.yml` change was generated by `node
scripts/build_kibana_platform_plugins.js --update-limits`.

- There are no consumers using this yet, so you'll need to make some
minor adjustments if you'd like to test the access services. In a plugin
(`infra` for example) the server side access could look like this:

```ts
const [, { logsDataAccess }] = await getStartServices();
const logSourcesService = await logsDataAccess.services.getLogSourcesService(request);
await logSourcesService.setLogSources([
  { indexPattern: 'logs-*-*' },
  { indexPattern: 'test-logs-*' },
]);
```

Public access could look like this:

```ts
const logSourcesService = plugins.logsDataAccess.services.logSourcesService;
logSourcesService.setLogSources([{ indexPattern: 'client-side-logs-*' }]);
```

- I haven't added any tests here yet as any unit tests would more or
less just be re-testing a UI settings mock. Functional tests will be
valuable once there are consumers.


## UI

<img width="1425" alt="Screenshot 2024-06-20 at 10 41 16"
src="30608dcf-4c34-4d29-9a22-441b06757c28">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-management/settings/setting_ids/index.ts

@@ -142,6 +142,7 @@ export const OBSERVABILITY_APM_ENABLE_SERVICE_INVENTORY_TABLE_SEARCH_BAR =
 export const OBSERVABILITY_LOGS_EXPLORER_ALLOWED_DATA_VIEWS_ID =
   'observability:logsExplorer:allowedDataViews';
 export const OBSERVABILITY_APM_ENABLE_MULTI_SIGNAL = 'observability:apmEnableMultiSignal';
+export const OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID = 'observability:logSources';
 
 // Reporting settings
 export const XPACK_REPORTING_CUSTOM_PDF_LOGO_ID = 'xpackReporting:customPdfLogo';

packages/kbn-optimizer/limits.yml

@@ -95,6 +95,7 @@ pageLoadAssetSize:
   licensing: 29004
   links: 44490
   lists: 22900
+  logsDataAccess: 16759
   logsExplorer: 60000
   logsShared: 281060
   logstash: 53548

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -499,6 +499,13 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
       _meta: { description: 'Non-default value of setting.' },
     },
   },
+  'observability:logSources': {
+    type: 'array',
+    items: {
+      type: 'keyword',
+      _meta: { description: 'Non-default value of setting.' },
+    },
+  },
   'banners:placement': {
     type: 'keyword',
     _meta: { description: 'Non-default value of setting.' },

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -53,6 +53,7 @@ export interface UsageStats {
   'observability:apmEnableTableSearchBar': boolean;
   'observability:apmEnableServiceInventoryTableSearchBar': boolean;
   'observability:logsExplorer:allowedDataViews': string[];
+  'observability:logSources': string[];
   'observability:aiAssistantLogsIndexPattern': string;
   'observability:aiAssistantResponseLanguage': string;
   'observability:aiAssistantSimulatedFunctionCalling': boolean;

src/plugins/telemetry/schema/oss_plugins.json

@@ -10343,6 +10343,15 @@
             }
           }
         },
+        "observability:logSources": {
+          "type": "array",
+          "items": {
+            "type": "keyword",
+            "_meta": {
+              "description": "Non-default value of setting."
+            }
+          }
+        },
         "banners:placement": {
           "type": "keyword",
           "_meta": {

x-pack/.i18nrc.json

@@ -47,6 +47,7 @@
     "xpack.idxMgmtPackage": "packages/index-management",
     "xpack.indexLifecycleMgmt": "plugins/index_lifecycle_management",
     "xpack.infra": "plugins/observability_solution/infra",
+    "xpack.logsDataAccess": "plugins/observability_solution/logs_data_access",
     "xpack.logsExplorer": "plugins/observability_solution/logs_explorer",
     "xpack.logsShared": "plugins/observability_solution/logs_shared",
     "xpack.fleet": "plugins/fleet",

x-pack/plugins/observability_solution/logs_data_access/common/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DEFAULT_LOG_SOURCES = ['logs-*-*'];

x-pack/plugins/observability_solution/logs_data_access/common/types.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface LogSource {
+  indexPattern: string;
+}

x-pack/plugins/observability_solution/logs_data_access/common/ui_settings.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { UiSettingsParams } from '@kbn/core-ui-settings-common';
+import { i18n } from '@kbn/i18n';
+import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids';
+import { DEFAULT_LOG_SOURCES } from './constants';
+
+/**
+ * uiSettings definitions for the logs_data_access plugin.
+ */
+export const uiSettings: Record<string, UiSettingsParams> = {
+  [OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID]: {
+    category: ['observability'],
+    name: i18n.translate('xpack.logsDataAccess.logSources', {
+      defaultMessage: 'Log sources',
+    }),
+    value: DEFAULT_LOG_SOURCES,
+    description: i18n.translate('xpack.logsDataAccess.logSourcesDescription', {
+      defaultMessage:
+        'Sources to be used for logs data. If the data contained in these indices is not logs data, you may experience degraded functionality.',
+    }),
+    type: 'array',
+    schema: schema.arrayOf(schema.string()),
+    requiresPageReload: true,
+  },
+};

x-pack/plugins/observability_solution/logs_data_access/kibana.jsonc

@@ -5,7 +5,7 @@
   "plugin": {
     "id": "logsDataAccess",
     "server": true,
-    "browser": false,
+    "browser": true,
     "requiredPlugins": [
       "data", 
       "dataViews"

x-pack/plugins/observability_solution/logs_data_access/public/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PluginInitializer } from '@kbn/core/public';
+import {
+  LogsDataAccessPlugin,
+  LogsDataAccessPluginSetup,
+  LogsDataAccessPluginStart,
+} from './plugin';
+import { LogsDataAccessPluginSetupDeps, LogsDataAccessPluginStartDeps } from './types';
+
+export const plugin: PluginInitializer<
+  LogsDataAccessPluginSetup,
+  LogsDataAccessPluginStart,
+  LogsDataAccessPluginSetupDeps,
+  LogsDataAccessPluginStartDeps
+> = () => {
+  return new LogsDataAccessPlugin();
+};

x-pack/plugins/observability_solution/logs_data_access/public/plugin.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/public';
+import { Plugin } from '@kbn/core/public';
+import { registerServices } from './services/register_services';
+import { LogsDataAccessPluginSetupDeps, LogsDataAccessPluginStartDeps } from './types';
+export type LogsDataAccessPluginSetup = ReturnType<LogsDataAccessPlugin['setup']>;
+export type LogsDataAccessPluginStart = ReturnType<LogsDataAccessPlugin['start']>;
+
+export class LogsDataAccessPlugin
+  implements
+    Plugin<
+      LogsDataAccessPluginSetup,
+      LogsDataAccessPluginStart,
+      LogsDataAccessPluginSetupDeps,
+      LogsDataAccessPluginStartDeps
+    >
+{
+  public setup() {}
+
+  public start(core: CoreStart, plugins: LogsDataAccessPluginStartDeps) {
+    const services = registerServices({
+      deps: {
+        uiSettings: core.uiSettings,
+      },
+    });
+
+    return {
+      services,
+    };
+  }
+
+  public stop() {}
+}

x-pack/plugins/observability_solution/logs_data_access/public/services/log_sources_service/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids';
+import { LogSource } from '../../../common/types';
+import { RegisterServicesParams } from '../register_services';
+
+export function createLogSourcesService(params: RegisterServicesParams) {
+  const { uiSettings } = params.deps;
+  return {
+    getLogSources: (): LogSource[] => {
+      const logSources = uiSettings.get<string[]>(OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID);
+      return logSources.map((logSource) => ({
+        indexPattern: logSource,
+      }));
+    },
+    setLogSources: async (sources: LogSource[]) => {
+      return await uiSettings.set(
+        OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID,
+        sources.map((source) => source.indexPattern)
+      );
+    },
+  };
+}

x-pack/plugins/observability_solution/logs_data_access/public/services/register_services.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IUiSettingsClient } from '@kbn/core-ui-settings-browser';
+import { createLogSourcesService } from './log_sources_service';
+
+export interface RegisterServicesParams {
+  deps: {
+    uiSettings: IUiSettingsClient;
+  };
+}
+
+export function registerServices(params: RegisterServicesParams) {
+  return {
+    logSourcesService: createLogSourcesService(params),
+  };
+}

x-pack/plugins/observability_solution/logs_data_access/public/types.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface LogsDataAccessPluginSetupDeps {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface LogsDataAccessPluginStartDeps {}

x-pack/plugins/observability_solution/logs_data_access/server/plugin.ts

@@ -12,6 +12,7 @@ import type {
   Plugin,
   PluginInitializerContext,
 } from '@kbn/core/server';
+import { uiSettings } from '../common/ui_settings';
 import { registerServices } from './services/register_services';
 import { LogsDataAccessPluginStartDeps, LogsDataAccessPluginSetupDeps } from './types';
 
@@ -32,12 +33,17 @@ export class LogsDataAccessPlugin
   constructor(initializerContext: PluginInitializerContext) {
     this.logger = initializerContext.logger.get();
   }
-  public setup(core: CoreSetup, plugins: LogsDataAccessPluginSetupDeps) {}
+  public setup(core: CoreSetup, plugins: LogsDataAccessPluginSetupDeps) {
+    core.uiSettings.register(uiSettings);
+  }
 
   public start(core: CoreStart, plugins: LogsDataAccessPluginStartDeps) {
     const services = registerServices({
       logger: this.logger,
-      deps: {},
+      deps: {
+        savedObjects: core.savedObjects,
+        uiSettings: core.uiSettings,
+      },
     });
 
     return {

x-pack/plugins/observability_solution/logs_data_access/server/services/log_sources_service/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaRequest } from '@kbn/core-http-server';
+import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids';
+import { LogSource } from '../../../common/types';
+import { RegisterServicesParams } from '../register_services';
+
+export function createGetLogSourcesService(params: RegisterServicesParams) {
+  return async (request: KibanaRequest) => {
+    const { savedObjects, uiSettings } = params.deps;
+    const soClient = savedObjects.getScopedClient(request);
+    const uiSettingsClient = uiSettings.asScopedToClient(soClient);
+    return {
+      getLogSources: async (): Promise<LogSource[]> => {
+        const logSources = await uiSettingsClient.get<string[]>(
+          OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID
+        );
+        return logSources.map((logSource) => ({
+          indexPattern: logSource,
+        }));
+      },
+      setLogSources: async (sources: LogSource[]) => {
+        return await uiSettingsClient.set(
+          OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID,
+          sources.map((source) => source.indexPattern)
+        );
+      },
+    };
+  };
+}

x-pack/plugins/observability_solution/logs_data_access/server/services/register_services.ts

@@ -5,16 +5,23 @@
  * 2.0.
  */
 
+import { SavedObjectsServiceStart } from '@kbn/core-saved-objects-server';
+import { UiSettingsServiceStart } from '@kbn/core-ui-settings-server';
 import { Logger } from '@kbn/logging';
 import { createGetLogsRatesService } from './get_logs_rates_service';
+import { createGetLogSourcesService } from './log_sources_service';
 
 export interface RegisterServicesParams {
   logger: Logger;
-  deps: {};
+  deps: {
+    savedObjects: SavedObjectsServiceStart;
+    uiSettings: UiSettingsServiceStart;
+  };
 }
 
 export function registerServices(params: RegisterServicesParams) {
   return {
     getLogsRatesService: createGetLogsRatesService(params),
+    getLogSourcesService: createGetLogSourcesService(params),
   };
 }

x-pack/plugins/observability_solution/logs_data_access/tsconfig.json

@@ -3,12 +3,20 @@
   "compilerOptions": {
     "outDir": "target/types"
   },
-  "include": ["common/**/*", "server/**/*", "jest.config.js"],
+  "include": ["common/**/*", "server/**/*",  "public/**/*", "jest.config.js"],
   "exclude": ["target/**/*"],
   "kbn_references": [
     "@kbn/core",
     "@kbn/logging",
     "@kbn/data-plugin",
     "@kbn/data-views-plugin",
+    "@kbn/core-http-server",
+    "@kbn/management-settings-ids",
+    "@kbn/config-schema",
+    "@kbn/core-ui-settings-common",
+    "@kbn/i18n",
+    "@kbn/core-saved-objects-server",
+    "@kbn/core-ui-settings-server",
+    "@kbn/core-ui-settings-browser",
   ]
 }