github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Telemetry] Add index metadata EBT event (#218546)

Browse source 
## Summary



- Fix https://github.com/elastic/kibana/issues/216044
- Add a new EBT event collecting index template info
    ```typescript
    export interface IndexTemplateInfo {
      template_name: string;
      index_mode: Nullable<string>;
      datastream: boolean;
      package_name: Nullable<string>;
      managed_by: Nullable<string>;
      beat: Nullable<string>;
      is_managed: Nullable<boolean>;
      composed_of: string[];
      source_enabled: Nullable<boolean>;
      source_includes: string[];
      source_excludes: string[];
    }
    ```

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
This commit is contained in:
Sebastián Zaffarano 2025-04-17 20:00:26 +02:00 • committed by GitHub
parent 7160b360c7
commit c9b3a3e27b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 237 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

99
x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/event_based/events.ts
View file

@ -15,6 +15,7 @@ import type {
DataStreams,
IlmPolicies,
IlmsStats,
IndexTemplatesStats,
IndicesSettings,
IndicesStats,
} from '../indices.metadata.types';
@ -650,6 +651,103 @@ export const TELEMETRY_ILM_STATS_EVENT: EventTypeOpts<IlmsStats> = {
},
};
export const TELEMETRY_INDEX_TEMPLATES_EVENT: EventTypeOpts<IndexTemplatesStats> = {
eventType: 'telemetry_index_templates_event',
schema: {
items: {
type: 'array',
items: {
properties: {
template_name: {
type: 'keyword',
_meta: { description: 'The name of the template.' },
},
index_mode: {
type: 'keyword',
_meta: {
optional: true,
description: 'The index mode.',
},
},
datastream: {
type: 'boolean',
_meta: {
description: 'Datastream dataset',
},
},
package_name: {
type: 'keyword',
_meta: {
optional: true,
description: 'The package name',
},
},
managed_by: {
type: 'keyword',
_meta: {
optional: true,
description: 'Managed by',
},
},
beat: {
type: 'keyword',
_meta: {
optional: true,
description: 'Shipper name',
},
},
is_managed: {
type: 'boolean',
_meta: {
optional: true,
description: 'Whether the template is managed',
},
},
composed_of: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description: 'List of template components',
},
},
_meta: { description: '' },
},
source_enabled: {
type: 'boolean',
_meta: {
optional: true,
description:
'The _source field contains the original JSON document body that was provided at index time',
},
},
source_includes: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description: 'Fields included in _source, if enabled',
},
},
_meta: { description: '' },
},
source_excludes: {
type: 'array',
items: {
type: 'keyword',
_meta: {
description: '',
},
},
_meta: { description: 'Fields excludes from _source, if enabled' },
},
},
},
_meta: { description: 'Index templates info' },
},
},
};
export const TELEMETRY_NODE_INGEST_PIPELINES_STATS_EVENT: EventTypeOpts<NodeIngestPipelinesStats> =
{
eventType: 'telemetry_node_ingest_pipelines_stats_event',
@ -1306,6 +1404,7 @@ export const events = [
TELEMETRY_ILM_STATS_EVENT,
TELEMETRY_INDEX_SETTINGS_EVENT,
TELEMETRY_INDEX_STATS_EVENT,
TELEMETRY_INDEX_TEMPLATES_EVENT,
TELEMETRY_NODE_INGEST_PIPELINES_STATS_EVENT,
SIEM_MIGRATIONS_MIGRATION_SUCCESS,
SIEM_MIGRATIONS_MIGRATION_FAILURE,

18
x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/indices.metadata.types.ts
View file

@ -41,6 +41,24 @@ export interface IlmStats {
policy_name?: string;
}
export interface IndexTemplatesStats {
items: IndexTemplateInfo[];
}
export interface IndexTemplateInfo {
template_name: string;
index_mode: Nullable<string>;
datastream: boolean;
package_name: Nullable<string>;
managed_by: Nullable<string>;
beat: Nullable<string>;
is_managed: Nullable<boolean>;
composed_of: string[];
source_enabled: Nullable<boolean>;
source_includes: string[];
source_excludes: string[];
}
export interface IndicesStats {
items: IndexStats[];
}

55
x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/receiver.ts
View file

@ -31,6 +31,7 @@ import type {
IndicesGetRequest,
NodesStatsRequest,
Duration,
IndicesGetIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
import {
@ -101,6 +102,7 @@ import type {
Index,
IndexSettings,
IndexStats,
IndexTemplateInfo,
} from './indices.metadata.types';
import { chunkStringsByMaxLength } from './collections_helpers';
import type {
@ -265,6 +267,7 @@ export interface ITelemetryReceiver {
getDataStreams(): Promise<DataStream[]>;
getIndicesStats(indices: string[], chunkSize: number): AsyncGenerator<IndexStats, void, unknown>;
getIlmsStats(indices: string[], chunkSize: number): AsyncGenerator<IlmStats, void, unknown>;
getIndexTemplatesStats(): Promise<IndexTemplateInfo[]>;
getIlmsPolicies(ilms: string[], chunkSize: number): AsyncGenerator<IlmPolicy, void, unknown>;
getIngestPipelinesStats(timeout: Duration): Promise<NodeIngestPipelinesStats[]>;
@ -1373,11 +1376,11 @@ export class TelemetryReceiver implements ITelemetryReceiver {
name: '*',
expand_wildcards: ['open', 'hidden'],
filter_path: [
'data_streams.ilm_policy',
'data_streams.indices.ilm_policy',
'data_streams.indices.index_name',
'data_streams.name',
'ilm_policy',
'template',
'data_streams.template',
],
};
@ -1509,6 +1512,54 @@ export class TelemetryReceiver implements ITelemetryReceiver {
}
}
public async getIndexTemplatesStats(): Promise<IndexTemplateInfo[]> {
const es = this.esClient();
this.logger.l('Fetching datstreams');
const request: IndicesGetIndexTemplateRequest = {
name: '*',
filter_path: [
'index_templates.name',
'index_templates.index_template.template.settings.index.mode',
'index_templates.index_template.data_stream',
'index_templates.index_template._meta.package.name',
'index_templates.index_template._meta.managed_by',
'index_templates.index_template._meta.beat',
'index_templates.index_template._meta.managed',
'index_templates.index_template.composed_of',
'index_templates.index_template.template.mappings._source.enabled',
'index_templates.index_template.template.mappings._source.includes',
'index_templates.index_template.template.mappings._source.excludes',
],
};
return es.indices
.getIndexTemplate(request)
.then((response) =>
response.index_templates.map((props) => {
const datastream = props.index_template?.data_stream !== undefined;
return {
template_name: props.name,
index_mode: props.index_template.template?.settings?.index?.mode,
package_name: props.index_template._meta?.package?.name,
datastream,
managed_by: props.index_template._meta?.managed_by,
beat: props.index_template._meta?.beat,
is_managed: props.index_template._meta?.managed,
composed_of: props.index_template.composed_of,
source_enabled: props.index_template.template?.mappings?._source?.enabled,
source_includes: props.index_template.template?.mappings?._source?.includes ?? [],
source_excludes: props.index_template.template?.mappings?._source?.excludes ?? [],
} as IndexTemplateInfo;
})
)
.catch((error) => {
this.logger.warn('Error fetching index templates', { error_message: error } as LogMeta);
throw error;
});
}
public async *getIlmsPolicies(ilms: string[], chunkSize: number) {
const es = this.esClient();
const safeChunkSize = Math.min(chunkSize, 3000);

34
x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/indices.metadata.ts
View file

@ -21,6 +21,7 @@ import {
TELEMETRY_ILM_STATS_EVENT,
TELEMETRY_INDEX_SETTINGS_EVENT,
TELEMETRY_INDEX_STATS_EVENT,
TELEMETRY_INDEX_TEMPLATES_EVENT,
} from '../event_based/events';
import { telemetryConfiguration } from '../configuration';
import type {
@ -29,6 +30,8 @@ import type {
IlmPolicies,
IlmsStats,
IndexSettings,
IndexTemplateInfo,
IndexTemplatesStats,
IndicesSettings,
IndicesStats,
} from '../indices.metadata.types';
@ -81,6 +84,19 @@ export function createTelemetryIndicesMetadataTaskConfig() {
return indicesStats.items.length;
};
const publishIndexTemplatesStats = async (
indexTemplates: IndexTemplateInfo[]
): Promise<number> => {
const templateStats: IndexTemplatesStats = {
items: indexTemplates,
};
sender.reportEBT(TELEMETRY_INDEX_TEMPLATES_EVENT, templateStats);
log.info(`Sent index templates`, { count: indexTemplates.length } as LogMeta);
return templateStats.items.length;
};
const publishIndicesSettings = (settings: IndexSettings[]): number => {
const indicesSettings: IndicesSettings = {
items: settings,
@ -137,9 +153,10 @@ export function createTelemetryIndicesMetadataTaskConfig() {
try {
// 1. Get cluster stats and list of indices and datastreams
const [indicesSettings, dataStreams] = await Promise.all([
const [indicesSettings, dataStreams, indexTemplates] = await Promise.all([
receiver.getIndices(),
receiver.getDataStreams(),
receiver.getIndexTemplatesStats(),
]);
const indices = indicesSettings.map((index) => index.index_name);
@ -194,11 +211,26 @@ export function createTelemetryIndicesMetadataTaskConfig() {
return 0;
});
// 7. Publish index templates
const indexTemplatesCount: number = await publishIndexTemplatesStats(
indexTemplates.slice(0, taskConfig.indices_threshold)
)
.then((count) => {
incrementCounter(TelemetryCounter.DOCS_SENT, 'index-templates', count);
return count;
})
.catch((err) => {
log.warn(`Error getting index templates`, { error: err.message } as LogMeta);
incrementCounter(TelemetryCounter.RUNTIME_ERROR, 'index-templates', 1);
return 0;
});
log.info(`Sent EBT events`, {
datastreams: dsCount,
indicesSettings: indicesSettingsCount,
ilms: ilmNames.size,
indices: indicesCount,
indexTemplates: indexTemplatesCount,
policies: policyCount,
} as LogMeta);

38
x-pack/test/security_solution_api_integration/test_suites/telemetry/tasks/indices_metadata.ts
View file

@ -29,6 +29,7 @@ const TELEMETRY_ILM_STATS_EVENT = 'telemetry_ilm_stats_event';
const TELEMETRY_ILM_POLICY_EVENT = 'telemetry_ilm_policy_event';
const TELEMETRY_DATA_STREAM_EVENT = 'telemetry_data_stream_event';
const TELEMETRY_INDEX_SETTINGS_EVENT = 'telemetry_index_settings_event';
const TELEMETRY_INDEX_TEMPLATES_EVENT = 'telemetry_index_templates_event';
export default ({ getService }: FtrProviderContext) => {
const ebtServer = getService('kibana_ebt_server');
@ -42,8 +43,7 @@ export default ({ getService }: FtrProviderContext) => {
let defaultPipeline: string;
let finalPipeline: string;
// FLAKY: https://github.com/elastic/kibana/issues/216044
describe.skip('@ess @serverless indices metadata', () => {
describe('@ess @serverless indices metadata', () => {
beforeEach(async () => {
dsName = await randomDatastream(es);
await ensureBackingIndices(dsName, NUM_INDICES, es);
@ -59,7 +59,18 @@ export default ({ getService }: FtrProviderContext) => {
dsName,
});
expect(events.length).toEqual(1);
expect(events.length).toBeGreaterThanOrEqual(1);
});
it('should include `template` in data stream events when defined', async () => {
const events = await launchTaskAndWaitForEvents({
eventTypes: [TELEMETRY_DATA_STREAM_EVENT],
dsName,
});
expect(events.length).toBeGreaterThanOrEqual(1);
const event = events[0] as any;
expect(event.template).toBeDefined();
});
it('should publish index stats events', async () => {
@ -89,13 +100,24 @@ export default ({ getService }: FtrProviderContext) => {
await cleanupIngestPipelines(es);
});
it('should include `ilm_policy` in data stream events when defined', async () => {
const events = await launchTaskAndWaitForEvents({
eventTypes: [TELEMETRY_DATA_STREAM_EVENT],
dsName,
});
expect(events.length).toBeGreaterThanOrEqual(1);
const event = events[0] as any;
expect(event.ilm_policy).toBeDefined();
});
it('should publish ilm policy events', async () => {
const events = await launchTaskAndWaitForEvents({
eventTypes: [TELEMETRY_ILM_POLICY_EVENT],
policyName,
});
expect(events.length).toEqual(1);
expect(events.length).toBeGreaterThanOrEqual(1);
});
it('should publish ilm stats events', async () => {
@ -205,6 +227,14 @@ export default ({ getService }: FtrProviderContext) => {
);
expect(events.filter((v) => v.final_pipeline === finalPipeline)).toHaveLength(NUM_INDICES);
});
it('should publish index templates', async () => {
const events = await launchTaskAndWaitForEvents({
eventTypes: [TELEMETRY_INDEX_TEMPLATES_EVENT],
});
expect(events.length).toBeGreaterThanOrEqual(1);
});
});
const indexRandomDocs = async (index: string, count: number) => {