github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[eem] narrow down index patterns in definition templates (#189182)

Browse source 
In https://github.com/elastic/kibana/pull/188410 we moved history and
latest index templates from global scope to definition scope. The
definition-scoped templates have a wide pattern that would grep any
other definition template already installed and throw the following
error because of conflicting priority. This change narrows down the
index patterns defined in the templates to only grep the ones from the
installed definition

```
{
  "statusCode": 500,
  "error": "Internal Server Error",
  "message": """[illegal_argument_exception
	Root causes:
		illegal_argument_exception: index template [entities_v1_history_admin-console-services_index_template] has index patterns [.entities.v1.history.*] matching patterns from existing templates [entities_v1_history_builtin_services_from_ecs_data_index_template] with patterns (entities_v1_history_builtin_services_from_ecs_data_index_template => [.entities.v1.history.*]) that have the same priority [200], multiple index templates may not match during index creation, please use a different priority]: index template [entities_v1_history_admin-console-services_index_template] has index patterns [.entities.v1.history.*] matching patterns from existing templates [entities_v1_history_builtin_services_from_ecs_data_index_template] with patterns (entities_v1_history_builtin_services_from_ecs_data_index_template => [.entities.v1.history.*]) that have the same priority [200], multiple index templates may not match during index creation, please use a different priority"""
}
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Kevin Lacabane 2024-07-26 11:35:40 +02:00 • committed by GitHub
parent 61ad27e863
commit 10bfb4b4ae
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 307 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_ingest_pipeline.ts
View file

@ -24,7 +24,7 @@ export async function deleteHistoryIngestPipeline(
esClient.ingest.deletePipeline({ id: historyPipelineId }, { ignore: [404] })
);
} catch (e) {
logger.error(`Unable to delete history ingest pipeline [${definition.id}].`);
logger.error(`Unable to delete history ingest pipeline [${definition.id}]: ${e}`);
throw e;
}
}
@ -40,7 +40,7 @@ export async function deleteLatestIngestPipeline(
esClient.ingest.deletePipeline({ id: latestPipelineId }, { ignore: [404] })
);
} catch (e) {
logger.error(`Unable to delete latest ingest pipeline [${definition.id}].`);
logger.error(`Unable to delete latest ingest pipeline [${definition.id}]: ${e}`);
throw e;
}
}

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition_with_backfill.ts
View file

@ -7,7 +7,7 @@
import { entityDefinitionSchema } from '@kbn/entities-schema';
export const entityDefinitionWithBackfill = entityDefinitionSchema.parse({
id: 'admin-console-services',
id: 'admin-console-services-backfill',
version: '999.999.999',
name: 'Services for Admin Console',
type: 'service',

9
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/index.ts Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { entityDefinition } from './entity_definition';
export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';

6
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts
View file

@ -36,8 +36,8 @@ import {
import { uninstallEntityDefinition } from './uninstall_entity_definition';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
import { getEntitiesLatestIndexTemplateConfig } from '../../templates/entities_latest_template';
import { getEntitiesHistoryIndexTemplateConfig } from '../../templates/entities_history_template';
import { getEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
import { getEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
export interface InstallDefinitionParams {
esClient: ElasticsearchClient;
@ -111,7 +111,7 @@ export async function installEntityDefinition({
return entityDefinition;
} catch (e) {
logger.error(`Failed to install entity definition ${definition.id}`, e);
logger.error(`Failed to install entity definition ${definition.id}: ${e}`);
// Clean up anything that was successful.
if (installState.definition) {
await deleteEntityDefinition(soClient, definition, logger);

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/read_entity_definition.ts
View file

@ -30,7 +30,7 @@ export async function readEntityDefinition(
try {
return entityDefinitionSchema.parse(response.saved_objects[0].attributes);
} catch (e) {
logger.error(`Unable to parse entity definition with [${id}]`);
logger.error(`Unable to parse entity definition with [${id}]: ${e}`);
throw e;
}
}

4
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/stop_and_delete_transform.ts
View file

@ -38,7 +38,7 @@ export async function stopAndDeleteHistoryTransform(
{ logger }
);
} catch (e) {
logger.error(`Cannot stop or delete history transform [${definition.id}]`);
logger.error(`Cannot stop or delete history transform [${definition.id}]: ${e}`);
throw e;
}
}
@ -67,7 +67,7 @@ export async function stopAndDeleteHistoryBackfillTransform(
{ logger }
);
} catch (e) {
logger.error(`Cannot stop or delete history backfill transform [${definition.id}]`);
logger.error(`Cannot stop or delete history backfill transform [${definition.id}]: ${e}`);
throw e;
}
}

78
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap generated Normal file
View file

@ -0,0 +1,78 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getEntitiesHistoryIndexTemplateConfig(definitionId) should generate a valid index template 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",
"ecs_version": "8.0.0",
"managed": true,
"managed_by": "elastic_entity_model",
},
"composed_of": Array [
"entities_v1_history_base",
"entities_v1_entity",
"entities_v1_event",
"admin-console-services@platform",
"admin-console-services-history@platform",
"admin-console-services@custom",
"admin-console-services-history@custom",
],
"ignore_missing_component_templates": Array [
"admin-console-services@platform",
"admin-console-services-history@platform",
"admin-console-services@custom",
"admin-console-services-history@custom",
],
"index_patterns": Array [
".entities.v1.history.admin-console-services.*",
],
"name": "entities_v1_history_admin-console-services_index_template",
"priority": 200,
"template": Object {
"mappings": Object {
"_meta": Object {
"version": "1.6.0",
},
"date_detection": false,
"dynamic_templates": Array [
Object {
"strings_as_keyword": Object {
"mapping": Object {
"fields": Object {
"text": Object {
"type": "text",
},
},
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
},
},
Object {
"entity_metrics": Object {
"mapping": Object {
"type": "{dynamic_type}",
},
"match_mapping_type": Array [
"long",
"double",
],
"path_match": "entity.metrics.*",
},
},
],
},
"settings": Object {
"index": Object {
"codec": "best_compression",
"mapping": Object {
"total_fields": Object {
"limit": 2000,
},
},
},
},
},
}
`;

78
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap generated Normal file
View file

@ -0,0 +1,78 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getEntitiesLatestIndexTemplateConfig(definitionId) should generate a valid index template 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",
"ecs_version": "8.0.0",
"managed": true,
"managed_by": "elastic_entity_model",
},
"composed_of": Array [
"entities_v1_latest_base",
"entities_v1_entity",
"entities_v1_event",
"admin-console-services@platform",
"admin-console-services-latest@platform",
"admin-console-services@custom",
"admin-console-services-latest@custom",
],
"ignore_missing_component_templates": Array [
"admin-console-services@platform",
"admin-console-services-latest@platform",
"admin-console-services@custom",
"admin-console-services-latest@custom",
],
"index_patterns": Array [
".entities.v1.latest.admin-console-services",
],
"name": "entities_v1_latest_admin-console-services_index_template",
"priority": 200,
"template": Object {
"mappings": Object {
"_meta": Object {
"version": "1.6.0",
},
"date_detection": false,
"dynamic_templates": Array [
Object {
"strings_as_keyword": Object {
"mapping": Object {
"fields": Object {
"text": Object {
"type": "text",
},
},
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
},
},
Object {
"entity_metrics": Object {
"mapping": Object {
"type": "{dynamic_type}",
},
"match_mapping_type": Array [
"long",
"double",
],
"path_match": "entity.metrics.*",
},
},
],
},
"settings": Object {
"index": Object {
"codec": "best_compression",
"mapping": Object {
"total_fields": Object {
"limit": 2000,
},
},
},
},
},
}
`;

16
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinition } from '../helpers/fixtures/entity_definition';
import { getEntitiesHistoryIndexTemplateConfig } from './entities_history_template';
describe('getEntitiesHistoryIndexTemplateConfig(definitionId)', () => {
it('should generate a valid index template', () => {
const template = getEntitiesHistoryIndexTemplateConfig(entityDefinition.id);
expect(template).toMatchSnapshot();
});
});

8
x-pack/plugins/observability_solution/entity_manager/server/templates/entities_history_template.ts → x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts
View file

@ -6,14 +6,14 @@
*/
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
import { getEntityHistoryIndexTemplateV1 } from '../../common/helpers';
import { getEntityHistoryIndexTemplateV1 } from '../../../../common/helpers';
import {
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
ENTITY_HISTORY_INDEX_PREFIX_V1,
} from '../../common/constants_entities';
import { getCustomHistoryTemplateComponents } from './components/helpers';
} from '../../../../common/constants_entities';
import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers';
export const getEntitiesHistoryIndexTemplateConfig = (
definitionId: string
@ -33,7 +33,7 @@ export const getEntitiesHistoryIndexTemplateConfig = (
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
...getCustomHistoryTemplateComponents(definitionId),
],
index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.*`],
index_patterns: [`${ENTITY_HISTORY_INDEX_PREFIX_V1}.${definitionId}.*`],
priority: 200,
template: {
mappings: {

16
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinition } from '../helpers/fixtures/entity_definition';
import { getEntitiesLatestIndexTemplateConfig } from './entities_latest_template';
describe('getEntitiesLatestIndexTemplateConfig(definitionId)', () => {
it('should generate a valid index template', () => {
const template = getEntitiesLatestIndexTemplateConfig(entityDefinition.id);
expect(template).toMatchSnapshot();
});
});

10
x-pack/plugins/observability_solution/entity_manager/server/templates/entities_latest_template.ts → x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts
View file

@ -6,14 +6,14 @@
*/
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
import { getEntityLatestIndexTemplateV1 } from '../../common/helpers';
import { getEntityLatestIndexTemplateV1 } from '../../../../common/helpers';
import {
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
ENTITY_LATEST_BASE_COMPONENT_TEMPLATE_V1,
ENTITY_LATEST_INDEX_PREFIX_V1,
} from '../../common/constants_entities';
import { getCustomLatestTemplateComponents } from './components/helpers';
} from '../../../../common/constants_entities';
import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers';
export const getEntitiesLatestIndexTemplateConfig = (
definitionId: string
@ -33,8 +33,8 @@ export const getEntitiesLatestIndexTemplateConfig = (
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
...getCustomLatestTemplateComponents(definitionId),
],
index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.*`],
priority: 1,
index_patterns: [`${ENTITY_LATEST_INDEX_PREFIX_V1}.${definitionId}`],
priority: 200,
template: {
mappings: {
_meta: {

4
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap generated
View file

@ -9,7 +9,7 @@ Object {
"defer_validation": true,
"dest": Object {
"index": ".entities.v1.history.noop",
"pipeline": "entities-v1-history-admin-console-services",
"pipeline": "entities-v1-history-admin-console-services-backfill",
},
"frequency": "5m",
"pivot": Object {
@ -143,7 +143,7 @@ Object {
"field": "@timestamp",
},
},
"transform_id": "entities-v1-history-backfill-admin-console-services",
"transform_id": "entities-v1-history-backfill-admin-console-services-backfill",
}
`;

42
x-pack/test/api_integration/apis/entity_manager/definitions.ts Normal file
View file

@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { EntityDefinition } from '@kbn/entities-schema';
import {
entityDefinition as mockDefinition,
entityDefinitionWithBackfill as mockBackfillDefinition,
} from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures';
import { FtrProviderContext } from '../../ftr_provider_context';
import { installDefinition, uninstallDefinition, getInstalledDefinitions } from './helpers/request';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
describe('Entity definitions', () => {
describe('definitions installations', () => {
it('can install multiple definitions', async () => {
await installDefinition(supertest, mockDefinition);
await installDefinition(supertest, mockBackfillDefinition);
const { definitions } = await getInstalledDefinitions(supertest);
expect(definitions.length).to.eql(2);
expect(
definitions.find((definition: EntityDefinition) => definition.id === mockDefinition.id)
);
expect(
definitions.find(
(definition: EntityDefinition) => definition.id === mockBackfillDefinition.id
)
);
await uninstallDefinition(supertest, mockDefinition.id);
await uninstallDefinition(supertest, mockBackfillDefinition.id);
});
});
});
}

26
x-pack/test/api_integration/apis/entity_manager/enablement.ts
View file

@ -11,11 +11,7 @@ import { builtInDefinitions } from '@kbn/entityManager-plugin/server/lib/entitie
import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
import { FtrProviderContext } from '../../ftr_provider_context';
import { createAdmin, createRuntimeUser } from './helpers/user';
interface Auth {
username: string;
password: string;
}
import { Auth, getInstalledDefinitions } from './helpers/request';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
@ -32,16 +28,6 @@ export default function ({ getService }: FtrProviderContext) {
return response.body;
};
const getInstalledDefinitions = async (auth: Auth) => {
const response = await supertest
.get('/internal/entities/definition')
.auth(auth.username, auth.password)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
return response.body;
};
const entityDiscoveryState = enablementRequest('get');
const enableEntityDiscovery = enablementRequest('put');
const disableEntityDiscovery = enablementRequest('delete');
@ -62,7 +48,7 @@ export default function ({ getService }: FtrProviderContext) {
const enableResponse = await enableEntityDiscovery(authorizedUser);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
let definitionsResponse = await getInstalledDefinitions(authorizedUser);
let definitionsResponse = await getInstalledDefinitions(supertest, authorizedUser);
expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
expect(
builtInDefinitions.every((builtin) => {
@ -93,7 +79,7 @@ export default function ({ getService }: FtrProviderContext) {
stateResponse = await entityDiscoveryState(authorizedUser);
expect(stateResponse.enabled).to.eql(false, 'EEM is not disabled');
definitionsResponse = await getInstalledDefinitions(authorizedUser);
definitionsResponse = await getInstalledDefinitions(supertest, authorizedUser);
expect(definitionsResponse.definitions).to.eql([]);
});
});
@ -107,7 +93,7 @@ export default function ({ getService }: FtrProviderContext) {
const stateResponse = await entityDiscoveryState(unauthorizedUser);
expect(stateResponse.enabled).to.eql(false, 'EEM is enabled');
const definitionsResponse = await getInstalledDefinitions(unauthorizedUser);
const definitionsResponse = await getInstalledDefinitions(supertest, unauthorizedUser);
expect(definitionsResponse.definitions).to.eql([]);
});
@ -115,11 +101,11 @@ export default function ({ getService }: FtrProviderContext) {
const enableResponse = await enableEntityDiscovery(authorizedUser);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
let disableResponse = await enableEntityDiscovery(unauthorizedUser);
let disableResponse = await disableEntityDiscovery(unauthorizedUser);
expect(disableResponse.success).to.eql(false, 'unauthorized user can disable EEM');
expect(disableResponse.reason).to.eql(ERROR_USER_NOT_AUTHORIZED);
disableResponse = await enableEntityDiscovery(authorizedUser);
disableResponse = await disableEntityDiscovery(authorizedUser);
expect(disableResponse.success).to.eql(true, "authorized user can't disable EEM");
});
});

39
x-pack/test/api_integration/apis/entity_manager/helpers/request.ts Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Agent } from 'supertest';
import { EntityDefinition } from '@kbn/entities-schema';
export interface Auth {
username: string;
password: string;
}
export const getInstalledDefinitions = async (supertest: Agent, auth?: Auth) => {
let req = supertest.get('/internal/entities/definition').set('kbn-xsrf', 'xxx');
if (auth) {
req = req.auth(auth.username, auth.password);
}
const response = await req.send().expect(200);
return response.body;
};
export const installDefinition = async (supertest: Agent, definition: EntityDefinition) => {
return supertest
.post('/internal/entities/definition')
.set('kbn-xsrf', 'xxx')
.send(definition)
.expect(200);
};
export const uninstallDefinition = (supertest: Agent, id: string) => {
return supertest
.delete(`/internal/entities/definition/${id}`)
.set('kbn-xsrf', 'xxx')
.send()
.expect(200);
};

1
x-pack/test/api_integration/apis/entity_manager/index.ts
View file

@ -12,5 +12,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
this.tags(['entityManager']);
loadTestFile(require.resolve('./enablement'));
loadTestFile(require.resolve('./definitions'));
});
}

3
x-pack/test/tsconfig.json
View file

@ -175,6 +175,7 @@
"@kbn/securitysolution-lists-common",
"@kbn/securitysolution-exceptions-common",
"@kbn/entityManager-plugin",
"@kbn/osquery-plugin"
"@kbn/osquery-plugin",
"@kbn/entities-schema"
]
}