github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[eem] update builtin definitions (#188351)

Browse source 
## Summary

Update built in definitions on plugin start. The update overwrites index
templates and ingest pipelines with the latest versions but has to
delete the transforms since we can only update a subset of settings in
the [update
api](https://www.elastic.co/guide/en/elasticsearch/reference/current/update-transform.html)
which does not include the aggregations.

## Testing
### api tests
Since the upgrade logic happens in plugin startup method we cannot
directly trigger it from api tests without some tweaks. I've added a
[fixture
plugin](a87ae8b210/x-pack/test/api_integration/apis/entity_manager/fixture_plugin/server/plugin.ts)
that is launched in the entity manager test server, this plugin creates
a test route exposing the upgrade api which can then be called in api
tests.

### manual
- install builtin definitions `PUT
kbn:/internal/api/entities/managed/enablement`
- bump builtin [service
definition](https://github.com/elastic/kibana/blob/main/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts#L23)
version
- restart kibana server
- logs should output `[INFO ][plugins.entityManager] Updating built-in
entity definition [builtin_services] from v0.1.0 to v<new version>`
- `GET kbn:/internal/api/entities/definition` should output the new
definition
- verify latest version of definition components are installed

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Kevin Lacabane 2024-08-15 17:00:31 +02:00 • committed by GitHub
parent 66a10805df
commit ae6c1f6213
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
51 changed files with 1174 additions and 492 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/CODEOWNERS vendored
View file

@ -395,6 +395,7 @@ x-pack/plugins/encrypted_saved_objects @elastic/kibana-security
x-pack/plugins/enterprise_search @elastic/search-kibana
x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
x-pack/packages/kbn-entities-schema @elastic/obs-entities
x-pack/test/api_integration/apis/entity_manager/fixture_plugin @elastic/obs-entities
x-pack/plugins/observability_solution/entity_manager @elastic/obs-entities
examples/error_boundary @elastic/appex-sharedux
packages/kbn-es @elastic/kibana-operations

1
package.json
View file

@ -457,6 +457,7 @@
"@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search",
"@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
"@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
"@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
"@kbn/entityManager-plugin": "link:x-pack/plugins/observability_solution/entity_manager",
"@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
"@kbn/es-errors": "link:packages/kbn-es-errors",

2
src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
View file

@ -90,7 +90,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
"endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
"endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
"enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
"entity-definition": "331a2ba0ee9f24936ef049683549c8af7e46f03a",
"entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
"entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
"epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
"epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",

2
tsconfig.base.json
View file

@ -784,6 +784,8 @@
"@kbn/entities-data-access-plugin/*": ["x-pack/plugins/observability_solution/entities_data_access/*"],
"@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"],
"@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
"@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
"@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
"@kbn/entityManager-plugin": ["x-pack/plugins/observability_solution/entity_manager"],
"@kbn/entityManager-plugin/*": ["x-pack/plugins/observability_solution/entity_manager/*"],
"@kbn/error-boundary-example-plugin": ["examples/error_boundary"],

9
x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
View file

@ -48,6 +48,15 @@ export const entityDefinitionSchema = z.object({
),
})
),
installStatus: z.optional(
z.union([
z.literal('installing'),
z.literal('upgrading'),
z.literal('installed'),
z.literal('failed'),
])
),
installStartedAt: z.optional(z.string()),
});
export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

1
x-pack/plugins/observability_solution/entity_manager/common/errors.ts
View file

@ -10,3 +10,4 @@ export const ERROR_API_KEY_NOT_VALID = 'api_key_not_valid';
export const ERROR_API_KEY_SERVICE_DISABLED = 'api_key_service_disabled';
export const ERROR_PARTIAL_BUILTIN_INSTALLATION = 'partial_builtin_installation';
export const ERROR_DEFINITION_STOPPED = 'error_definition_stopped';
export const ERROR_BUILTIN_UPGRADE_REQUIRED = 'builtin_upgrade_required';

22
x-pack/plugins/observability_solution/entity_manager/common/helpers.test.ts
View file

@ -1,22 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getEntityHistoryIndexTemplateV1, getEntityLatestIndexTemplateV1 } from './helpers';
describe('helpers', () => {
it('getEntityHistoryIndexTemplateV1 should return the correct value', () => {
const definitionId = 'test';
const result = getEntityHistoryIndexTemplateV1(definitionId);
expect(result).toEqual('entities_v1_history_test_index_template');
});
it('getEntityLatestIndexTemplateV1 should return the correct value', () => {
const definitionId = 'test';
const result = getEntityLatestIndexTemplateV1(definitionId);
expect(result).toEqual('entities_v1_latest_test_index_template');
});
});

19
x-pack/plugins/observability_solution/entity_manager/common/helpers.ts
View file

@ -1,19 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
ENTITY_BASE_PREFIX,
ENTITY_SCHEMA_VERSION_V1,
ENTITY_HISTORY,
ENTITY_LATEST,
} from '@kbn/entities-schema';
export const getEntityHistoryIndexTemplateV1 = (definitionId: string) =>
`${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definitionId}_index_template` as const;
export const getEntityLatestIndexTemplateV1 = (definitionId: string) =>
`${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definitionId}_index_template` as const;

2
x-pack/plugins/observability_solution/entity_manager/kibana.jsonc
View file

@ -6,7 +6,7 @@
"plugin": {
"id": "entityManager",
"configPath": ["xpack", "entityManager"],
"requiredPlugins": ["security", "encryptedSavedObjects"],
"requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
"browser": true,
"server": true,
"requiredBundles": []

23
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_entity_definition.ts
View file

@ -5,27 +5,22 @@
* 2.0.
*/
import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
import { SavedObjectsClientContract, SavedObjectsErrorHelpers } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import { EntityDefinitionNotFound } from './errors/entity_not_found';
export async function deleteEntityDefinition(
soClient: SavedObjectsClientContract,
definition: EntityDefinition,
logger: Logger
definition: EntityDefinition
) {
const response = await soClient.find<EntityDefinition>({
type: SO_ENTITY_DEFINITION_TYPE,
page: 1,
perPage: 1,
filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
});
try {
await soClient.delete(SO_ENTITY_DEFINITION_TYPE, definition.id);
} catch (err) {
if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
}
if (response.total === 0) {
logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
throw err;
}
await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
}

29
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts
View file

@ -11,12 +11,16 @@ import { EntityDefinition } from '@kbn/entities-schema';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import {
generateHistoryTransformId,
generateHistoryBackfillTransformId,
generateHistoryIngestPipelineId,
generateHistoryIndexTemplateId,
generateLatestTransformId,
generateLatestIngestPipelineId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import { BUILT_IN_ID_PREFIX } from './built_in';
import { EntityDefinitionWithState } from './types';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
export async function findEntityDefinitions({
soClient,
@ -60,19 +64,32 @@ async function getEntityDefinitionState(
) {
const historyIngestPipelineId = generateHistoryIngestPipelineId(definition);
const latestIngestPipelineId = generateLatestIngestPipelineId(definition);
const [ingestPipelines, transforms] = await Promise.all([
esClient.ingest.getPipeline({
id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
const transformIds = [
generateHistoryTransformId(definition),
generateLatestTransformId(definition),
...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []),
];
const [ingestPipelines, indexTemplatesInstalled, transforms] = await Promise.all([
esClient.ingest.getPipeline(
{
id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
},
{ ignore: [404] }
),
esClient.indices.existsIndexTemplate({
name: `${
(generateLatestIndexTemplateId(definition), generateHistoryIndexTemplateId(definition))
}`,
}),
esClient.transform.getTransformStats({
transform_id: [generateHistoryTransformId(definition), generateLatestTransformId(definition)],
transform_id: transformIds,
}),
]);
const ingestPipelinesInstalled = !!(
ingestPipelines[historyIngestPipelineId] && ingestPipelines[latestIngestPipelineId]
);
const transformsInstalled = transforms.count === 2;
const transformsInstalled = transforms.count === transformIds.length;
const transformsRunning =
transformsInstalled &&
transforms.transforms.every(
@ -80,7 +97,7 @@ async function getEntityDefinitionState(
);
return {
installed: ingestPipelinesInstalled && transformsInstalled,
installed: ingestPipelinesInstalled && transformsInstalled && indexTemplatesInstalled,
running: transformsRunning,
};
}

24
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { entityDefinitionSchema } from '@kbn/entities-schema';
export const builtInEntityDefinition = entityDefinitionSchema.parse({
id: 'builtin_mock_entity_definition',
version: '1.0.0',
name: 'Mock builtin definition',
type: 'service',
indexPatterns: ['kbn-data-forge-fake_stack.*'],
managed: true,
history: {
timestampField: '@timestamp',
interval: '1m',
},
identityFields: ['log.logger', { field: 'event.category', optional: true }],
displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',
metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }],
metrics: [],
});

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts
View file

@ -8,7 +8,7 @@
import { entityDefinitionSchema } from '@kbn/entities-schema';
export const rawEntityDefinition = {
id: 'admin-console-services',
version: '999.999.999',
version: '1.0.0',
name: 'Services for Admin Console',
type: 'service',
indexPatterns: ['kbn-data-forge-fake_stack.*'],

1
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/index.ts
View file

@ -7,3 +7,4 @@
export { entityDefinition } from './entity_definition';
export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';
export { builtInEntityDefinition } from './builtin_entity_definition';

8
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts
View file

@ -6,6 +6,7 @@
*/
import {
ENTITY_BASE_PREFIX,
ENTITY_HISTORY,
ENTITY_LATEST,
ENTITY_SCHEMA_VERSION_V1,
@ -38,6 +39,10 @@ export function generateHistoryIndexName(definition: EntityDefinition) {
});
}
export function generateHistoryIndexTemplateId(definition: EntityDefinition) {
return `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definition.id}_index_template` as const;
}
// Latest
function generateLatestId(definition: EntityDefinition) {
return `${ENTITY_LATEST_PREFIX_V1}-${definition.id}` as const;
@ -53,3 +58,6 @@ export function generateLatestIndexName(definition: EntityDefinition) {
definitionId: definition.id,
});
}
export const generateLatestIndexTemplateId = (definition: EntityDefinition) =>
`${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definition.id}_index_template` as const;

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap generated
View file

@ -23,7 +23,7 @@ Array [
Object {
"set": Object {
"field": "entity.definitionVersion",
"value": "999.999.999",
"value": "1.0.0",
},
},
Object {

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap generated
View file

@ -23,7 +23,7 @@ Array [
Object {
"set": Object {
"field": "entity.definitionVersion",
"value": "999.999.999",
"value": "1.0.0",
},
},
Object {

401
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts
View file

@ -5,23 +5,32 @@
* 2.0.
*/
import semver from 'semver';
import moment from 'moment';
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
import { loggerMock } from '@kbn/logging-mocks';
import { EntityDefinition } from '@kbn/entities-schema';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { installBuiltInEntityDefinitions } from './install_entity_definition';
import { builtInServicesFromLogsEntityDefinition } from './built_in/services';
import {
installBuiltInEntityDefinitions,
installEntityDefinition,
} from './install_entity_definition';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import {
generateHistoryIndexTemplateId,
generateHistoryIngestPipelineId,
generateHistoryTransformId,
generateLatestIndexTemplateId,
generateLatestIngestPipelineId,
generateLatestTransformId,
} from './helpers/generate_component_id';
import { generateHistoryTransform } from './transform/generate_history_transform';
import { generateLatestTransform } from './transform/generate_latest_transform';
import { entityDefinition as mockEntityDefinition } from './helpers/fixtures/entity_definition';
import { EntityDefinitionIdInvalid } from './errors/entity_definition_id_invalid';
import { EntityIdConflict } from './errors/entity_id_conflict_error';
const assertHasCreatedDefinition = (
definition: EntityDefinition,
@ -29,10 +38,22 @@ const assertHasCreatedDefinition = (
esClient: ElasticsearchClient
) => {
expect(soClient.create).toBeCalledTimes(1);
expect(soClient.create).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition, {
id: definition.id,
overwrite: true,
managed: definition.managed,
expect(soClient.create).toBeCalledWith(
SO_ENTITY_DEFINITION_TYPE,
{
...definition,
installStatus: 'installing',
installStartedAt: expect.any(String),
},
{
id: definition.id,
overwrite: true,
managed: definition.managed,
}
);
expect(soClient.update).toBeCalledTimes(1);
expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, {
installStatus: 'installed',
});
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
@ -49,51 +70,119 @@ const assertHasCreatedDefinition = (
expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition),
id: generateHistoryIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: '1.0.0',
managed: true,
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateLatestIngestPipelineId(builtInServicesFromLogsEntityDefinition),
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: '1.0.0',
managed: true,
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.transform.putTransform).toBeCalledTimes(2);
expect(esClient.transform.putTransform).toBeCalledWith(
generateHistoryTransform(builtInServicesFromLogsEntityDefinition)
);
expect(esClient.transform.putTransform).toBeCalledWith(
generateLatestTransform(builtInServicesFromLogsEntityDefinition)
);
expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition));
expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition));
};
const assertHasStartedTransform = (definition: EntityDefinition, esClient: ElasticsearchClient) => {
expect(esClient.transform.startTransform).toBeCalledTimes(2);
expect(esClient.transform.startTransform).toBeCalledWith(
{
transform_id: generateHistoryTransformId(builtInServicesFromLogsEntityDefinition),
},
expect.anything()
);
expect(esClient.transform.startTransform).toBeCalledWith(
{
transform_id: generateLatestTransformId(builtInServicesFromLogsEntityDefinition),
},
expect.anything()
);
};
const assertHasUninstalledDefinition = (
const assertHasUpgradedDefinition = (
definition: EntityDefinition,
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient
) => {
expect(soClient.update).toBeCalledTimes(2);
expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, {
...definition,
installStatus: 'upgrading',
installStartedAt: expect.any(String),
});
expect(soClient.update).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id, {
installStatus: 'installed',
});
expect(esClient.indices.putIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_history_${definition.id}_index_template`,
})
);
expect(esClient.indices.putIndexTemplate).toBeCalledWith(
expect.objectContaining({
name: `entities_v1_latest_${definition.id}_index_template`,
})
);
expect(esClient.ingest.putPipeline).toBeCalledTimes(2);
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateHistoryIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.ingest.putPipeline).toBeCalledWith({
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
managed: definition.managed,
},
});
expect(esClient.transform.putTransform).toBeCalledTimes(2);
expect(esClient.transform.putTransform).toBeCalledWith(generateHistoryTransform(definition));
expect(esClient.transform.putTransform).toBeCalledWith(generateLatestTransform(definition));
};
const assertHasDeletedDefinition = (
definition: EntityDefinition,
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient
) => {
assertHasDeletedTransforms(definition, esClient);
expect(esClient.ingest.deletePipeline).toBeCalledTimes(2);
expect(esClient.ingest.deletePipeline).toBeCalledWith(
{
id: generateHistoryIngestPipelineId(definition),
},
{ ignore: [404] }
);
expect(esClient.ingest.deletePipeline).toBeCalledWith(
{
id: generateLatestIngestPipelineId(definition),
},
{ ignore: [404] }
);
expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: generateHistoryIndexTemplateId(definition),
},
{ ignore: [404] }
);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: generateLatestIndexTemplateId(definition),
},
{ ignore: [404] }
);
expect(soClient.delete).toBeCalledTimes(1);
expect(soClient.delete).toBeCalledWith(SO_ENTITY_DEFINITION_TYPE, definition.id);
};
const assertHasDeletedTransforms = (
definition: EntityDefinition,
esClient: ElasticsearchClient
) => {
expect(esClient.transform.stopTransform).toBeCalledTimes(2);
expect(esClient.transform.stopTransform).toBeCalledWith(
@ -122,28 +211,92 @@ const assertHasUninstalledDefinition = (
);
expect(esClient.transform.deleteTransform).toBeCalledTimes(2);
expect(esClient.ingest.deletePipeline).toBeCalledTimes(2);
expect(soClient.delete).toBeCalledTimes(1);
expect(esClient.indices.deleteIndexTemplate).toBeCalledTimes(2);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: `entities_v1_history_${definition.id}_index_template`,
},
{ ignore: [404] }
);
expect(esClient.indices.deleteIndexTemplate).toBeCalledWith(
{
name: `entities_v1_latest_${definition.id}_index_template`,
},
{ ignore: [404] }
);
};
describe('install_entity_definition', () => {
describe('installEntityDefinition', () => {
it('should reject invalid ids', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
await expect(
installEntityDefinition({
esClient,
soClient,
definition: { id: 'a'.repeat(40) } as EntityDefinition,
logger: loggerMock.create(),
})
).rejects.toThrow(EntityDefinitionIdInvalid);
});
it('should reject if id already exists', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValueOnce({
saved_objects: [
{
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
score: 0,
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
},
},
],
total: 1,
page: 1,
per_page: 10,
});
await expect(
installEntityDefinition({
esClient,
soClient,
definition: mockEntityDefinition,
logger: loggerMock.create(),
})
).rejects.toThrow(EntityIdConflict);
});
it('should install a definition', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
await installEntityDefinition({
esClient,
soClient,
definition: mockEntityDefinition,
logger: loggerMock.create(),
});
assertHasCreatedDefinition(mockEntityDefinition, soClient, esClient);
});
it('should rollback the installation on failure', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
esClient.transform.putTransform.mockRejectedValue(new Error('cannot install transform'));
await expect(
installEntityDefinition({
esClient,
soClient,
definition: mockEntityDefinition,
logger: loggerMock.create(),
})
).rejects.toThrow(/cannot install transform/);
assertHasDeletedDefinition(mockEntityDefinition, soClient, esClient);
});
});
describe('installBuiltInEntityDefinitions', () => {
it('should install and start definition when not found', async () => {
const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
it('should install definition when not found', async () => {
const builtInDefinitions = [mockEntityDefinition];
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
@ -151,94 +304,158 @@ describe('install_entity_definition', () => {
await installBuiltInEntityDefinitions({
esClient,
soClient,
builtInDefinitions,
definitions: builtInDefinitions,
logger: loggerMock.create(),
});
assertHasCreatedDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
assertHasCreatedDefinition(mockEntityDefinition, soClient, esClient);
});
it('should reinstall when partial state found', async () => {
const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
const builtInDefinitions = [mockEntityDefinition];
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
// mock partially installed definition
esClient.ingest.getPipeline.mockResolvedValue({});
esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
const soClient = savedObjectsClientMock.create();
const definitionSOResult = {
soClient.find.mockResolvedValueOnce({
saved_objects: [
{
id: builtInServicesFromLogsEntityDefinition.id,
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
score: 0,
attributes: builtInServicesFromLogsEntityDefinition,
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
},
},
],
total: 1,
page: 1,
per_page: 10,
};
soClient.find
.mockResolvedValueOnce(definitionSOResult)
.mockResolvedValueOnce(definitionSOResult)
.mockResolvedValueOnce({
saved_objects: [],
total: 0,
page: 1,
per_page: 10,
});
});
await installBuiltInEntityDefinitions({
esClient,
soClient,
builtInDefinitions,
definitions: builtInDefinitions,
logger: loggerMock.create(),
});
assertHasUninstalledDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
assertHasCreatedDefinition(builtInServicesFromLogsEntityDefinition, soClient, esClient);
assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(mockEntityDefinition, soClient, esClient);
});
it('should start a stopped definition', async () => {
const builtInDefinitions = [builtInServicesFromLogsEntityDefinition];
it('should reinstall when outdated version', async () => {
const updatedDefinition = {
...mockEntityDefinition,
version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
};
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
// mock installed but stopped definition
esClient.ingest.getPipeline.mockResolvedValue({
[generateHistoryIngestPipelineId(builtInServicesFromLogsEntityDefinition)]: {},
[generateLatestIngestPipelineId(builtInServicesFromLogsEntityDefinition)]: {},
});
esClient.transform.getTransformStats.mockResolvedValue({
// @ts-expect-error
transforms: [{ state: 'stopped' }, { state: 'stopped' }],
count: 2,
});
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({
soClient.find.mockResolvedValueOnce({
saved_objects: [
{
id: builtInServicesFromLogsEntityDefinition.id,
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
score: 0,
attributes: builtInServicesFromLogsEntityDefinition,
attributes: {
...mockEntityDefinition,
installStatus: 'installed',
},
},
],
total: 1,
page: 1,
per_page: 10,
});
await installBuiltInEntityDefinitions({
esClient,
soClient,
builtInDefinitions,
definitions: [updatedDefinition],
logger: loggerMock.create(),
});
expect(soClient.create).toHaveBeenCalledTimes(0);
assertHasStartedTransform(builtInServicesFromLogsEntityDefinition, esClient);
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(updatedDefinition, soClient, esClient);
});
it('should reinstall when stale upgrade', async () => {
const updatedDefinition = {
...mockEntityDefinition,
version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
};
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValueOnce({
saved_objects: [
{
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
score: 0,
attributes: {
...mockEntityDefinition,
// upgrading for 1h
installStatus: 'upgrading',
installStartedAt: moment().subtract(1, 'hour').toISOString(),
},
},
],
total: 1,
page: 1,
per_page: 10,
});
await installBuiltInEntityDefinitions({
esClient,
soClient,
definitions: [updatedDefinition],
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(updatedDefinition, soClient, esClient);
});
it('should reinstall when failed installation', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValueOnce({
saved_objects: [
{
id: mockEntityDefinition.id,
type: 'entity-definition',
references: [],
score: 0,
attributes: {
...mockEntityDefinition,
installStatus: 'failed',
installStartedAt: new Date().toISOString(),
},
},
],
total: 1,
page: 1,
per_page: 10,
});
await installBuiltInEntityDefinitions({
esClient,
soClient,
definitions: [mockEntityDefinition],
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(mockEntityDefinition, soClient, esClient);
});
});
});

327
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts
View file

@ -5,14 +5,15 @@
* 2.0.
*/
import semver from 'semver';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { EntityDefinition } from '@kbn/entities-schema';
import { Logger } from '@kbn/logging';
import {
getEntityHistoryIndexTemplateV1,
getEntityLatestIndexTemplateV1,
} from '../../../common/helpers';
generateHistoryIndexTemplateId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import {
createAndInstallHistoryIngestPipeline,
createAndInstallLatestIngestPipeline,
@ -26,18 +27,23 @@ import { validateDefinitionCanCreateValidTransformIds } from './transform/valida
import { deleteEntityDefinition } from './delete_entity_definition';
import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
import { findEntityDefinitions } from './find_entity_definition';
import { saveEntityDefinition } from './save_entity_definition';
import { startTransform } from './start_transform';
import {
entityDefinitionExists,
saveEntityDefinition,
updateEntityDefinition,
} from './save_entity_definition';
import {
stopAndDeleteHistoryBackfillTransform,
stopAndDeleteHistoryTransform,
stopAndDeleteLatestTransform,
} from './stop_and_delete_transform';
import { uninstallEntityDefinition } from './uninstall_entity_definition';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import { deleteTemplate, upsertTemplate } from '../manage_index_templates';
import { getEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
import { getEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_latest_template';
import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template';
import { EntityIdConflict } from './errors/entity_id_conflict_error';
import { EntityDefinitionNotFound } from './errors/entity_not_found';
import { EntityDefinitionWithState } from './types';
export interface InstallDefinitionParams {
esClient: ElasticsearchClient;
@ -46,110 +52,77 @@ export interface InstallDefinitionParams {
logger: Logger;
}
const throwIfRejected = (values: Array<PromiseFulfilledResult<any> | PromiseRejectedResult>) => {
const rejectedPromise = values.find(
(value) => value.status === 'rejected'
) as PromiseRejectedResult;
if (rejectedPromise) {
throw new Error(rejectedPromise.reason);
}
return values;
};
// install an entity definition from scratch with all its required components
// after verifying that the definition id is valid and available.
// attempt to remove all installed components if the installation fails.
export async function installEntityDefinition({
esClient,
soClient,
definition,
logger,
}: InstallDefinitionParams): Promise<EntityDefinition> {
const installState = {
ingestPipelines: {
history: false,
latest: false,
},
transforms: {
history: false,
backfill: false,
latest: false,
},
definition: false,
indexTemplates: {
history: false,
latest: false,
},
};
validateDefinitionCanCreateValidTransformIds(definition);
try {
logger.debug(() => `Installing definition ${JSON.stringify(definition)}`);
validateDefinitionCanCreateValidTransformIds(definition);
const entityDefinition = await saveEntityDefinition(soClient, definition);
installState.definition = true;
// install scoped index template
await upsertTemplate({
esClient,
logger,
template: getEntitiesHistoryIndexTemplateConfig(definition),
});
installState.indexTemplates.history = true;
await upsertTemplate({
esClient,
logger,
template: getEntitiesLatestIndexTemplateConfig(definition),
});
installState.indexTemplates.latest = true;
// install ingest pipelines
logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
await createAndInstallHistoryIngestPipeline(esClient, entityDefinition, logger);
installState.ingestPipelines.history = true;
await createAndInstallLatestIngestPipeline(esClient, entityDefinition, logger);
installState.ingestPipelines.latest = true;
// install transforms
logger.debug(`Installing transforms for definition ${definition.id}`);
await createAndInstallHistoryTransform(esClient, entityDefinition, logger);
installState.transforms.history = true;
if (isBackfillEnabled(entityDefinition)) {
await createAndInstallHistoryBackfillTransform(esClient, entityDefinition, logger);
installState.transforms.backfill = true;
if (await entityDefinitionExists(soClient, definition.id)) {
throw new EntityIdConflict(
`Entity definition with [${definition.id}] already exists.`,
definition
);
}
await createAndInstallLatestTransform(esClient, entityDefinition, logger);
installState.transforms.latest = true;
return entityDefinition;
const entityDefinition = await saveEntityDefinition(soClient, {
...definition,
installStatus: 'installing',
installStartedAt: new Date().toISOString(),
});
return await install({ esClient, soClient, logger, definition: entityDefinition });
} catch (e) {
logger.error(`Failed to install entity definition ${definition.id}: ${e}`);
// Clean up anything that was successful.
if (installState.definition) {
await deleteEntityDefinition(soClient, definition, logger);
}
if (installState.ingestPipelines.history) {
await deleteHistoryIngestPipeline(esClient, definition, logger);
}
if (installState.ingestPipelines.latest) {
await deleteLatestIngestPipeline(esClient, definition, logger);
}
await Promise.all([
stopAndDeleteHistoryTransform(esClient, definition, logger),
isBackfillEnabled(definition)
? stopAndDeleteHistoryBackfillTransform(esClient, definition, logger)
: Promise.resolve(),
stopAndDeleteLatestTransform(esClient, definition, logger),
]);
if (installState.transforms.history) {
await stopAndDeleteHistoryTransform(esClient, definition, logger);
}
await Promise.all([
deleteHistoryIngestPipeline(esClient, definition, logger),
deleteLatestIngestPipeline(esClient, definition, logger),
]);
if (installState.transforms.backfill) {
await stopAndDeleteHistoryBackfillTransform(esClient, definition, logger);
}
if (installState.transforms.latest) {
await stopAndDeleteLatestTransform(esClient, definition, logger);
}
if (installState.indexTemplates.history) {
await deleteTemplate({
await Promise.all([
deleteTemplate({
esClient,
logger,
name: getEntityHistoryIndexTemplateV1(definition.id),
});
}
if (installState.indexTemplates.latest) {
await deleteTemplate({
name: generateHistoryIndexTemplateId(definition),
}),
deleteTemplate({
esClient,
logger,
name: getEntityLatestIndexTemplateV1(definition.id),
});
}
name: generateLatestIndexTemplateId(definition),
}),
]);
await deleteEntityDefinition(soClient, definition).catch((err) => {
if (err instanceof EntityDefinitionNotFound) {
return;
}
throw err;
});
throw e;
}
@ -159,62 +132,158 @@ export async function installBuiltInEntityDefinitions({
esClient,
soClient,
logger,
builtInDefinitions,
installOnly,
definitions,
}: Omit<InstallDefinitionParams, 'definition'> & {
builtInDefinitions: EntityDefinition[];
installOnly?: boolean;
definitions: EntityDefinition[];
}): Promise<EntityDefinition[]> {
if (builtInDefinitions.length === 0) return [];
if (definitions.length === 0) return [];
logger.debug(`Starting installation of ${builtInDefinitions.length} built-in definitions`);
const installPromises = builtInDefinitions.map(async (builtInDefinition) => {
const definitions = await findEntityDefinitions({
logger.debug(`Starting installation of ${definitions.length} built-in definitions`);
const installPromises = definitions.map(async (builtInDefinition) => {
const installedDefinitions = await findEntityDefinitions({
esClient,
soClient,
id: builtInDefinition.id,
});
if (definitions.length === 0) {
return await installAndStartDefinition({
if (installedDefinitions.length === 0) {
return await installEntityDefinition({
definition: builtInDefinition,
esClient,
soClient,
logger,
installOnly,
});
}
const definition = definitions[0];
// verify current installation
if (!definition.state.installed) {
logger.debug(`Detected partial installation of definition [${definition.id}], reinstalling`);
await uninstallEntityDefinition({ esClient, soClient, logger, definition });
return await installAndStartDefinition({
definition: builtInDefinition,
esClient,
soClient,
logger,
installOnly,
});
// verify existing installation
const installedDefinition = installedDefinitions[0];
if (!shouldReinstall(installedDefinition, builtInDefinition)) {
return installedDefinition;
}
if (!definition.state.running) {
logger.debug(`Starting transforms for definition [${definition.id}]`);
await startTransform(esClient, definition, logger);
}
return definition;
logger.debug(
`Detected failed or outdated installation of definition [${installedDefinition.id}] v${installedDefinition.version}, installing v${builtInDefinition.version}`
);
return await reinstall({
soClient,
esClient,
logger,
definition: installedDefinition,
latestDefinition: builtInDefinition,
});
});
return await Promise.all(installPromises);
}
async function installAndStartDefinition(
params: InstallDefinitionParams & { installOnly?: boolean }
) {
const definition = await installEntityDefinition(params);
if (!params.installOnly) {
await startTransform(params.esClient, definition, params.logger);
}
return definition;
// perform installation of an entity definition components.
// assume definition saved object is already persisted
async function install({
esClient,
soClient,
definition,
logger,
}: InstallDefinitionParams): Promise<EntityDefinition> {
logger.debug(
() =>
`Installing definition ${definition.id} v${definition.version}\n${JSON.stringify(
definition,
null,
2
)}`
);
logger.debug(`Installing index templates for definition ${definition.id}`);
await Promise.allSettled([
upsertTemplate({
esClient,
logger,
template: generateEntitiesHistoryIndexTemplateConfig(definition),
}),
upsertTemplate({
esClient,
logger,
template: generateEntitiesLatestIndexTemplateConfig(definition),
}),
]).then(throwIfRejected);
logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
await Promise.allSettled([
createAndInstallHistoryIngestPipeline(esClient, definition, logger),
createAndInstallLatestIngestPipeline(esClient, definition, logger),
]).then(throwIfRejected);
logger.debug(`Installing transforms for definition ${definition.id}`);
await Promise.allSettled([
createAndInstallHistoryTransform(esClient, definition, logger),
isBackfillEnabled(definition)
? createAndInstallHistoryBackfillTransform(esClient, definition, logger)
: Promise.resolve(),
createAndInstallLatestTransform(esClient, definition, logger),
]).then(throwIfRejected);
await updateEntityDefinition(soClient, definition.id, { installStatus: 'installed' });
return { ...definition, installStatus: 'installed' };
}
// stop and delete the current transforms and reinstall all the components
async function reinstall({
esClient,
soClient,
definition,
latestDefinition,
logger,
}: InstallDefinitionParams & { latestDefinition: EntityDefinition }): Promise<EntityDefinition> {
logger.debug(
`Reinstalling definition ${definition.id} from v${definition.version} to v${latestDefinition.version}`
);
try {
await updateEntityDefinition(soClient, latestDefinition.id, {
...latestDefinition,
installStatus: 'upgrading',
installStartedAt: new Date().toISOString(),
});
logger.debug(`Stopping transforms for definition ${definition.id} v${definition.version}`);
await Promise.all([
stopAndDeleteHistoryTransform(esClient, definition, logger),
isBackfillEnabled(definition)
? stopAndDeleteHistoryBackfillTransform(esClient, definition, logger)
: Promise.resolve(),
stopAndDeleteLatestTransform(esClient, definition, logger),
]);
return await install({
esClient,
soClient,
logger,
definition: latestDefinition,
});
} catch (err) {
await updateEntityDefinition(soClient, latestDefinition.id, {
installStatus: 'failed',
});
throw err;
}
}
const INSTALLATION_TIMEOUT = 5 * 60 * 1000;
const shouldReinstall = (
definition: EntityDefinitionWithState,
latestDefinition: EntityDefinition
) => {
const { installStatus, installStartedAt } = definition;
const isStale =
(installStatus === 'installing' || installStatus === 'upgrading') &&
Date.now() - Date.parse(installStartedAt!) >= INSTALLATION_TIMEOUT;
const isOutdated =
installStatus === 'installed' && semver.neq(definition.version, latestDefinition.version);
const isFailed = installStatus === 'failed';
const isPartial = installStatus === 'installed' && !definition.state.installed;
return isStale || isOutdated || isFailed || isPartial;
};

37
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/save_entity_definition.ts
View file

@ -8,26 +8,11 @@
import { SavedObjectsClientContract } from '@kbn/core/server';
import { EntityDefinition } from '@kbn/entities-schema';
import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
import { EntityIdConflict } from './errors/entity_id_conflict_error';
export async function saveEntityDefinition(
soClient: SavedObjectsClientContract,
definition: EntityDefinition
): Promise<EntityDefinition> {
const response = await soClient.find<EntityDefinition>({
type: SO_ENTITY_DEFINITION_TYPE,
page: 1,
perPage: 1,
filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
});
if (response.total === 1) {
throw new EntityIdConflict(
`Entity definition with [${definition.id}] already exists.`,
definition
);
}
await soClient.create<EntityDefinition>(SO_ENTITY_DEFINITION_TYPE, definition, {
id: definition.id,
managed: definition.managed,
@ -36,3 +21,25 @@ export async function saveEntityDefinition(
return definition;
}
export async function entityDefinitionExists(
soClient: SavedObjectsClientContract,
id: string
): Promise<boolean> {
const response = await soClient.find<EntityDefinition>({
type: SO_ENTITY_DEFINITION_TYPE,
page: 1,
perPage: 1,
filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${id})`,
});
return response.total === 1;
}
export async function updateEntityDefinition(
soClient: SavedObjectsClientContract,
id: string,
definition: Partial<EntityDefinition>
) {
await soClient.update<EntityDefinition>(SO_ENTITY_DEFINITION_TYPE, id, definition);
}

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_history_template.test.ts.snap generated
View file

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getEntitiesHistoryIndexTemplateConfig(definitionId) should generate a valid index template 1`] = `
exports[`generateEntitiesHistoryIndexTemplateConfig(definition) should generate a valid index template 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/__snapshots__/entities_latest_template.test.ts.snap generated
View file

@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getEntitiesLatestIndexTemplateConfig(definitionId) should generate a valid index template 1`] = `
exports[`generateEntitiesLatestIndexTemplateConfig(definition) should generate a valid index template 1`] = `
Object {
"_meta": Object {
"description": "Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",

6
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.test.ts
View file

@ -6,11 +6,11 @@
*/
import { entityDefinition } from '../helpers/fixtures/entity_definition';
import { getEntitiesHistoryIndexTemplateConfig } from './entities_history_template';
import { generateEntitiesHistoryIndexTemplateConfig } from './entities_history_template';
describe('getEntitiesHistoryIndexTemplateConfig(definitionId)', () => {
describe('generateEntitiesHistoryIndexTemplateConfig(definition)', () => {
it('should generate a valid index template', () => {
const template = getEntitiesHistoryIndexTemplateConfig(entityDefinition);
const template = generateEntitiesHistoryIndexTemplateConfig(entityDefinition);
expect(template).toMatchSnapshot();
});
});

6
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_history_template.ts
View file

@ -13,7 +13,7 @@ import {
entitiesAliasPattern,
ENTITY_SCHEMA_VERSION_V1,
} from '@kbn/entities-schema';
import { getEntityHistoryIndexTemplateV1 } from '../../../../common/helpers';
import { generateHistoryIndexTemplateId } from '../helpers/generate_component_id';
import {
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
@ -21,10 +21,10 @@ import {
} from '../../../../common/constants_entities';
import { getCustomHistoryTemplateComponents } from '../../../templates/components/helpers';
export const getEntitiesHistoryIndexTemplateConfig = (
export const generateEntitiesHistoryIndexTemplateConfig = (
definition: EntityDefinition
): IndicesPutIndexTemplateRequest => ({
name: getEntityHistoryIndexTemplateV1(definition.id),
name: generateHistoryIndexTemplateId(definition),
_meta: {
description:
"Index template for indices managed by the Elastic Entity Model's entity discovery framework for the history dataset",

6
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.test.ts
View file

@ -6,11 +6,11 @@
*/
import { entityDefinition } from '../helpers/fixtures/entity_definition';
import { getEntitiesLatestIndexTemplateConfig } from './entities_latest_template';
import { generateEntitiesLatestIndexTemplateConfig } from './entities_latest_template';
describe('getEntitiesLatestIndexTemplateConfig(definitionId)', () => {
describe('generateEntitiesLatestIndexTemplateConfig(definition)', () => {
it('should generate a valid index template', () => {
const template = getEntitiesLatestIndexTemplateConfig(entityDefinition);
const template = generateEntitiesLatestIndexTemplateConfig(entityDefinition);
expect(template).toMatchSnapshot();
});
});

6
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/templates/entities_latest_template.ts
View file

@ -13,7 +13,7 @@ import {
entitiesIndexPattern,
entitiesAliasPattern,
} from '@kbn/entities-schema';
import { getEntityLatestIndexTemplateV1 } from '../../../../common/helpers';
import { generateLatestIndexTemplateId } from '../helpers/generate_component_id';
import {
ENTITY_ENTITY_COMPONENT_TEMPLATE_V1,
ENTITY_EVENT_COMPONENT_TEMPLATE_V1,
@ -21,10 +21,10 @@ import {
} from '../../../../common/constants_entities';
import { getCustomLatestTemplateComponents } from '../../../templates/components/helpers';
export const getEntitiesLatestIndexTemplateConfig = (
export const generateEntitiesLatestIndexTemplateConfig = (
definition: EntityDefinition
): IndicesPutIndexTemplateRequest => ({
name: getEntityLatestIndexTemplateV1(definition.id),
name: generateLatestIndexTemplateId(definition),
_meta: {
description:
"Index template for indices managed by the Elastic Entity Model's entity discovery framework for the latest dataset",

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap generated
View file

@ -155,7 +155,7 @@ Object {
exports[`generateHistoryTransform(definition) should generate a valid history transform 1`] = `
Object {
"_meta": Object {
"definitionVersion": "999.999.999",
"definitionVersion": "1.0.0",
"managed": false,
},
"defer_validation": true,

2
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap generated
View file

@ -3,7 +3,7 @@
exports[`generateLatestTransform(definition) should generate a valid latest transform 1`] = `
Object {
"_meta": Object {
"definitionVersion": "999.999.999",
"definitionVersion": "1.0.0",
"managed": false,
},
"defer_validation": true,

9
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/validate_transform_ids.ts
View file

@ -8,6 +8,7 @@
import { EntityDefinition } from '@kbn/entities-schema';
import { EntityDefinitionIdInvalid } from '../errors/entity_definition_id_invalid';
import {
generateHistoryBackfillTransformId,
generateHistoryTransformId,
generateLatestTransformId,
} from '../helpers/generate_component_id';
@ -17,9 +18,15 @@ const TRANSFORM_ID_MAX_LENGTH = 64;
export function validateDefinitionCanCreateValidTransformIds(definition: EntityDefinition) {
const historyTransformId = generateHistoryTransformId(definition);
const latestTransformId = generateLatestTransformId(definition);
const historyBackfillTransformId = generateHistoryBackfillTransformId(definition);
const spareChars =
TRANSFORM_ID_MAX_LENGTH - Math.max(historyTransformId.length, latestTransformId.length);
TRANSFORM_ID_MAX_LENGTH -
Math.max(
historyTransformId.length,
latestTransformId.length,
historyBackfillTransformId.length
);
if (spareChars < 0) {
throw new EntityDefinitionIdInvalid(

4
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts
View file

@ -7,6 +7,10 @@
import { EntityDefinition } from '@kbn/entities-schema';
// state is the *live* state of the definition. since a definition
// is composed of several elasticsearch components that can be
// modified or deleted outside of the entity manager apis, this can
// be used to verify the actual installation is complete and running
export type EntityDefinitionWithState = EntityDefinition & {
state: { installed: boolean; running: boolean };
};

37
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/uninstall_entity_definition.ts
View file

@ -9,10 +9,6 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { EntityDefinition } from '@kbn/entities-schema';
import { Logger } from '@kbn/logging';
import {
getEntityHistoryIndexTemplateV1,
getEntityLatestIndexTemplateV1,
} from '../../../common/helpers';
import { deleteEntityDefinition } from './delete_entity_definition';
import { deleteIndices } from './delete_index';
import { deleteHistoryIngestPipeline, deleteLatestIngestPipeline } from './delete_ingest_pipeline';
@ -23,6 +19,10 @@ import {
stopAndDeleteLatestTransform,
} from './stop_and_delete_transform';
import { isBackfillEnabled } from './helpers/is_backfill_enabled';
import {
generateHistoryIndexTemplateId,
generateLatestIndexTemplateId,
} from './helpers/generate_component_id';
import { deleteTemplate } from '../manage_index_templates';
export async function uninstallEntityDefinition({
@ -38,20 +38,29 @@ export async function uninstallEntityDefinition({
logger: Logger;
deleteData?: boolean;
}) {
await stopAndDeleteHistoryTransform(esClient, definition, logger);
if (isBackfillEnabled(definition)) {
await stopAndDeleteHistoryBackfillTransform(esClient, definition, logger);
}
await stopAndDeleteLatestTransform(esClient, definition, logger);
await deleteHistoryIngestPipeline(esClient, definition, logger);
await deleteLatestIngestPipeline(esClient, definition, logger);
await deleteEntityDefinition(soClient, definition, logger);
await deleteTemplate({ esClient, logger, name: getEntityHistoryIndexTemplateV1(definition.id) });
await deleteTemplate({ esClient, logger, name: getEntityLatestIndexTemplateV1(definition.id) });
await Promise.all([
stopAndDeleteHistoryTransform(esClient, definition, logger),
stopAndDeleteLatestTransform(esClient, definition, logger),
isBackfillEnabled(definition)
? stopAndDeleteHistoryBackfillTransform(esClient, definition, logger)
: Promise.resolve(),
]);
await Promise.all([
deleteHistoryIngestPipeline(esClient, definition, logger),
deleteLatestIngestPipeline(esClient, definition, logger),
]);
if (deleteData) {
await deleteIndices(esClient, definition, logger);
}
await Promise.all([
deleteTemplate({ esClient, logger, name: generateHistoryIndexTemplateId(definition) }),
deleteTemplate({ esClient, logger, name: generateLatestIndexTemplateId(definition) }),
]);
await deleteEntityDefinition(soClient, definition);
}
export async function uninstallBuiltInEntityDefinitions({

53
x-pack/plugins/observability_solution/entity_manager/server/lib/entities/upgrade_entity_definition.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EntityDefinition } from '@kbn/entities-schema';
import { installBuiltInEntityDefinitions } from './install_entity_definition';
import { startTransform } from './start_transform';
import { EntityManagerServerSetup } from '../../types';
import { checkIfEntityDiscoveryAPIKeyIsValid, readEntityDiscoveryAPIKey } from '../auth';
import { getClientsFromAPIKey } from '../utils';
import { ERROR_API_KEY_NOT_FOUND } from '../../../common/errors';
export async function upgradeBuiltInEntityDefinitions({
definitions,
server,
}: {
definitions: EntityDefinition[];
server: EntityManagerServerSetup;
}): Promise<
{ success: true; definitions: EntityDefinition[] } | { success: false; reason: string }
> {
const { logger } = server;
const apiKey = await readEntityDiscoveryAPIKey(server);
if (!apiKey) {
return { success: false, reason: ERROR_API_KEY_NOT_FOUND };
}
const isValid = await checkIfEntityDiscoveryAPIKeyIsValid(server, apiKey);
if (!isValid) {
throw new Error(
'Stored API key is not valid, skipping built-in definition upgrade. You can re-enable Entity Discovery to update the key.'
);
}
const { esClient, soClient } = getClientsFromAPIKey({ apiKey, server });
logger.debug(`Starting built-in definitions upgrade`);
const upgradedDefinitions = await installBuiltInEntityDefinitions({
esClient,
soClient,
definitions,
logger,
});
await Promise.all(
upgradedDefinitions.map((definition) => startTransform(esClient, definition, logger))
);
return { success: true, definitions: upgradedDefinitions };
}

47
x-pack/plugins/observability_solution/entity_manager/server/lib/manage_index_templates.ts
View file

@ -10,6 +10,10 @@ import {
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { entitiesHistoryBaseComponentTemplateConfig } from '../templates/components/base_history';
import { entitiesLatestBaseComponentTemplateConfig } from '../templates/components/base_latest';
import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity';
import { entitiesEventComponentTemplateConfig } from '../templates/components/event';
import { retryTransientEsErrors } from './entities/helpers/retry';
interface TemplateManagementOptions {
@ -24,6 +28,37 @@ interface ComponentManagementOptions {
logger: Logger;
}
export const installEntityManagerTemplates = async ({
esClient,
logger,
}: {
esClient: ElasticsearchClient;
logger: Logger;
}) => {
await Promise.all([
upsertComponent({
esClient,
logger,
component: entitiesHistoryBaseComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger,
component: entitiesLatestBaseComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger,
component: entitiesEventComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger,
component: entitiesEntityComponentTemplateConfig,
}),
]);
};
interface DeleteTemplateOptions {
esClient: ElasticsearchClient;
name: string;
@ -33,15 +68,11 @@ interface DeleteTemplateOptions {
export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) {
try {
await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger });
logger.debug(() => `Installed entity manager index template: ${JSON.stringify(template)}`);
} catch (error: any) {
logger.error(`Error updating entity manager index template: ${error.message}`);
throw error;
}
logger.info(
`Entity manager index template is up to date (use debug logging to see what was installed)`
);
logger.debug(() => `Entity manager index template: ${JSON.stringify(template)}`);
}
export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) {
@ -61,13 +92,9 @@ export async function upsertComponent({ esClient, component, logger }: Component
await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), {
logger,
});
logger.debug(() => `Installed entity manager component template: ${JSON.stringify(component)}`);
} catch (error: any) {
logger.error(`Error updating entity manager component template: ${error.message}`);
throw error;
}
logger.info(
`Entity manager component template is up to date (use debug logging to see what was installed)`
);
logger.debug(() => `Entity manager component template: ${JSON.stringify(component)}`);
}

47
x-pack/plugins/observability_solution/entity_manager/server/plugin.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { firstValueFrom } from 'rxjs';
import {
Plugin,
CoreSetup,
@ -14,7 +15,7 @@ import {
PluginConfigDescriptor,
Logger,
} from '@kbn/core/server';
import { upsertComponent } from './lib/manage_index_templates';
import { installEntityManagerTemplates } from './lib/manage_index_templates';
import { setupRoutes } from './routes';
import {
EntityManagerPluginSetupDependencies,
@ -22,11 +23,9 @@ import {
EntityManagerServerSetup,
} from './types';
import { EntityManagerConfig, configSchema, exposeToBrowserConfig } from '../common/config';
import { entitiesEventComponentTemplateConfig } from './templates/components/event';
import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects';
import { entitiesEntityComponentTemplateConfig } from './templates/components/entity';
import { entitiesLatestBaseComponentTemplateConfig } from './templates/components/base_latest';
import { entitiesHistoryBaseComponentTemplateConfig } from './templates/components/base_history';
import { upgradeBuiltInEntityDefinitions } from './lib/entities/upgrade_entity_definition';
import { builtInDefinitions } from './lib/entities/built_in';
export type EntityManagerServerPluginSetup = ReturnType<EntityManagerServerPlugin['setup']>;
export type EntityManagerServerPluginStart = ReturnType<EntityManagerServerPlugin['start']>;
@ -89,29 +88,21 @@ export class EntityManagerServerPlugin
const esClient = core.elasticsearch.client.asInternalUser;
// Install entities component templates and index template
Promise.all([
upsertComponent({
esClient,
logger: this.logger,
component: entitiesHistoryBaseComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger: this.logger,
component: entitiesLatestBaseComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger: this.logger,
component: entitiesEventComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger: this.logger,
component: entitiesEntityComponentTemplateConfig,
}),
]).catch(() => {});
installEntityManagerTemplates({ esClient, logger: this.logger })
.then(async () => {
// the api key validation requires a check against the cluster license
// which is lazily loaded. we ensure it gets loaded before the update
await firstValueFrom(plugins.licensing.license$);
const { success } = await upgradeBuiltInEntityDefinitions({
definitions: builtInDefinitions,
server: this.server!,
});
if (success) {
this.logger.info('Builtin definitions were successfully upgraded');
}
})
.catch((err) => this.logger.error(err));
return {};
}

20
x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts
View file

@ -5,12 +5,14 @@
* 2.0.
*/
import semver from 'semver';
import { RequestHandlerContext } from '@kbn/core/server';
import { SetupRouteOptions } from '../types';
import { checkIfEntityDiscoveryAPIKeyIsValid, readEntityDiscoveryAPIKey } from '../../lib/auth';
import {
ERROR_API_KEY_NOT_FOUND,
ERROR_API_KEY_NOT_VALID,
ERROR_BUILTIN_UPGRADE_REQUIRED,
ERROR_DEFINITION_STOPPED,
ERROR_PARTIAL_BUILTIN_INSTALLATION,
} from '../../../common/errors';
@ -54,17 +56,21 @@ export function checkEntityDiscoveryEnabledRoute<T extends RequestHandlerContext
id: builtInDefinition.id,
});
return definitions[0];
return { installedDefinition: definitions[0], builtInDefinition };
})
).then((results) =>
results.reduce(
(state, definition) => {
(state, { installedDefinition, builtInDefinition }) => {
return {
installed: Boolean(state.installed && definition?.state.installed),
running: Boolean(state.running && definition?.state.running),
installed: Boolean(state.installed && installedDefinition?.state.installed),
running: Boolean(state.running && installedDefinition?.state.running),
outdated:
state.outdated ||
(installedDefinition &&
semver.neq(installedDefinition.version, builtInDefinition.version)),
};
},
{ installed: true, running: true }
{ installed: true, running: true, outdated: false }
)
);
@ -76,6 +82,10 @@ export function checkEntityDiscoveryEnabledRoute<T extends RequestHandlerContext
return res.ok({ body: { enabled: false, reason: ERROR_DEFINITION_STOPPED } });
}
if (entityDiscoveryState.outdated) {
return res.ok({ body: { enabled: false, reason: ERROR_BUILTIN_UPGRADE_REQUIRED } });
}
return res.ok({ body: { enabled: true } });
} catch (err) {
logger.error(err);

1
x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/disable.ts
View file

@ -39,6 +39,7 @@ export function disableEntityDiscoveryRoute<T extends RequestHandlerContext>({
},
});
}
const soClient = (await context.core).savedObjects.getClient({
includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
});

16
x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/enable.ts
View file

@ -25,6 +25,7 @@ import { builtInDefinitions } from '../../lib/entities/built_in';
import { installBuiltInEntityDefinitions } from '../../lib/entities/install_entity_definition';
import { ERROR_API_KEY_SERVICE_DISABLED } from '../../../common/errors';
import { EntityDiscoveryApiKeyType } from '../../saved_objects';
import { startTransform } from '../../lib/entities/start_transform';
export function enableEntityDiscoveryRoute<T extends RequestHandlerContext>({
router,
@ -89,14 +90,21 @@ export function enableEntityDiscoveryRoute<T extends RequestHandlerContext>({
await saveEntityDiscoveryAPIKey(soClient, apiKey);
await installBuiltInEntityDefinitions({
logger,
builtInDefinitions,
const installedDefinitions = await installBuiltInEntityDefinitions({
esClient,
soClient,
installOnly: req.query.installOnly,
logger,
definitions: builtInDefinitions,
});
if (!req.query.installOnly) {
await Promise.all(
installedDefinitions.map((installedDefinition) =>
startTransform(esClient, installedDefinition, logger)
)
);
}
return res.ok({ body: { success: true } });
} catch (err) {
logger.error(err);

15
x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts
View file

@ -49,5 +49,20 @@ export const entityDefinition: SavedObjectsType = {
},
],
},
'2': {
changes: [
{
type: 'data_backfill',
backfillFn: () => {
return {
attributes: {
installStatus: 'installed',
installStartedAt: new Date().toISOString(),
},
};
},
},
],
},
},
};

2
x-pack/plugins/observability_solution/entity_manager/server/types.ts
View file

@ -11,6 +11,7 @@ import {
EncryptedSavedObjectsPluginSetup,
EncryptedSavedObjectsPluginStart,
} from '@kbn/encrypted-saved-objects-plugin/server';
import { LicensingPluginStart } from '@kbn/licensing-plugin/server';
import { EntityManagerConfig } from '../common/config';
export interface EntityManagerServerSetup {
@ -33,4 +34,5 @@ export interface EntityManagerPluginSetupDependencies {
export interface EntityManagerPluginStartDependencies {
security: SecurityPluginStart;
encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
licensing: LicensingPluginStart;
}

1
x-pack/plugins/observability_solution/entity_manager/tsconfig.json
View file

@ -28,5 +28,6 @@
"@kbn/security-plugin",
"@kbn/encrypted-saved-objects-plugin",
"@kbn/logging-mocks",
"@kbn/licensing-plugin",
]
}

192
x-pack/test/api_integration/apis/entity_manager/builtin_definitions.ts Normal file
View file

@ -0,0 +1,192 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import semver from 'semver';
import expect from '@kbn/expect';
import { builtInDefinitions } from '@kbn/entityManager-plugin/server/lib/entities/built_in';
import { ERROR_API_KEY_NOT_FOUND } from '@kbn/entityManager-plugin/public';
import { builtInEntityDefinition as mockBuiltInEntityDefinition } from '@kbn/entityManager-plugin/server/lib/entities/helpers/fixtures';
import { EntityDefinition } from '@kbn/entities-schema';
import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
import { FtrProviderContext } from '../../ftr_provider_context';
import { createAdmin, createRuntimeUser } from './helpers/user';
import { Auth, getInstalledDefinitions, upgradeBuiltinDefinitions } from './helpers/request';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
const supertest = getService('supertest');
const supertestWithoutAuth = getService('supertestWithoutAuth');
const enablementRequest =
(method: 'get' | 'put' | 'delete') =>
async (auth: Auth, expectedCode: number, query: { [key: string]: any } = {}) => {
const response = await supertestWithoutAuth[method]('/internal/entities/managed/enablement')
.auth(auth.username, auth.password)
.query(query)
.set('kbn-xsrf', 'xxx')
.send()
.expect(expectedCode);
return response.body;
};
const entityDiscoveryState = enablementRequest('get');
const enableEntityDiscovery = enablementRequest('put');
const disableEntityDiscovery = enablementRequest('delete');
const expectNoInstalledDefinitions = async () => {
const definitionsResponse = await getInstalledDefinitions(supertest);
expect(definitionsResponse.definitions).to.eql([]);
};
const isInstalledAndRunning = (
definition: EntityDefinition,
installedDefinitions: EntityDefinitionWithState[]
) => {
return installedDefinitions.find((installedDefinition) => {
return (
installedDefinition.id === definition.id &&
installedDefinition.version === definition.version &&
installedDefinition.state.installed &&
installedDefinition.state.running
);
});
};
describe('Entity discovery builtin definitions', () => {
let authorizedUser: { username: string; password: string };
let unauthorizedUser: { username: string; password: string };
before(async () => {
[authorizedUser, unauthorizedUser] = await Promise.all([
createAdmin({ esClient }),
createRuntimeUser({ esClient }),
]);
});
describe('enablement/disablement', () => {
describe('with authorized user', () => {
it('should enable and disable entity discovery', async () => {
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
const definitionsResponse = await getInstalledDefinitions(
supertestWithoutAuth,
authorizedUser
);
expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
expect(
builtInDefinitions.every((builtin) =>
isInstalledAndRunning(builtin, definitionsResponse.definitions)
)
).to.eql(true, 'all builtin definitions are not installed/running');
let stateResponse = await entityDiscoveryState(authorizedUser, 200);
expect(stateResponse.enabled).to.eql(
true,
`EEM is not enabled; response: ${JSON.stringify(stateResponse)}`
);
const disableResponse = await disableEntityDiscovery(authorizedUser, 200, {
deleteData: false,
});
expect(disableResponse.success).to.eql(
true,
`authorized user failed to disable EEM; response: ${JSON.stringify(disableResponse)}`
);
stateResponse = await entityDiscoveryState(authorizedUser, 200);
expect(stateResponse.enabled).to.eql(false, 'EEM is not disabled');
await expectNoInstalledDefinitions();
});
});
describe('with unauthorized user', () => {
it('should fail to enable entity discovery', async () => {
await enableEntityDiscovery(unauthorizedUser, 403);
const stateResponse = await entityDiscoveryState(unauthorizedUser, 200);
expect(stateResponse.enabled).to.eql(false, 'EEM is enabled');
await expectNoInstalledDefinitions();
});
it('should fail to disable entity discovery', async () => {
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
await disableEntityDiscovery(unauthorizedUser, 403);
const disableResponse = await disableEntityDiscovery(authorizedUser, 200);
expect(disableResponse.success).to.eql(true, "authorized user can't disable EEM");
});
});
});
describe('upgrade', () => {
it('should noop when no api key stored', async () => {
const result = await upgradeBuiltinDefinitions(supertest, []);
expect(result).to.eql({ success: false, reason: ERROR_API_KEY_NOT_FOUND });
});
it('should upgrade existing definitions', async () => {
await expectNoInstalledDefinitions();
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
let definitionsResponse = await getInstalledDefinitions(supertest);
expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
// increment the version of builtin definitions
const updatedBuiltinDefinitions = definitionsResponse.definitions.map((definition) => {
return {
...definition,
version: semver.inc(definition.version, 'minor')!,
};
});
const upgradeResponse = await upgradeBuiltinDefinitions(
supertest,
updatedBuiltinDefinitions
);
expect(upgradeResponse.success).to.eql(true);
// check builtin definitions are running the latest version
definitionsResponse = await getInstalledDefinitions(supertest);
expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
expect(
updatedBuiltinDefinitions.every((builtin) =>
isInstalledAndRunning(builtin, definitionsResponse.definitions)
)
).to.eql(true, 'all builtin definitions are not installed/running');
await disableEntityDiscovery(authorizedUser, 200);
});
});
it('should install new builtin definitions', async () => {
await expectNoInstalledDefinitions();
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
// inject definition to simulate release of new builtin definition
const latestBuiltInDefinitions = [...builtInDefinitions, mockBuiltInEntityDefinition];
const upgradeResponse = await upgradeBuiltinDefinitions(supertest, latestBuiltInDefinitions);
expect(upgradeResponse.success).to.eql(true, 'upgrade was not successful');
const definitionsResponse = await getInstalledDefinitions(supertest);
expect(definitionsResponse.definitions.length).to.eql(latestBuiltInDefinitions.length);
expect(
isInstalledAndRunning(mockBuiltInEntityDefinition, definitionsResponse.definitions)
).to.ok();
await disableEntityDiscovery(authorizedUser, 200);
});
});
}

9
x-pack/test/api_integration/apis/entity_manager/config.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { resolve } from 'path';
import { FtrConfigProviderContext, GenericFtrProviderContext } from '@kbn/test';
import { FtrProviderContext } from '../../ftr_provider_context';
@ -21,10 +22,18 @@ export default async function createTestConfig({
}: FtrConfigProviderContext): Promise<EntityManagerConfig> {
const baseIntegrationTestsConfig = await readConfigFile(require.resolve('../../config.ts'));
const services = baseIntegrationTestsConfig.get('services');
const entityManagerFixturePlugin = resolve(__dirname, './fixture_plugin');
return {
...baseIntegrationTestsConfig.getAll(),
testFiles: [require.resolve('.')],
kbnTestServer: {
...baseIntegrationTestsConfig.get('kbnTestServer'),
serverArgs: [
...baseIntegrationTestsConfig.get('kbnTestServer.serverArgs'),
`--plugin-path=${entityManagerFixturePlugin}`,
],
},
services: {
...services,
},

111
x-pack/test/api_integration/apis/entity_manager/enablement.ts
View file

@ -1,111 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { builtInDefinitions } from '@kbn/entityManager-plugin/server/lib/entities/built_in';
import { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
import { FtrProviderContext } from '../../ftr_provider_context';
import { createAdmin, createRuntimeUser } from './helpers/user';
import { Auth, getInstalledDefinitions } from './helpers/request';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
const supertest = getService('supertestWithoutAuth');
const enablementRequest =
(method: 'get' | 'put' | 'delete') =>
async (auth: Auth, expectedCode: number, query: { [key: string]: any } = {}) => {
const response = await supertest[method]('/internal/entities/managed/enablement')
.auth(auth.username, auth.password)
.query(query)
.set('kbn-xsrf', 'xxx')
.send()
.expect(expectedCode);
return response.body;
};
const entityDiscoveryState = enablementRequest('get');
const enableEntityDiscovery = enablementRequest('put');
const disableEntityDiscovery = enablementRequest('delete');
describe('Entity discovery enablement', () => {
let authorizedUser: { username: string; password: string };
let unauthorizedUser: { username: string; password: string };
before(async () => {
[authorizedUser, unauthorizedUser] = await Promise.all([
createAdmin({ esClient }),
createRuntimeUser({ esClient }),
]);
});
describe('with authorized user', () => {
it('should enable and disable entity discovery', async () => {
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
let definitionsResponse = await getInstalledDefinitions(supertest, authorizedUser);
expect(definitionsResponse.definitions.length).to.eql(builtInDefinitions.length);
expect(
builtInDefinitions.every((builtin) => {
return definitionsResponse.definitions.find(
(installedDefinition: EntityDefinitionWithState) => {
return (
installedDefinition.id === builtin.id &&
installedDefinition.state.installed &&
installedDefinition.state.running
);
}
);
})
).to.eql(true, 'all builtin definitions are not installed/running');
let stateResponse = await entityDiscoveryState(authorizedUser, 200);
expect(stateResponse.enabled).to.eql(
true,
`EEM is not enabled; response: ${JSON.stringify(stateResponse)}`
);
const disableResponse = await disableEntityDiscovery(authorizedUser, 200, {
deleteData: false,
});
expect(disableResponse.success).to.eql(
true,
`authorized user failed to disable EEM; response: ${JSON.stringify(disableResponse)}`
);
stateResponse = await entityDiscoveryState(authorizedUser, 200);
expect(stateResponse.enabled).to.eql(false, 'EEM is not disabled');
definitionsResponse = await getInstalledDefinitions(supertest, authorizedUser);
expect(definitionsResponse.definitions).to.eql([]);
});
});
describe('with unauthorized user', () => {
it('should fail to enable entity discovery', async () => {
await enableEntityDiscovery(unauthorizedUser, 403);
const stateResponse = await entityDiscoveryState(unauthorizedUser, 200);
expect(stateResponse.enabled).to.eql(false, 'EEM is enabled');
const definitionsResponse = await getInstalledDefinitions(supertest, unauthorizedUser);
expect(definitionsResponse.definitions).to.eql([]);
});
it('should fail to disable entity discovery', async () => {
const enableResponse = await enableEntityDiscovery(authorizedUser, 200);
expect(enableResponse.success).to.eql(true, "authorized user can't enable EEM");
let disableResponse = await disableEntityDiscovery(unauthorizedUser, 403);
disableResponse = await disableEntityDiscovery(authorizedUser, 200);
expect(disableResponse.success).to.eql(true, "authorized user can't disable EEM");
});
});
});
}

15
x-pack/test/api_integration/apis/entity_manager/fixture_plugin/kibana.jsonc Normal file
View file

@ -0,0 +1,15 @@
{
"type": "plugin",
"id": "@kbn/entity-manager-fixture-plugin",
"owner": "@elastic/obs-entities",
"plugin": {
"id": "entityManagerFixture",
"server": true,
"browser": false,
"requiredPlugins": [
"encryptedSavedObjects",
"security",
],
"optionalPlugins": []
}
}

13
x-pack/test/api_integration/apis/entity_manager/fixture_plugin/package.json Normal file
View file

@ -0,0 +1,13 @@
{
"name": "@kbn/entity-manager-fixture-plugin",
"version": "1.0.0",
"kibana": {
"version": "kibana",
"templateVersion": "1.0.0"
},
"scripts": {
"kbn": "node ../../../../../../scripts/kbn.js",
"build": "rm -rf './target' && ../../../../../../node_modules/.bin/tsc"
},
"license": "Elastic License 2.0"
}

13
x-pack/test/api_integration/apis/entity_manager/fixture_plugin/server/index.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PluginInitializerContext } from '@kbn/core-plugins-server';
import { FixturePlugin } from './plugin';
export const plugin = async (context: PluginInitializerContext<{}>) => {
return new FixturePlugin(context);
};

74
x-pack/test/api_integration/apis/entity_manager/fixture_plugin/server/plugin.ts Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { z } from 'zod';
import {
Plugin,
CoreSetup,
RequestHandlerContext,
KibanaRequest,
KibanaResponseFactory,
Logger,
PluginInitializerContext,
} from '@kbn/core/server';
import { buildRouteValidationWithZod } from '@kbn/zod-helpers';
import { upgradeBuiltInEntityDefinitions } from '@kbn/entityManager-plugin/server/lib/entities/upgrade_entity_definition';
import { SecurityPluginStart } from '@kbn/security-plugin-types-server';
import { EncryptedSavedObjectsPluginStart } from '@kbn/encrypted-saved-objects-plugin/server';
import { entityDefinitionSchema } from '@kbn/entities-schema';
interface FixtureStartDeps {
encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
security: SecurityPluginStart;
}
export class FixturePlugin implements Plugin<void, void, {}, FixtureStartDeps> {
private logger: Logger;
constructor(context: PluginInitializerContext<{}>) {
this.logger = context.logger.get();
}
public setup(core: CoreSetup<FixtureStartDeps>) {
core.http.createRouter().post(
{
path: '/api/entities/upgrade_builtin_definitions',
validate: {
body: buildRouteValidationWithZod(
z.object({
definitions: z.array(entityDefinitionSchema),
})
),
},
},
async (
context: RequestHandlerContext,
req: KibanaRequest<any, any, any, any>,
res: KibanaResponseFactory
) => {
const [coreStart, { encryptedSavedObjects, security }] = await core.getStartServices();
const result = await upgradeBuiltInEntityDefinitions({
definitions: req.body.definitions,
server: {
encryptedSavedObjects,
security,
core: coreStart,
logger: this.logger,
config: {},
isServerless: false,
},
});
return res.ok({ body: result });
}
);
}
public start() {}
public stop() {}
}

22
x-pack/test/api_integration/apis/entity_manager/fixture_plugin/tsconfig.json Normal file
View file

@ -0,0 +1,22 @@
{
"extends": "../../../../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"isolatedModules": true
},
"include": [
"server/**/**/*",
],
"kbn_references": [
"@kbn/core",
"@kbn/encrypted-saved-objects-plugin",
"@kbn/core-plugins-server",
"@kbn/zod-helpers",
"@kbn/entityManager-plugin",
"@kbn/security-plugin-types-server",
"@kbn/entities-schema",
],
"exclude": [
"target/**/*",
]
}

12
x-pack/test/api_integration/apis/entity_manager/helpers/request.ts
View file

@ -47,3 +47,15 @@ export const uninstallDefinition = (supertest: Agent, id: string, deleteData = f
.send()
.expect(200);
};
export const upgradeBuiltinDefinitions = async (
supertest: Agent,
definitions: EntityDefinition[]
): Promise<{ success: boolean }> => {
const response = await supertest
.post('/api/entities/upgrade_builtin_definitions')
.set('kbn-xsrf', 'xxx')
.send({ definitions })
.expect(200);
return response.body;
};

2
x-pack/test/api_integration/apis/entity_manager/index.ts
View file

@ -11,7 +11,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('Entity Manager', function () {
this.tags(['entityManager']);
loadTestFile(require.resolve('./enablement'));
loadTestFile(require.resolve('./builtin_definitions'));
loadTestFile(require.resolve('./definitions'));
});
}

4
yarn.lock
View file

@ -4859,6 +4859,10 @@
version "0.0.0"
uid ""
"@kbn/entity-manager-fixture-plugin@link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin":
version "0.0.0"
uid ""
"@kbn/entityManager-plugin@link:x-pack/plugins/observability_solution/entity_manager":
version "0.0.0"
uid ""