github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [eem] rename fields to snake case (#195895) (#197527)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[eem] rename fields to snake case
(#195895)](https://github.com/elastic/kibana/pull/195895)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kevin
Lacabane","email":"kevin.lacabane@elastic.co"},"sourceCommit":{"committedDate":"2024-10-23T20:06:54Z","message":"[eem]
rename fields to snake case
(#195895)","sha":"0617ad44406daecd0342a8fbaf84d9cdef8c5d50","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-major","ci:project-deploy-observability","Team:obs-ux-infra_services"],"title":"[eem]
rename fields to snake
case","number":195895,"url":"https://github.com/elastic/kibana/pull/195895","mergeCommit":{"message":"[eem]
rename fields to snake case
(#195895)","sha":"0617ad44406daecd0342a8fbaf84d9cdef8c5d50"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195895","number":195895,"mergeCommit":{"message":"[eem]
rename fields to snake case
(#195895)","sha":"0617ad44406daecd0342a8fbaf84d9cdef8c5d50"}}]}]
BACKPORT-->

Co-authored-by: Kevin Lacabane <kevin.lacabane@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-24 08:48:23 +11:00 • committed by GitHub
parent 495de32666
commit 741e5e2b12
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
46 changed files with 334 additions and 429 deletions
Download patch file Download diff file Expand all files Collapse all files

6
packages/kbn-apm-synthtrace-client/src/lib/entities/container_entity.ts
View file

@ -15,8 +15,8 @@ class ContainerEntity extends Serializable<EntityFields> {
super({
...fields,
'entity.type': 'container',
'entity.definitionId': 'builtin_containers_from_ecs_data',
'entity.identityFields': ['container.id'],
'entity.definition_id': 'builtin_containers_from_ecs_data',
'entity.identity_fields': ['container.id'],
});
}
}
@ -36,7 +36,7 @@ export function containerEntity({
'source_data_stream.type': dataStreamType,
'agent.name': agentName,
'container.id': containerId,
'entity.displayName': containerId,
'entity.display_name': containerId,
'entity.id': entityId,
});
}

6
packages/kbn-apm-synthtrace-client/src/lib/entities/host_entity.ts
View file

@ -15,8 +15,8 @@ class HostEntity extends Serializable<EntityFields> {
super({
...fields,
'entity.type': 'host',
'entity.definitionId': 'builtin_hosts_from_ecs_data',
'entity.identityFields': ['host.name'],
'entity.definition_id': 'builtin_hosts_from_ecs_data',
'entity.identity_fields': ['host.name'],
});
}
}
@ -36,7 +36,7 @@ export function hostEntity({
'source_data_stream.type': dataStreamType,
'agent.name': agentName,
'host.name': hostName,
'entity.displayName': hostName,
'entity.display_name': hostName,
'entity.id': entityId,
});
}

14
packages/kbn-apm-synthtrace-client/src/lib/entities/index.ts
View file

@ -20,15 +20,15 @@ export type EntityFields = Fields &
'source_data_stream.type': string | string[];
'source_data_stream.dataset': string | string[];
'event.ingested': string;
sourceIndex: string;
'entity.lastSeenTimestamp': string;
'entity.schemaVersion': string;
'entity.definitionVersion': string;
'entity.displayName': string;
'entity.identityFields': string | string[];
source_index: string;
'entity.last_seen_timestamp': string;
'entity.schema_version': string;
'entity.definition_version': string;
'entity.display_name': string;
'entity.identity_fields': string | string[];
'entity.id': string;
'entity.type': string;
'entity.definitionId': string;
'entity.definition_id': string;
[key: string]: any;
}>;

6
packages/kbn-apm-synthtrace-client/src/lib/entities/service_entity.ts
View file

@ -15,8 +15,8 @@ class ServiceEntity extends Serializable<EntityFields> {
super({
...fields,
'entity.type': 'service',
'entity.definitionId': 'builtin_services_from_ecs_data',
'entity.identityFields': ['service.name'],
'entity.definition_id': 'builtin_services_from_ecs_data',
'entity.identity_fields': ['service.name'],
});
}
}
@ -36,7 +36,7 @@ export function serviceEntity({
}) {
return new ServiceEntity({
'service.name': serviceName,
'entity.displayName': serviceName,
'entity.display_name': serviceName,
'service.environment': environment,
'source_data_stream.type': dataStreamType,
'agent.name': agentName,

2
packages/kbn-apm-synthtrace/src/lib/entities/entities_synthtrace_es_client.ts
View file

@ -62,7 +62,7 @@ function lastSeenTimestampTransform() {
const timestamp = document['@timestamp'];
if (timestamp) {
const isoString = new Date(timestamp).toISOString();
document['entity.lastSeenTimestamp'] = isoString;
document['entity.last_seen_timestamp'] = isoString;
document['event.ingested'] = isoString;
delete document['@timestamp'];
}

13
packages/kbn-investigation-shared/src/rest_specs/entity.ts
View file

@ -19,13 +19,12 @@ const metricsSchema = z.object({
const entitySchema = z.object({
id: z.string(),
definitionId: z.string(),
definitionVersion: z.string(),
displayName: z.string(),
firstSeenTimestamp: z.string(),
lastSeenTimestamp: z.string(),
identityFields: z.array(z.string()),
schemaVersion: z.string(),
definition_id: z.string(),
definition_version: z.string(),
display_name: z.string(),
last_seen_timestamp: z.string(),
identity_fields: z.array(z.string()),
schema_version: z.string(),
type: z.string(),
metrics: metricsSchema,
});

20
x-pack/packages/kbn-entities-schema/src/schema/entity.test.ts
View file

@ -9,19 +9,15 @@ import { entityLatestSchema, entityMetadataSchema } from './entity';
const entity = {
entity: {
lastSeenTimestamp: '2024-08-06T17:03:50.722Z',
schemaVersion: 'v1',
definitionVersion: '999.999.999',
displayName: 'message_processor',
identityFields: ['log.logger', 'event.category'],
last_seen_timestamp: '2024-08-06T17:03:50.722Z',
schema_version: 'v1',
definition_version: '999.999.999',
display_name: 'message_processor',
identity_fields: ['log.logger', 'event.category'],
id: '6UHVPiduEC2qk6rMjs1Jzg==',
metrics: {
logRate: 100,
errorRate: 0,
},
type: 'service',
firstSeenTimestamp: '2024-08-06T16:50:00.000Z',
definitionId: 'admin-console-services',
metrics: {},
definition_id: 'admin-console-services',
},
};
@ -47,7 +43,7 @@ const metadata = {
ingested: '2024-08-06T17:06:24.444700Z',
category: '',
},
sourceIndex: ['kbn-data-forge-fake_stack.message_processor-2024-08-01'],
source_index: ['kbn-data-forge-fake_stack.message_processor-2024-08-01'],
log: {
logger: 'message_processor',
},

19
x-pack/packages/kbn-entities-schema/src/schema/entity.ts
View file

@ -11,12 +11,12 @@ import { arrayOfStringsSchema } from './common';
export const entityBaseSchema = z.object({
id: z.string(),
type: z.string(),
identityFields: arrayOfStringsSchema,
displayName: z.string(),
identity_fields: arrayOfStringsSchema,
display_name: z.string(),
metrics: z.record(z.string(), z.number()),
definitionVersion: z.string(),
schemaVersion: z.string(),
definitionId: z.string(),
definition_version: z.string(),
schema_version: z.string(),
definition_id: z.string(),
});
export interface MetadataRecord {
@ -34,15 +34,8 @@ export const entityLatestSchema = z
.object({
entity: entityBaseSchema.merge(
z.object({
lastSeenTimestamp: z.string(),
last_seen_timestamp: z.string(),
})
),
})
.and(entityMetadataSchema);
export const entityHistorySchema = z
.object({
'@timestamp': z.string(),
entity: entityBaseSchema,
})
.and(entityMetadataSchema);

2
x-pack/plugins/entity_manager/server/lib/auth/privileges.ts
View file

@ -59,7 +59,7 @@ export const entityDefinitionRuntimePrivileges = {
index: [
{
names: [ENTITY_INTERNAL_INDICES_PATTERN],
privileges: ['create_index', 'index', 'create_doc', 'auto_configure', 'read'],
privileges: ['create_index', 'delete_index', 'index', 'create_doc', 'auto_configure', 'read'],
},
{
names: [...BUILT_IN_ALLOWED_INDICES, ENTITY_INTERNAL_INDICES_PATTERN],

4
x-pack/plugins/entity_manager/server/lib/entities/built_in/services_from_ecs_data.ts
View file

@ -29,7 +29,7 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
identityFields: ['service.name'],
displayNameTemplate: '{{service.name}}',
metadata: [
{ source: '_index', destination: 'sourceIndex' },
{ source: '_index', destination: 'source_index' },
{
source: 'data_stream.type',
destination: 'source_data_stream.type',
@ -38,7 +38,7 @@ export const builtInServicesFromEcsEntityDefinition: EntityDefinition =
source: 'data_stream.dataset',
destination: 'source_data_stream.dataset',
},
{ source: 'agent.name', aggregation: { type: 'terms', limit: 100 } },
'agent.name',
'service.environment',
'service.name',
'service.namespace',

2
x-pack/plugins/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
View file

@ -25,7 +25,7 @@ export async function createAndInstallIngestPipelines(
id: latestId,
processors: latestProcessors,
_meta: {
definitionVersion: definition.version,
definition_version: definition.version,
managed: definition.managed,
},
}),

6
x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts
View file

@ -14,11 +14,13 @@ export async function deleteIndices(
definition: EntityDefinition,
logger: Logger
) {
const index = generateLatestIndexName(definition);
try {
const index = generateLatestIndexName(definition);
await esClient.indices.delete({ index, ignore_unavailable: true });
} catch (e) {
logger.error(`Unable to remove entity definition index [${definition.id}}]`);
logger.error(
`Unable to remove entity definition index ${index} for definition [${definition.id}]`
);
throw e;
}
}

20
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap generated
View file

@ -16,25 +16,25 @@ Array [
},
Object {
"set": Object {
"field": "entity.definitionId",
"field": "entity.definition_id",
"value": "builtin_mock_entity_definition",
},
},
Object {
"set": Object {
"field": "entity.definitionVersion",
"field": "entity.definition_version",
"value": "1.0.0",
},
},
Object {
"set": Object {
"field": "entity.schemaVersion",
"field": "entity.schema_version",
"value": "v1",
},
},
Object {
"set": Object {
"field": "entity.identityFields",
"field": "entity.identity_fields",
"value": Array [
"log.logger",
],
@ -92,7 +92,7 @@ if (ctx.entity?.metadata?.sourceIndex?.data != null) {
},
Object {
"set": Object {
"field": "entity.displayName",
"field": "entity.display_name",
"value": "{{log.logger}}",
},
},
@ -121,25 +121,25 @@ Array [
},
Object {
"set": Object {
"field": "entity.definitionId",
"field": "entity.definition_id",
"value": "admin-console-services",
},
},
Object {
"set": Object {
"field": "entity.definitionVersion",
"field": "entity.definition_version",
"value": "1.0.0",
},
},
Object {
"set": Object {
"field": "entity.schemaVersion",
"field": "entity.schema_version",
"value": "v1",
},
},
Object {
"set": Object {
"field": "entity.identityFields",
"field": "entity.identity_fields",
"value": Array [
"log.logger",
],
@ -197,7 +197,7 @@ if (ctx.entity?.metadata?.sourceIndex?.data != null) {
},
Object {
"set": Object {
"field": "entity.displayName",
"field": "entity.display_name",
"value": "{{log.logger}}",
},
},

10
x-pack/plugins/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
View file

@ -117,25 +117,25 @@ export function generateLatestProcessors(definition: EntityDefinition) {
},
{
set: {
field: 'entity.definitionId',
field: 'entity.definition_id',
value: definition.id,
},
},
{
set: {
field: 'entity.definitionVersion',
field: 'entity.definition_version',
value: definition.version,
},
},
{
set: {
field: 'entity.schemaVersion',
field: 'entity.schema_version',
value: ENTITY_SCHEMA_VERSION_V1,
},
},
{
set: {
field: 'entity.identityFields',
field: 'entity.identity_fields',
value: definition.identityFields.map((identityField) => identityField.field),
},
},
@ -173,7 +173,7 @@ export function generateLatestProcessors(definition: EntityDefinition) {
// This must happen AFTER we lift the identity fields into the root of the document
{
set: {
field: 'entity.displayName',
field: 'entity.display_name',
value: definition.displayNameTemplate,
},
},

54
x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.test.ts
View file

@ -74,7 +74,7 @@ const assertHasCreatedDefinition = (
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
definition_version: definition.version,
managed: definition.managed,
},
});
@ -112,7 +112,7 @@ const assertHasUpgradedDefinition = (
id: generateLatestIngestPipelineId(definition),
processors: expect.anything(),
_meta: {
definitionVersion: definition.version,
definition_version: definition.version,
managed: definition.managed,
},
});
@ -260,7 +260,7 @@ describe('install_entity_definition', () => {
describe('installBuiltInEntityDefinitions', () => {
it('should install definition when not found', async () => {
const builtInDefinitions = [mockEntityDefinition];
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const clusterClient = elasticsearchClientMock.createScopedClusterClient();
const soClient = savedObjectsClientMock.create();
soClient.find.mockResolvedValue({ saved_objects: [], total: 0, page: 1, per_page: 10 });
soClient.update.mockResolvedValue({
@ -271,18 +271,19 @@ describe('install_entity_definition', () => {
});
await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions: builtInDefinitions,
logger: loggerMock.create(),
});
assertHasCreatedDefinition(mockEntityDefinition, soClient, esClient);
assertHasCreatedDefinition(mockEntityDefinition, soClient, clusterClient.asSecondaryAuthUser);
});
it('should reinstall when partial state found', async () => {
const builtInDefinitions = [mockEntityDefinition];
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const clusterClient = elasticsearchClientMock.createScopedClusterClient();
const esClient = clusterClient.asInternalUser;
// mock partially installed definition
esClient.ingest.getPipeline.mockResolvedValue({});
esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
@ -314,14 +315,18 @@ describe('install_entity_definition', () => {
});
await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions: builtInDefinitions,
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(mockEntityDefinition, soClient, esClient);
assertHasDeletedTransforms(mockEntityDefinition, clusterClient.asSecondaryAuthUser);
assertHasUpgradedDefinition(
mockEntityDefinition,
soClient,
clusterClient.asSecondaryAuthUser
);
});
it('should reinstall when outdated version', async () => {
@ -329,7 +334,8 @@ describe('install_entity_definition', () => {
...mockEntityDefinition,
version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
};
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const clusterClient = elasticsearchClientMock.createScopedClusterClient();
const esClient = clusterClient.asInternalUser;
esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
const soClient = savedObjectsClientMock.create();
@ -359,14 +365,14 @@ describe('install_entity_definition', () => {
});
await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions: [updatedDefinition],
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(updatedDefinition, soClient, esClient);
assertHasDeletedTransforms(mockEntityDefinition, clusterClient.asSecondaryAuthUser);
assertHasUpgradedDefinition(updatedDefinition, soClient, clusterClient.asSecondaryAuthUser);
});
it('should reinstall when stale upgrade', async () => {
@ -374,7 +380,8 @@ describe('install_entity_definition', () => {
...mockEntityDefinition,
version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0',
};
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const clusterClient = elasticsearchClientMock.createScopedClusterClient();
const esClient = clusterClient.asInternalUser;
esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
const soClient = savedObjectsClientMock.create();
@ -406,18 +413,19 @@ describe('install_entity_definition', () => {
});
await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions: [updatedDefinition],
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(updatedDefinition, soClient, esClient);
assertHasDeletedTransforms(mockEntityDefinition, clusterClient.asSecondaryAuthUser);
assertHasUpgradedDefinition(updatedDefinition, soClient, clusterClient.asSecondaryAuthUser);
});
it('should reinstall when failed installation', async () => {
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
const clusterClient = elasticsearchClientMock.createScopedClusterClient();
const esClient = clusterClient.asInternalUser;
esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 });
const soClient = savedObjectsClientMock.create();
@ -448,14 +456,18 @@ describe('install_entity_definition', () => {
});
await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions: [mockEntityDefinition],
logger: loggerMock.create(),
});
assertHasDeletedTransforms(mockEntityDefinition, esClient);
assertHasUpgradedDefinition(mockEntityDefinition, soClient, esClient);
assertHasDeletedTransforms(mockEntityDefinition, clusterClient.asSecondaryAuthUser);
assertHasUpgradedDefinition(
mockEntityDefinition,
soClient,
clusterClient.asSecondaryAuthUser
);
});
});
});

64
x-pack/plugins/entity_manager/server/lib/entities/install_entity_definition.ts
View file

@ -6,7 +6,7 @@
*/
import semver from 'semver';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { ElasticsearchClient, IScopedClusterClient } from '@kbn/core-elasticsearch-server';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
import { Logger } from '@kbn/logging';
@ -29,6 +29,7 @@ import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update';
import { EntityDefinitionWithState } from './types';
import { stopLatestTransform, stopTransforms } from './stop_transforms';
import { deleteLatestTransform, deleteTransforms } from './delete_transforms';
import { deleteIndices } from './delete_index';
export interface InstallDefinitionParams {
esClient: ElasticsearchClient;
@ -49,10 +50,7 @@ export async function installEntityDefinition({
validateDefinitionCanCreateValidTransformIds(definition);
if (await entityDefinitionExists(soClient, definition.id)) {
throw new EntityIdConflict(
`Entity definition with [${definition.id}] already exists.`,
definition
);
throw new EntityIdConflict(`Entity definition [${definition.id}] already exists.`, definition);
}
try {
@ -65,7 +63,7 @@ export async function installEntityDefinition({
return await install({ esClient, soClient, logger, definition: entityDefinition });
} catch (e) {
logger.error(`Failed to install entity definition ${definition.id}: ${e}`);
logger.error(`Failed to install entity definition [${definition.id}]: ${e}`);
await stopLatestTransform(esClient, definition, logger);
await deleteLatestTransform(esClient, definition, logger);
@ -90,28 +88,32 @@ export async function installEntityDefinition({
}
export async function installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
logger,
definitions,
}: Omit<InstallDefinitionParams, 'definition'> & {
}: Omit<InstallDefinitionParams, 'definition' | 'esClient'> & {
clusterClient: IScopedClusterClient;
definitions: EntityDefinition[];
}): Promise<EntityDefinition[]> {
if (definitions.length === 0) return [];
logger.debug(`Starting installation of ${definitions.length} built-in definitions`);
logger.info(`Checking installation of ${definitions.length} built-in definitions`);
const installPromises = definitions.map(async (builtInDefinition) => {
const installedDefinition = await findEntityDefinitionById({
esClient,
soClient,
esClient: clusterClient.asInternalUser,
id: builtInDefinition.id,
includeState: true,
});
if (!installedDefinition) {
// clean data from previous installation
await deleteIndices(clusterClient.asCurrentUser, builtInDefinition, logger);
return await installEntityDefinition({
definition: builtInDefinition,
esClient,
esClient: clusterClient.asSecondaryAuthUser,
soClient,
logger,
});
@ -127,15 +129,16 @@ export async function installBuiltInEntityDefinitions({
return installedDefinition;
}
logger.debug(
logger.info(
`Detected failed or outdated installation of definition [${installedDefinition.id}] v${installedDefinition.version}, installing v${builtInDefinition.version}`
);
return await reinstallEntityDefinition({
soClient,
esClient,
clusterClient,
logger,
definition: installedDefinition,
definitionUpdate: builtInDefinition,
deleteData: true,
});
});
@ -150,22 +153,16 @@ async function install({
definition,
logger,
}: InstallDefinitionParams): Promise<EntityDefinition> {
logger.debug(
() =>
`Installing definition ${definition.id} v${definition.version}\n${JSON.stringify(
definition,
null,
2
)}`
);
logger.info(`Installing definition [${definition.id}] v${definition.version}`);
logger.debug(() => JSON.stringify(definition, null, 2));
logger.debug(`Installing index templates for definition ${definition.id}`);
logger.debug(`Installing index templates for definition [${definition.id}]`);
const templates = await createAndInstallTemplates(esClient, definition, logger);
logger.debug(`Installing ingest pipelines for definition ${definition.id}`);
logger.debug(`Installing ingest pipelines for definition [${definition.id}]`);
const pipelines = await createAndInstallIngestPipelines(esClient, definition, logger);
logger.debug(`Installing transforms for definition ${definition.id}`);
logger.debug(`Installing transforms for definition [${definition.id}]`);
const transforms = await createAndInstallTransforms(esClient, definition, logger);
const updatedProps = await updateEntityDefinition(soClient, definition.id, {
@ -177,20 +174,23 @@ async function install({
// stop and delete the current transforms and reinstall all the components
export async function reinstallEntityDefinition({
esClient,
clusterClient,
soClient,
definition,
definitionUpdate,
logger,
}: InstallDefinitionParams & {
deleteData = false,
}: Omit<InstallDefinitionParams, 'esClient'> & {
clusterClient: IScopedClusterClient;
definitionUpdate: EntityDefinitionUpdate;
deleteData?: boolean;
}): Promise<EntityDefinition> {
try {
const updatedDefinition = mergeEntityDefinitionUpdate(definition, definitionUpdate);
logger.debug(
() =>
`Reinstalling definition ${definition.id} from v${definition.version} to v${
`Reinstalling definition [${definition.id}] from v${definition.version} to v${
definitionUpdate.version
}\n${JSON.stringify(updatedDefinition, null, 2)}`
);
@ -201,13 +201,17 @@ export async function reinstallEntityDefinition({
installStartedAt: new Date().toISOString(),
});
logger.debug(`Deleting transforms for definition ${definition.id} v${definition.version}`);
await stopAndDeleteTransforms(esClient, definition, logger);
logger.debug(`Deleting transforms for definition [${definition.id}] v${definition.version}`);
await stopAndDeleteTransforms(clusterClient.asSecondaryAuthUser, definition, logger);
if (deleteData) {
await deleteIndices(clusterClient.asCurrentUser, definition, logger);
}
return await install({
esClient,
soClient,
logger,
esClient: clusterClient.asSecondaryAuthUser,
definition: updatedDefinition,
});
} catch (err) {

4
x-pack/plugins/entity_manager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap generated
View file

@ -3,7 +3,7 @@
exports[`generateLatestTransform(definition) should generate a valid latest transform 1`] = `
Object {
"_meta": Object {
"definitionVersion": "1.0.0",
"definition_version": "1.0.0",
"managed": false,
},
"defer_validation": true,
@ -42,7 +42,7 @@ Object {
},
},
},
"entity.lastSeenTimestamp": Object {
"entity.last_seen_timestamp": Object {
"max": Object {
"field": "@timestamp",
},

4
x-pack/plugins/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
View file

@ -69,7 +69,7 @@ const generateTransformPutRequest = ({
return {
transform_id: transformId,
_meta: {
definitionVersion: definition.version,
definition_version: definition.version,
managed: definition.managed,
},
defer_validation: true,
@ -113,7 +113,7 @@ const generateTransformPutRequest = ({
aggs: {
...generateLatestMetricAggregations(definition),
...generateLatestMetadataAggregations(definition),
'entity.lastSeenTimestamp': {
'entity.last_seen_timestamp': {
max: {
field: definition.latest.timestampField,
},

8
x-pack/plugins/entity_manager/server/lib/entities/upgrade_entity_definition.ts
View file

@ -35,18 +35,20 @@ export async function upgradeBuiltInEntityDefinitions({
);
}
const { esClient, soClient } = getClientsFromAPIKey({ apiKey, server });
const { clusterClient, soClient } = getClientsFromAPIKey({ apiKey, server });
logger.debug(`Starting built-in definitions upgrade`);
const upgradedDefinitions = await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
definitions,
logger,
});
await Promise.all(
upgradedDefinitions.map((definition) => startTransforms(esClient, definition, logger))
upgradedDefinitions.map((definition) =>
startTransforms(clusterClient.asSecondaryAuthUser, definition, logger)
)
);
return { success: true, definitions: upgradedDefinitions };

33
x-pack/plugins/entity_manager/server/lib/entity_client.ts
View file

@ -40,6 +40,9 @@ export class EntityClient {
definition: EntityDefinition;
installOnly?: boolean;
}) {
this.options.logger.info(
`Creating definition [${definition.id}] v${definition.version} (installOnly=${installOnly})`
);
const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
const installedDefinition = await installEntityDefinition({
definition,
@ -62,16 +65,15 @@ export class EntityClient {
id: string;
definitionUpdate: EntityDefinitionUpdate;
}) {
const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
const definition = await findEntityDefinitionById({
id,
soClient: this.options.soClient,
esClient: secondaryAuthClient,
esClient: this.options.clusterClient.asInternalUser,
includeState: true,
});
if (!definition) {
const message = `Unable to find entity definition with [${id}]`;
const message = `Unable to find entity definition [${id}]`;
this.options.logger.error(message);
throw new EntityDefinitionNotFound(message);
}
@ -86,37 +88,46 @@ export class EntityClient {
definition as EntityDefinitionWithState
).state.components.transforms.some((transform) => transform.running);
this.options.logger.info(
`Updating definition [${definition.id}] from v${definition.version} to v${definitionUpdate.version}`
);
const updatedDefinition = await reinstallEntityDefinition({
definition,
definitionUpdate,
soClient: this.options.soClient,
esClient: secondaryAuthClient,
clusterClient: this.options.clusterClient,
logger: this.options.logger,
});
if (shouldRestartTransforms) {
await startTransforms(secondaryAuthClient, updatedDefinition, this.options.logger);
await startTransforms(
this.options.clusterClient.asSecondaryAuthUser,
updatedDefinition,
this.options.logger
);
}
return updatedDefinition;
}
async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser;
const definition = await findEntityDefinitionById({
id,
esClient: secondaryAuthClient,
esClient: this.options.clusterClient.asInternalUser,
soClient: this.options.soClient,
});
if (!definition) {
const message = `Unable to find entity definition with [${id}]`;
const message = `Unable to find entity definition [${id}]`;
this.options.logger.error(message);
throw new EntityDefinitionNotFound(message);
}
this.options.logger.info(
`Uninstalling definition [${definition.id}] v${definition.version} (deleteData=${deleteData})`
);
await uninstallEntityDefinition({
definition,
esClient: secondaryAuthClient,
esClient: this.options.clusterClient.asSecondaryAuthUser,
soClient: this.options.soClient,
logger: this.options.logger,
});
@ -148,7 +159,7 @@ export class EntityClient {
builtIn?: boolean;
}) {
const definitions = await findEntityDefinitions({
esClient: this.options.clusterClient.asSecondaryAuthUser,
esClient: this.options.clusterClient.asInternalUser,
soClient: this.options.soClient,
page,
perPage,
@ -162,6 +173,7 @@ export class EntityClient {
}
async startEntityDefinition(definition: EntityDefinition) {
this.options.logger.info(`Starting transforms for definition [${definition.id}]`);
return startTransforms(
this.options.clusterClient.asSecondaryAuthUser,
definition,
@ -170,6 +182,7 @@ export class EntityClient {
}
async stopEntityDefinition(definition: EntityDefinition) {
this.options.logger.info(`Stopping transforms for definition [${definition.id}]`);
return stopTransforms(
this.options.clusterClient.asSecondaryAuthUser,
definition,

6
x-pack/plugins/entity_manager/server/lib/manage_index_templates.ts
View file

@ -11,7 +11,6 @@ import {
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { entitiesHistoryBaseComponentTemplateConfig } from '../templates/components/base_history';
import { entitiesLatestBaseComponentTemplateConfig } from '../templates/components/base_latest';
import { entitiesEntityComponentTemplateConfig } from '../templates/components/entity';
import { entitiesEventComponentTemplateConfig } from '../templates/components/event';
@ -38,11 +37,6 @@ export const installEntityManagerTemplates = async ({
logger: Logger;
}) => {
await Promise.all([
upsertComponent({
esClient,
logger,
component: entitiesHistoryBaseComponentTemplateConfig,
}),
upsertComponent({
esClient,
logger,

8
x-pack/plugins/entity_manager/server/lib/utils.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import { getFakeKibanaRequest } from '@kbn/security-plugin/server/authentication/api_keys/fake_kibana_request';
import { EntityManagerServerSetup } from '../types';
@ -17,9 +17,9 @@ export const getClientsFromAPIKey = ({
}: {
apiKey: EntityDiscoveryAPIKey;
server: EntityManagerServerSetup;
}): { esClient: ElasticsearchClient; soClient: SavedObjectsClientContract } => {
}): { clusterClient: IScopedClusterClient; soClient: SavedObjectsClientContract } => {
const fakeRequest = getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey });
const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asSecondaryAuthUser;
const clusterClient = server.core.elasticsearch.client.asScoped(fakeRequest);
const soClient = server.core.savedObjects.getScopedClient(fakeRequest);
return { esClient, soClient };
return { clusterClient, soClient };
};

4
x-pack/plugins/entity_manager/server/routes/enablement/check.ts
View file

@ -61,13 +61,13 @@ export const checkEntityDiscoveryEnabledRoute = createEntityManagerServerRoute({
return response.ok({ body: { enabled: false, reason: ERROR_API_KEY_NOT_VALID } });
}
const { esClient, soClient } = getClientsFromAPIKey({ apiKey, server });
const { clusterClient, soClient } = getClientsFromAPIKey({ apiKey, server });
const entityDiscoveryState = await Promise.all(
builtInDefinitions.map(async (builtInDefinition) => {
const definitions = await findEntityDefinitions({
esClient,
soClient,
esClient: clusterClient.asSecondaryAuthUser,
id: builtInDefinition.id,
includeState: true,
});

4
x-pack/plugins/entity_manager/server/routes/enablement/disable.ts
View file

@ -67,12 +67,13 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
});
logger.info('Disabling managed entity discovery');
await uninstallBuiltInEntityDefinitions({
entityClient,
deleteData: params.query.deleteData,
});
server.logger.debug('reading entity discovery API key from saved object');
logger.debug('reading entity discovery API key from saved object');
const apiKey = await readEntityDiscoveryAPIKey(server);
// api key could be deleted outside of the apis, it does not affect the
// disablement flow
@ -82,6 +83,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({
ids: [apiKey.id],
});
}
logger.info('Managed entity discovery is disabled');
return response.ok({ body: { success: true } });
} catch (err) {

8
x-pack/plugins/entity_manager/server/routes/enablement/enable.ts
View file

@ -93,6 +93,7 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
});
}
logger.info(`Enabling managed entity discovery (installOnly=${params.query.installOnly})`);
const soClient = core.savedObjects.getClient({
includedHiddenTypes: [EntityDiscoveryApiKeyType.name],
});
@ -119,9 +120,9 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
await saveEntityDiscoveryAPIKey(soClient, apiKey);
const esClient = core.elasticsearch.client.asSecondaryAuthUser;
const clusterClient = core.elasticsearch.client;
const installedDefinitions = await installBuiltInEntityDefinitions({
esClient,
clusterClient,
soClient,
logger,
definitions: builtInDefinitions,
@ -130,10 +131,11 @@ export const enableEntityDiscoveryRoute = createEntityManagerServerRoute({
if (!params.query.installOnly) {
await Promise.all(
installedDefinitions.map((installedDefinition) =>
startTransforms(esClient, installedDefinition, logger)
startTransforms(clusterClient.asSecondaryAuthUser, installedDefinition, logger)
)
);
}
logger.info('Managed entity discovery is enabled');
return response.ok({ body: { success: true } });
} catch (err) {

78
x-pack/plugins/entity_manager/server/routes/entities/update.ts
View file

@ -5,21 +5,13 @@
* 2.0.
*/
import {
createEntityDefinitionQuerySchema,
entityDefinitionUpdateSchema,
} from '@kbn/entities-schema';
import { entityDefinitionUpdateSchema } from '@kbn/entities-schema';
import { z } from '@kbn/zod';
import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception';
import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error';
import { findEntityDefinitionById } from '../../lib/entities/find_entity_definition';
import { startTransforms } from '../../lib/entities/start_transforms';
import {
installationInProgress,
reinstallEntityDefinition,
} from '../../lib/entities/install_entity_definition';
import { createEntityManagerServerRoute } from '../create_entity_manager_server_route';
import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found';
import { EntityDefinitionUpdateConflict } from '../../lib/entities/errors/entity_definition_update_conflict';
/**
* @openapi
@ -29,13 +21,12 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_
* tags:
* - definitions
* parameters:
* - in: query
* name: installOnly
* description: If true, the definition transforms will not be started
* required: false
* - in: path
* name: id
* description: The entity definition ID
* schema:
* type: boolean
* default: false
* type: string
* required: true
* requestBody:
* description: The definition properties to update
* required: true
@ -63,58 +54,37 @@ export const updateEntityDefinitionRoute = createEntityManagerServerRoute({
endpoint: 'PATCH /internal/entities/definition/{id}',
params: z.object({
path: z.object({ id: z.string() }),
query: createEntityDefinitionQuerySchema,
body: entityDefinitionUpdateSchema,
}),
handler: async ({ context, response, params, logger }) => {
const core = await context.core;
const soClient = core.savedObjects.client;
const esClient = core.elasticsearch.client.asCurrentUser;
handler: async ({ request, response, params, logger, getScopedClient }) => {
const entityClient = await getScopedClient({ request });
try {
const installedDefinition = await findEntityDefinitionById({
soClient,
esClient,
const updatedDefinition = await entityClient.updateEntityDefinition({
id: params.path.id,
});
if (!installedDefinition) {
return response.notFound({
body: { message: `Entity definition [${params.path.id}] not found` },
});
}
if (installedDefinition.managed) {
return response.forbidden({
body: { message: `Managed definition cannot be modified` },
});
}
if (installationInProgress(installedDefinition)) {
return response.conflict({
body: { message: `Entity definition [${params.path.id}] has changes in progress` },
});
}
const updatedDefinition = await reinstallEntityDefinition({
soClient,
esClient,
logger,
definition: installedDefinition,
definitionUpdate: params.body,
});
if (!params.query.installOnly) {
await startTransforms(esClient, updatedDefinition, logger);
}
return response.ok({ body: updatedDefinition });
} catch (e) {
logger.error(e);
if (e instanceof EntityDefinitionNotFound) {
return response.notFound({
body: { message: `Entity definition [${params.path.id}] not found` },
});
}
if (e instanceof EntityDefinitionUpdateConflict) {
return response.conflict({
body: { message: `Entity definition [${params.path.id}] has changes in progress` },
});
}
if (e instanceof EntitySecurityException || e instanceof InvalidTransformError) {
return response.customError({ body: e, statusCode: 400 });
}
return response.customError({ body: e, statusCode: 500 });
}
},

36
x-pack/plugins/entity_manager/server/templates/components/base_history.ts
View file

@ -1,36 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
import { ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1 } from '../../../common/constants_entities';
export const entitiesHistoryBaseComponentTemplateConfig: ClusterPutComponentTemplateRequest = {
name: ENTITY_HISTORY_BASE_COMPONENT_TEMPLATE_V1,
_meta: {
description:
"Component template for the ECS fields used in the Elastic Entity Model's entity discovery framework's history data set",
documentation: 'https://www.elastic.co/guide/en/ecs/current/ecs-base.html',
ecs_version: '8.0.0',
managed: true,
},
template: {
mappings: {
properties: {
'@timestamp': {
type: 'date',
},
labels: {
type: 'object',
},
tags: {
ignore_above: 1024,
type: 'keyword',
},
},
},
},
};

5
x-pack/plugins/entity_manager/server/templates/components/base_latest.ts
View file

@ -22,7 +22,7 @@ export const entitiesLatestBaseComponentTemplateConfig: ClusterPutComponentTempl
properties: {
entity: {
properties: {
displayName: {
display_name: {
type: 'text',
fields: {
keyword: {
@ -31,9 +31,6 @@ export const entitiesLatestBaseComponentTemplateConfig: ClusterPutComponentTempl
},
},
},
firstSeenTimestamp: {
type: 'date',
},
},
},
labels: {

10
x-pack/plugins/entity_manager/server/templates/components/entity.ts
View file

@ -29,22 +29,22 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
ignore_above: 1024,
type: 'keyword',
},
definitionId: {
definition_id: {
ignore_above: 1024,
type: 'keyword',
},
definitionVersion: {
definition_version: {
ignore_above: 1024,
type: 'keyword',
},
schemaVersion: {
schema_version: {
ignore_above: 1024,
type: 'keyword',
},
lastSeenTimestamp: {
last_seen_timestamp: {
type: 'date',
},
identityFields: {
identity_fields: {
type: 'keyword',
},
},

5
x-pack/plugins/observability_solution/apm/server/routes/entities/types.ts
View file

@ -26,7 +26,6 @@ export interface EntityLatestServiceRaw {
interface Entity {
id: string;
lastSeenTimestamp: string;
firstSeenTimestamp: string;
identityFields: string[];
last_seen_timestamp: string;
identity_fields: string[];
}

75
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.test.ts
View file

@ -20,9 +20,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['metrics', 'logs'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
},
@ -49,9 +48,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['foo'] },
entity: {
firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-03-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:env-service-1',
},
},
@ -63,9 +61,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['bar'] },
entity: {
firstSeenTimestamp: '2024-03-05T10:34:40.810Z',
lastSeenTimestamp: '2024-03-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-03-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'apm-only-1:synthtrace-env-2',
},
},
@ -77,9 +74,8 @@ describe('mergeEntities', () => {
agent: { name: ['java'] },
source_data_stream: { type: ['baz'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-2:env-service-3',
},
},
@ -91,9 +87,8 @@ describe('mergeEntities', () => {
agent: { name: ['java'] },
source_data_stream: { type: ['baz'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-2:env-service-3',
},
},
@ -127,9 +122,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['metrics', 'logs'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
},
@ -141,9 +135,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['metrics', 'logs'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
},
@ -155,9 +148,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['foo'] },
entity: {
firstSeenTimestamp: '2024-23-05T10:34:40.810Z',
lastSeenTimestamp: '2024-23-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-23-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:prod',
},
},
@ -183,9 +175,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -209,9 +200,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -222,9 +212,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -250,9 +239,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -276,9 +264,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -289,9 +276,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: [] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name'],
id: 'service-1:test',
},
},
@ -318,9 +304,8 @@ describe('mergeEntities', () => {
agent: { name: ['nodejs'] },
source_data_stream: { type: ['metrics'] },
entity: {
firstSeenTimestamp: '2024-06-05T10:34:40.810Z',
lastSeenTimestamp: '2024-06-05T10:34:40.810Z',
identityFields: ['service.name', 'service.environment'],
last_seen_timestamp: '2024-06-05T10:34:40.810Z',
identity_fields: ['service.name', 'service.environment'],
id: 'service-1:test',
},
},

2
x-pack/plugins/observability_solution/apm/server/routes/entities/utils/merge_entities.ts
View file

@ -40,7 +40,7 @@ function mergeFunc(entity: EntityLatestServiceRaw, existingEntity?: MergedServic
const commonEntityFields = {
serviceName: entity.service.name,
agentName: entity.agent.name[0],
lastSeenTimestamp: entity.entity.lastSeenTimestamp,
lastSeenTimestamp: entity.entity.last_seen_timestamp,
};
if (!existingEntity) {

10
x-pack/plugins/observability_solution/inventory/common/utils/parse_identity_field_values_to_kql.test.ts
View file

@ -26,7 +26,7 @@ describe('parseIdentityFieldValuesToKql', () => {
it('should return the value when identityFields is a single string', () => {
const entity: ServiceEntity = {
'agent.name': 'node',
'entity.identityFields': 'service.name',
'entity.identity_fields': 'service.name',
'service.name': 'my-service',
'entity.type': 'service',
...commonEntityFields,
@ -39,7 +39,7 @@ describe('parseIdentityFieldValuesToKql', () => {
it('should return values when identityFields is an array of strings', () => {
const entity: ServiceEntity = {
'agent.name': 'node',
'entity.identityFields': ['service.name', 'service.environment'],
'entity.identity_fields': ['service.name', 'service.environment'],
'service.name': 'my-service',
'entity.type': 'service',
'service.environment': 'staging',
@ -53,7 +53,7 @@ describe('parseIdentityFieldValuesToKql', () => {
it('should return an empty string if identityFields is empty string', () => {
const entity: ServiceEntity = {
'agent.name': 'node',
'entity.identityFields': '',
'entity.identity_fields': '',
'service.name': 'my-service',
'entity.type': 'service',
...commonEntityFields,
@ -65,7 +65,7 @@ describe('parseIdentityFieldValuesToKql', () => {
it('should return an empty array if identityFields is empty array', () => {
const entity: ServiceEntity = {
'agent.name': 'node',
'entity.identityFields': [],
'entity.identity_fields': [],
'service.name': 'my-service',
'entity.type': 'service',
...commonEntityFields,
@ -77,7 +77,7 @@ describe('parseIdentityFieldValuesToKql', () => {
it('should ignore fields that are not present in the entity', () => {
const entity: HostEntity = {
'entity.identityFields': ['host.name', 'foo.bar'],
'entity.identity_fields': ['host.name', 'foo.bar'],
'host.name': 'my-host',
'entity.type': 'host',
'cloud.provider': null,

24
x-pack/plugins/observability_solution/inventory/public/components/alerts_badge/alerts_badge.test.tsx
View file

@ -28,13 +28,13 @@ describe('AlertsBadge', () => {
it('render alerts badge for a host entity', () => {
const entity: HostEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'host',
'entity.displayName': 'foo',
'entity.identityFields': 'host.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'host.name',
'host.name': 'foo',
'entity.definitionId': 'host',
'entity.definition_id': 'host',
'cloud.provider': null,
alertsCount: 1,
};
@ -46,14 +46,14 @@ describe('AlertsBadge', () => {
});
it('render alerts badge for a service entity', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'agent.name': 'node',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': 'service.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'service.name',
'service.name': 'bar',
'entity.definitionId': 'host',
'entity.definition_id': 'host',
'cloud.provider': null,
alertsCount: 5,
};
@ -65,15 +65,15 @@ describe('AlertsBadge', () => {
});
it('render alerts badge for a service entity with multiple identity fields', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'agent.name': 'node',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': ['service.name', 'service.environment'],
'entity.display_name': 'foo',
'entity.identity_fields': ['service.name', 'service.environment'],
'service.name': 'bar',
'service.environment': 'prod',
'entity.definitionId': 'host',
'entity.definition_id': 'host',
'cloud.provider': null,
alertsCount: 2,
};

48
x-pack/plugins/observability_solution/inventory/public/components/entities_grid/entity_name/entity_name.test.tsx
View file

@ -41,13 +41,13 @@ describe('EntityName', () => {
it('returns host link', () => {
const entity: HostEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'host',
'entity.displayName': 'foo',
'entity.identityFields': 'host.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'host.name',
'host.name': 'foo',
'entity.definitionId': 'host',
'entity.definition_id': 'host',
'cloud.provider': null,
};
render(<EntityName entity={entity} />);
@ -59,13 +59,13 @@ describe('EntityName', () => {
it('returns container link', () => {
const entity: ContainerEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'container',
'entity.displayName': 'foo',
'entity.identityFields': 'container.id',
'entity.display_name': 'foo',
'entity.identity_fields': 'container.id',
'container.id': 'foo',
'entity.definitionId': 'container',
'entity.definition_id': 'container',
'cloud.provider': null,
};
render(<EntityName entity={entity} />);
@ -77,13 +77,13 @@ describe('EntityName', () => {
it('returns service link without environment', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': 'service.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'service.name',
'service.name': 'foo',
'entity.definitionId': 'service',
'entity.definition_id': 'service',
'agent.name': 'bar',
};
render(<EntityName entity={entity} />);
@ -95,13 +95,13 @@ describe('EntityName', () => {
it('returns service link with environment', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': 'service.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'service.name',
'service.name': 'foo',
'entity.definitionId': 'service',
'entity.definition_id': 'service',
'agent.name': 'bar',
'service.environment': 'baz',
};
@ -114,13 +114,13 @@ describe('EntityName', () => {
it('returns service link with first environment when it is an array', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': 'service.name',
'entity.display_name': 'foo',
'entity.identity_fields': 'service.name',
'service.name': 'foo',
'entity.definitionId': 'service',
'entity.definition_id': 'service',
'agent.name': 'bar',
'service.environment': ['baz', 'bar', 'foo'],
};
@ -133,13 +133,13 @@ describe('EntityName', () => {
it('returns service link identity fields is an array', () => {
const entity: ServiceEntity = {
'entity.lastSeenTimestamp': 'foo',
'entity.last_seen_timestamp': 'foo',
'entity.id': '1',
'entity.type': 'service',
'entity.displayName': 'foo',
'entity.identityFields': ['service.name', 'service.environment'],
'entity.display_name': 'foo',
'entity.identity_fields': ['service.name', 'service.environment'],
'service.name': 'foo',
'entity.definitionId': 'service',
'entity.definition_id': 'service',
'agent.name': 'bar',
'service.environment': 'baz',
};

6
x-pack/plugins/observability_solution/inventory/server/routes/entities/get_identify_fields.test.ts
View file

@ -29,14 +29,14 @@ describe('getIdentityFields', () => {
it('should return a Map with unique entity types and their respective identity fields', () => {
const serviceEntity: ServiceEntity = {
'agent.name': 'node',
'entity.identityFields': ['service.name', 'service.environment'],
'entity.identity_fields': ['service.name', 'service.environment'],
'service.name': 'my-service',
'entity.type': 'service',
...commonEntityFields,
};
const hostEntity: HostEntity = {
'entity.identityFields': ['host.name'],
'entity.identity_fields': ['host.name'],
'host.name': 'my-host',
'entity.type': 'host',
'cloud.provider': null,
@ -44,7 +44,7 @@ describe('getIdentityFields', () => {
};
const containerEntity: ContainerEntity = {
'entity.identityFields': 'container.id',
'entity.identity_fields': 'container.id',
'host.name': 'my-host',
'entity.type': 'container',
'cloud.provider': null,

2
x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx
View file

@ -114,7 +114,7 @@ const formatEntityMetrics = (entity: EntityWithSource): string => {
.join(', ');
const entitySources = entity.sources.map((source) => source.dataStream).join(', ');
return dedent(`
Entity name: ${entity.displayName};
Entity name: ${entity.display_name};
Entity type: ${entity.type};
Entity metrics: ${entityMetrics};
Entity data streams: ${entitySources}

26
x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts
View file

@ -22,7 +22,7 @@ import {
} from '../clients/create_entities_es_client';
// the official types do not explicitly define sourceIndex in the schema, but it is present in the data at the time of writing this
type EntitiesLatest = z.infer<typeof entityLatestSchema> & { sourceIndex: string[] };
type EntitiesLatest = z.infer<typeof entityLatestSchema> & { source_index: string[] };
export async function getEntitiesWithSource({
serviceEnvironment,
@ -51,21 +51,21 @@ export async function getEntitiesWithSource({
for (const response of entityResponses) {
const processedEntities = await Promise.all(
response.map(async (entity: EntitiesLatest) => {
const sourceIndex = entity?.sourceIndex;
const sourceIndex = entity?.source_index;
if (!sourceIndex || !sourceIndex.length) return null;
const indices = await esClient.indices.get({ index: sourceIndex });
const sources = await fetchSources(indices);
return {
identityFields: entity?.entity.identityFields,
identity_fields: entity?.entity.identity_fields,
id: entity?.entity.id,
definitionId: entity?.entity.definitionId,
lastSeenTimestamp: entity?.entity.lastSeenTimestamp,
displayName: entity?.entity.displayName,
definition_id: entity?.entity.definition_id,
last_seen_timestamp: entity?.entity.last_seen_timestamp,
display_name: entity?.entity.display_name,
metrics: entity?.entity.metrics,
schemaVersion: entity?.entity.schemaVersion,
definitionVersion: entity?.entity.definitionVersion,
schema_version: entity?.entity.schema_version,
definition_version: entity?.entity.definition_version,
type: entity?.entity.type,
sources,
};
@ -104,7 +104,7 @@ const getFetchEntitiesPromises = ({
hostName?: string;
containerId?: string;
serviceEnvironment?: string;
}): Array<Promise<Array<{ sourceIndex: string[]; entity: EntitiesLatest['entity'] }>>> => {
}): Array<Promise<Array<{ source_index: string[]; entity: EntitiesLatest['entity'] }>>> => {
const shouldFilterForServiceEnvironment =
serviceEnvironment &&
serviceName &&
@ -139,7 +139,7 @@ const getFetchEntitiesPromises = ({
return [containersPromise, hostsPromise, servicesPromise].filter(
(promise) => promise !== null
) as Array<Promise<Array<{ sourceIndex: string[]; entity: EntitiesLatest['entity'] }>>>;
) as Array<Promise<Array<{ source_index: string[]; entity: EntitiesLatest['entity'] }>>>;
};
const getFetchEntityPromise = ({
@ -152,10 +152,10 @@ const getFetchEntityPromise = ({
shouldFetch: boolean;
shouldMatch: QueryDslQueryContainer[];
entitiesEsClient: EntitiesESClient;
}): Promise<Array<{ sourceIndex: string[]; entity: EntitiesLatest['entity'] }>> | null => {
}): Promise<Array<{ source_index: string[]; entity: EntitiesLatest['entity'] }>> | null => {
return shouldFetch
? entitiesEsClient
.search<{ sourceIndex: string[]; entity: EntitiesLatest['entity'] }>(index, {
.search<{ source_index: string[]; entity: EntitiesLatest['entity'] }>(index, {
body: {
query: {
bool: {
@ -167,7 +167,7 @@ const getFetchEntityPromise = ({
})
.then((response) => {
return response.hits.hits.map((hit) => {
return { sourceIndex: hit?._source.sourceIndex, entity: hit._source.entity };
return { source_index: hit?._source.source_index, entity: hit._source.entity };
});
})
: null;

10
x-pack/plugins/observability_solution/observability_shared/common/field_names/elasticsearch.ts
View file

@ -149,9 +149,9 @@ export const PROFILE_INUSE_SPACE = 'profile.inuse_space.bytes';
export const ENTITY = 'entity';
export const ENTITY_ID = 'entity.id';
export const ENTITY_TYPE = 'entity.type';
export const ENTITY_LAST_SEEN = 'entity.lastSeenTimestamp';
export const ENTITY_FIRST_SEEN = 'entity.firstSeenTimestamp';
export const ENTITY_DISPLAY_NAME = 'entity.displayName';
export const ENTITY_DEFINITION_ID = 'entity.definitionId';
export const ENTITY_IDENTITY_FIELDS = 'entity.identityFields';
export const ENTITY_LAST_SEEN = 'entity.last_seen_timestamp';
export const ENTITY_FIRST_SEEN = 'entity.first_seen_timestamp';
export const ENTITY_DISPLAY_NAME = 'entity.display_name';
export const ENTITY_DEFINITION_ID = 'entity.definition_id';
export const ENTITY_IDENTITY_FIELDS = 'entity.identity_fields';
export const SOURCE_DATA_STREAM_TYPE = 'source_data_stream.type';

2
x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/entities_list.tsx
View file

@ -34,7 +34,7 @@ export const EntitiesList: React.FC = () => {
const [limit, setLimit] = useState(10);
const { toggleStatus } = useQueryToggle(ENTITIES_LIST_TABLE_ID);
const [sorting, setSorting] = useState({
field: 'entity.lastSeenTimestamp',
field: 'entity.last_seen_timestamp',
direction: Direction.desc,
});

2
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_pipeline.ts
View file

@ -67,7 +67,7 @@ const buildIngestPipeline = ({
{
set: {
field: '@timestamp',
value: '{{entity.lastSeenTimestamp}}',
value: '{{entity.last_seen_timestamp}}',
},
},
{

13
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/elasticsearch_assets/ingest_processor_steps/remove_entity_definition_fields_step.ts
View file

@ -15,13 +15,12 @@ export const removeEntityDefinitionFieldsStep = (): IngestProcessorContainer =>
remove: {
ignore_failure: true,
field: [
'entity.lastSeenTimestamp',
'entity.schemaVersion',
'entity.definitionVersion',
'entity.identityFields',
'entity.definitionId',
'entity.displayName',
'entity.firstSeenTimestamp',
'entity.last_seen_timestamp',
'entity.schema_version',
'entity.definition_version',
'entity.identity_fields',
'entity.definition_id',
'entity.display_name',
],
},
});

2
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/entities/list.ts
View file

@ -53,7 +53,7 @@ export const listEntitiesRoute = (router: EntityAnalyticsRoutesDeps['router'], l
const {
page = 1,
per_page: perPage = 10,
sort_field: sortField = 'entity.lastSeenTimestamp',
sort_field: sortField = 'entity.last_seen_timestamp',
sort_order: sortOrder = 'desc',
entities_types: entityTypes,
filterQuery,

20
x-pack/test/api_integration/apis/entity_manager/definitions.ts
View file

@ -101,26 +101,6 @@ export default function ({ getService }: FtrProviderContext) {
await uninstallDefinition(supertest, { id: mockDefinition.id });
});
it('rejects updates to managed definitions', async () => {
await installDefinition(supertest, {
definition: { ...mockDefinition, managed: true },
installOnly: true,
});
await updateDefinition(supertest, {
id: mockDefinition.id,
update: {
version: '1.0.0',
latest: {
timestampField: '@updatedTimestampField',
},
},
expectedCode: 403,
});
await uninstallDefinition(supertest, { id: mockDefinition.id });
});
});
describe('entity data', () => {

28
x-pack/test/functional/es_archives/security_solution/entity_store/data.json
View file

@ -17,17 +17,16 @@
"hash": []
},
"entity": {
"lastSeenTimestamp": "2024-09-11T11:24:15.588Z",
"schemaVersion": "v1",
"definitionVersion": "1.0.0",
"displayName": "hinamatsumoto",
"identityFields": [
"last_seen_timestamp": "2024-09-11T11:24:15.588Z",
"schema_version": "v1",
"definition_version": "1.0.0",
"display_name": "hinamatsumoto",
"identity_fields": [
"user.name"
],
"id": "LBQAgKHGmpup0Kg9nlKmeQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "security_user_default"
"definition_id": "security_user_default"
}
}
}
@ -68,18 +67,17 @@
]
},
"entity": {
"lastSeenTimestamp": "2024-09-11T11:24:15.591Z",
"schemaVersion": "v1",
"definitionVersion": "1.0.0",
"displayName": "ali-ubuntu-server",
"identityFields": [
"last_seen_timestamp": "2024-09-11T11:24:15.591Z",
"schema_version": "v1",
"definition_version": "1.0.0",
"display_name": "ali-ubuntu-server",
"identity_fields": [
"host.name"
],
"id": "ZXKm6GEcUJY6NHkMgPPmGQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "security_host_default"
"definition_id": "security_host_default"
}
}
}
}
}

32
x-pack/test/functional/es_archives/security_solution/entity_store/mappings.json
View file

@ -35,15 +35,15 @@
"properties": {
"entity": {
"properties": {
"definitionId": {
"definition_id": {
"type": "keyword",
"ignore_above": 1024
},
"definitionVersion": {
"definition_version": {
"type": "keyword",
"ignore_above": 1024
},
"displayName": {
"display_name": {
"type": "text",
"fields": {
"keyword": {
@ -52,20 +52,17 @@
}
}
},
"firstSeenTimestamp": {
"type": "date"
},
"id": {
"type": "keyword",
"ignore_above": 1024
},
"identityFields": {
"identity_fields": {
"type": "keyword"
},
"lastSeenTimestamp": {
"last_seen_timestamp": {
"type": "date"
},
"schemaVersion": {
"schema_version": {
"type": "keyword",
"ignore_above": 1024
},
@ -196,15 +193,15 @@
"properties": {
"entity": {
"properties": {
"definitionId": {
"definition_id": {
"type": "keyword",
"ignore_above": 1024
},
"definitionVersion": {
"definition_version": {
"type": "keyword",
"ignore_above": 1024
},
"displayName": {
"display_name": {
"type": "text",
"fields": {
"keyword": {
@ -213,20 +210,17 @@
}
}
},
"firstSeenTimestamp": {
"type": "date"
},
"id": {
"type": "keyword",
"ignore_above": 1024
},
"identityFields": {
"identity_fields": {
"type": "keyword"
},
"lastSeenTimestamp": {
"last_seen_timestamp": {
"type": "date"
},
"schemaVersion": {
"schema_version": {
"type": "keyword",
"ignore_above": 1024
},
@ -300,4 +294,4 @@
}
}
}
}
}