[Synthtrace][Inventory] K8s entities support (#197077)

Part of https://github.com/elastic/kibana/issues/196155

New Synthtrace scenario created:
[k8s_entities.ts](https://github.com/elastic/kibana/pull/197077/files#diff-33a935a5fb8848d743e40d89b018ed8ac82ae992c0778cd6cea072d091aa0647)

```
node scripts/synthtrace k8s_entities.ts --clean --live
```


Cluster sample data:
```
 {
        "_index": ".entities.v1.latest.builtin_kubernetes_cluster_ecss_from_ecs_data",
        "_id": "2060900000000000",
        "_score": 1,
        "_source": {
          "entity": {
            "type": "kubernetes_cluster_ecs",
            "id": "2060900000000000",
            "definitionId": "builtin_kubernetes_cluster_ecs",
            "displayName": "cluster_foo",
            "lastSeenTimestamp": "2024-10-21T16:15:17.570Z"
          },
          "orchestrator": {
            "cluster": {
              "name": "cluster_foo"
            }
          },
          "event": {
            "ingested": "2024-10-21T16:15:17.570Z"
          }
        }
      },
      {
        "_index": ".entities.v1.latest.builtin_kubernetes_cluster_semconvs_from_ecs_data",
        "_id": "2060900000000000",
        "_score": 1,
        "_source": {
          "entity": {
            "type": "kubernetes_cluster_semconv",
            "id": "2060900000000000",
            "definitionId": "builtin_kubernetes_cluster_semconv",
            "displayName": "cluster_foo",
            "lastSeenTimestamp": "2024-10-21T16:15:17.570Z"
          },
          "k8s": {
            "cluster": {
              "uid": "cluster_foo"
            }
          },
          "event": {
            "ingested": "2024-10-21T16:15:17.570Z"
          }
        }
      },
```
This commit is contained in:
Cauê Marcondes 2024-10-22 16:43:29 +01:00 committed by GitHub
parent 2664ccf028
commit 5e40320d9f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 714 additions and 2 deletions

View file

@ -11,8 +11,19 @@ import { Fields } from '../entity';
import { serviceEntity } from './service_entity';
import { hostEntity } from './host_entity';
import { containerEntity } from './container_entity';
import { k8sClusterJobEntity } from './kubernetes/cluster_entity';
import { k8sCronJobEntity } from './kubernetes/cron_job_entity';
import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity';
import { k8sDeploymentEntity } from './kubernetes/deployment_entity';
import { k8sJobSetEntity } from './kubernetes/job_set_entity';
import { k8sNodeEntity } from './kubernetes/node_entity';
import { k8sPodEntity } from './kubernetes/pod_entity';
import { k8sReplicaSetEntity } from './kubernetes/replica_set';
import { k8sStatefulSetEntity } from './kubernetes/stateful_set';
import { k8sContainerEntity } from './kubernetes/container_entity';
export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
export type Schema = 'ecs' | 'semconv';
export type EntityFields = Fields &
Partial<{
@ -32,4 +43,20 @@ export type EntityFields = Fields &
[key: string]: any;
}>;
export const entities = { serviceEntity, hostEntity, containerEntity };
export const entities = {
serviceEntity,
hostEntity,
containerEntity,
k8s: {
k8sClusterJobEntity,
k8sCronJobEntity,
k8sDaemonSetEntity,
k8sDeploymentEntity,
k8sJobSetEntity,
k8sNodeEntity,
k8sPodEntity,
k8sReplicaSetEntity,
k8sStatefulSetEntity,
k8sContainerEntity,
},
};

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sClusterJobEntity({
schema,
name,
entityId,
...others
}: {
schema: Schema;
name: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cluster',
'orchestrator.cluster.name': name,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'cluster',
'k8s.cluster.uid': name,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sContainerEntity({
schema,
id,
entityId,
...others
}: {
schema: Schema;
id: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'container',
'kubernetes.container.id': id,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'container',
'container.id': id,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sCronJobEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'kubernetes.cronjob.name': name,
'kubernetes.cronjob.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'k8s.cronjob.name': name,
'k8s.cronjob.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sDaemonSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'kubernetes.daemonset.name': name,
'kubernetes.daemonset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'k8s.daemonset.name': name,
'k8s.daemonset.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sDeploymentEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'deployment',
'kubernetes.deployment.name': name,
'kubernetes.deployment.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'deployment',
'k8s.deployment.name': name,
'k8s.deployment.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { EntityFields, Schema } from '..';
import { Serializable } from '../../serializable';
const identityFieldsMap: Record<Schema, Record<string, string[]>> = {
ecs: {
pod: ['kubernetes.pod.name'],
cluster: ['orchestrator.cluster.name'],
cron_job: ['kubernetes.cronjob.name'],
daemon_set: ['kubernetes.daemonset.name'],
deployment: ['kubernetes.deployment.name'],
job: ['kubernetes.job.name'],
node: ['kubernetes.node.name'],
replica_set: ['kubernetes.replicaset.name'],
stateful_set: ['kubernetes.statefulset.name'],
container: ['kubernetes.container.id'],
},
semconv: {
pod: ['k8s.pod.name'],
cluster: ['k8s.cluster.uid'],
cron_job: ['k8s.cronjob.name'],
daemon_set: ['k8s.daemonset.name'],
deployment: ['k8s.deployment.name'],
job: ['k8s.job.name'],
node: ['k8s.node.uid'],
replica_set: ['k8s.replicaset.name'],
stateful_set: ['k8s.statefulset.name'],
container: ['container.id'],
},
};
export class K8sEntity extends Serializable<EntityFields> {
constructor(schema: Schema, fields: EntityFields) {
const entityType = fields['entity.type'];
if (entityType === undefined) {
throw new Error(`Entity type not defined: ${entityType}`);
}
const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`;
const identityFields = identityFieldsMap[schema][entityType];
if (identityFields === undefined || identityFields.length === 0) {
throw new Error(
`Identity fields not defined for schema: ${schema} and entity type: ${entityType}`
);
}
super({
...fields,
'entity.type': entityTypeWithSchema,
'entity.definitionId': `builtin_${entityTypeWithSchema}`,
'entity.identityFields': identityFields,
'entity.displayName': getDisplayName({ identityFields, fields }),
});
}
}
function getDisplayName({
identityFields,
fields,
}: {
identityFields: string[];
fields: EntityFields;
}) {
return identityFields
.map((field) => fields[field])
.filter((_) => _)
.join(':');
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sJobSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'job',
'kubernetes.job.name': name,
'kubernetes.job.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'job',
'k8s.job.name': name,
'k8s.job.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,46 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sNodeEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'node',
'kubernetes.node.name': name,
'kubernetes.node.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'node',
'k8s.node.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sPodEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'pod',
'kubernetes.pod.name': name,
'kubernetes.pod.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'pod',
'k8s.pod.name': name,
'k8s.pod.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sReplicaSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'replica_set',
'kubernetes.replicaset.name': name,
'kubernetes.replicaset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'replica_set',
'k8s.replicaset.name': name,
'k8s.replicaset.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sStatefulSetEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'stateful_set',
'kubernetes.statefulset.name': name,
'kubernetes.statefulset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}
return new K8sEntity(schema, {
'entity.type': 'stateful_set',
'k8s.statefulset.name': name,
'k8s.statefulset.uid': uid,
'k8s.cluster.name': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -82,7 +82,8 @@ function getRoutingTransform() {
const entityIndexName = `${entityType}s`;
document._action = {
index: {
_index: `.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`,
_index:
`.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`.toLocaleLowerCase(),
_id: document['entity.id'],
},
};

View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { EntityFields, entities, generateShortId } from '@kbn/apm-synthtrace-client';
import { Schema } from '@kbn/apm-synthtrace-client/src/lib/entities';
import { Scenario } from '../cli/scenario';
import { withClient } from '../lib/utils/with_client';
const CLUSTER_NAME = 'cluster_foo';
const CLUSTER_ENTITY_ID = generateShortId();
const POD_ENTITY_ID = generateShortId();
const POD_UID = generateShortId();
const REPLICA_SET_ENTITY_ID = generateShortId();
const REPLICA_SET_UID = generateShortId();
const DEPLOYMENT_ENTITY_ID = generateShortId();
const DEPLOYMENT_UID = generateShortId();
const STATEFUL_SET_ENTITY_ID = generateShortId();
const STATEFUL_SET_UID = generateShortId();
const DAEMON_SET_ENTITY_ID = generateShortId();
const DAEMON_SET_UID = generateShortId();
const JOB_SET_ENTITY_ID = generateShortId();
const JOB_SET_UID = generateShortId();
const CRON_JOB_ENTITY_ID = generateShortId();
const CRON_JOB_UID = generateShortId();
const NODE_ENTITY_ID = generateShortId();
const NODE_UID = generateShortId();
const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
const { logger } = runOptions;
return {
bootstrap: async ({ entitiesKibanaClient }) => {
await entitiesKibanaClient.installEntityIndexPatterns();
},
generate: ({ range, clients: { entitiesEsClient } }) => {
const getK8sEntitiesEvents = (schema: Schema) =>
range
.interval('1m')
.rate(1)
.generator((timestamp) => {
return [
entities.k8s
.k8sClusterJobEntity({
schema,
name: CLUSTER_NAME,
entityId: CLUSTER_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sPodEntity({
schema,
clusterName: CLUSTER_NAME,
name: 'pod_foo',
uid: POD_UID,
entityId: POD_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sReplicaSetEntity({
clusterName: CLUSTER_NAME,
name: 'replica_set_foo',
schema,
uid: REPLICA_SET_UID,
entityId: REPLICA_SET_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sDeploymentEntity({
clusterName: CLUSTER_NAME,
name: 'deployment_foo',
schema,
uid: DEPLOYMENT_UID,
entityId: DEPLOYMENT_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sStatefulSetEntity({
clusterName: CLUSTER_NAME,
name: 'stateful_set_foo',
schema,
uid: STATEFUL_SET_UID,
entityId: STATEFUL_SET_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sDaemonSetEntity({
clusterName: CLUSTER_NAME,
name: 'daemon_set_foo',
schema,
uid: DAEMON_SET_UID,
entityId: DAEMON_SET_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sJobSetEntity({
clusterName: CLUSTER_NAME,
name: 'job_set_foo',
schema,
uid: JOB_SET_UID,
entityId: JOB_SET_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sCronJobEntity({
clusterName: CLUSTER_NAME,
name: 'cron_job_foo',
schema,
uid: CRON_JOB_UID,
entityId: CRON_JOB_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sNodeEntity({
clusterName: CLUSTER_NAME,
name: 'node_job_foo',
schema,
uid: NODE_UID,
entityId: NODE_ENTITY_ID,
})
.timestamp(timestamp),
entities.k8s
.k8sContainerEntity({
id: '123',
schema,
entityId: NODE_ENTITY_ID,
})
.timestamp(timestamp),
];
});
const ecsEntities = getK8sEntitiesEvents('ecs');
const otelEntities = getK8sEntitiesEvents('semconv');
return [
withClient(
entitiesEsClient,
logger.perf('generating_entities_ecs_events', () => ecsEntities)
),
withClient(
entitiesEsClient,
logger.perf('generating_entities_otel_events', () => otelEntities)
),
];
},
};
};
export default scenario;