[Inventory] Inventory k8s entities fixes (#201260)

closes [#201226](https://github.com/elastic/kibana/issues/201226)

## Summary

This PR makes the final adjustments on k8s entities after the
https://github.com/elastic/kibana/pull/196916 was merged.

I had to fix most of the ECS entities because they had their `entityId`
defined with `uid` field. Most of the ECS K8s entities don't have this
field and should use the `name` field instead (see
[metricsets](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-kubernetes.html#_metricsets_41))

~I also had to fix the transforms to include an aggregation for the
`displayNameTemplate` field, when it doesn't match with the `entity.Id`~

### Real data Otel

The screenshots below are from a tests running the `opentemeletry-demo`
sending otel data

<img width="710" alt="image"
src="https://github.com/user-attachments/assets/d223858d-3b99-4bb0-b69e-3b70112a2c17">

<img width="710" alt="image"
src="https://github.com/user-attachments/assets/1ef27dcd-682c-4681-b92b-be9b4f2b32b8">

### Real data ECS

The screenshots below are from a tests running the `opentemeletry-demo`
with elastic agent installed with Kubernetes integration

<img width="710" alt="image"
src="https://github.com/user-attachments/assets/e2fb6cbd-60a0-4995-bc94-8ccbc8911db1">
<img width="710" alt="image"
src="https://github.com/user-attachments/assets/a3d9dba3-fee7-42af-972b-34a151b52f2b">


### Additional test with synthtrace

<img width="709" alt="image"
src="https://github.com/user-attachments/assets/bf56da1e-0bbb-40a0-9f44-06b9f86427a8">

### Fix

ECS k8s service entity was missing the link to its corresponding
dashboard

<img width="709" alt="image"
src="https://github.com/user-attachments/assets/488891a6-1c61-4001-b604-208478e1c798">

<img width="709" alt="image"
src="https://github.com/user-attachments/assets/0b30c56b-db52-4d82-a7e7-100dae91a35e">


### How to test

- Start local kibana and es instances
- run ` node scripts/synthtrace k8s_entities.ts --clean --live`
- Navigate to Inventory and enable EEM

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Carlos Crespo 2024-11-26 09:29:23 +01:00 committed by GitHub
parent 3188cda4e3
commit aead7b9acd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 282 additions and 107 deletions

View file

@ -15,15 +15,16 @@ import { k8sClusterJobEntity } from './kubernetes/cluster_entity';
import { k8sCronJobEntity } from './kubernetes/cron_job_entity';
import { k8sDaemonSetEntity } from './kubernetes/daemon_set_entity';
import { k8sDeploymentEntity } from './kubernetes/deployment_entity';
import { k8sJobSetEntity } from './kubernetes/job_set_entity';
import { k8sJobEntity } from './kubernetes/job_entity';
import { k8sNodeEntity } from './kubernetes/node_entity';
import { k8sPodEntity } from './kubernetes/pod_entity';
import { k8sReplicaSetEntity } from './kubernetes/replica_set';
import { k8sStatefulSetEntity } from './kubernetes/stateful_set';
import { k8sServiceEntity } from './kubernetes/service';
import { k8sContainerEntity } from './kubernetes/container_entity';
export type EntityDataStreamType = 'metrics' | 'logs' | 'traces';
export type Schema = 'ecs' | 'semconv';
export type Schema = 'ecs' | 'otel';
export type EntityFields = Fields &
Partial<{
@ -52,11 +53,12 @@ export const entities = {
k8sCronJobEntity,
k8sDaemonSetEntity,
k8sDeploymentEntity,
k8sJobSetEntity,
k8sJobEntity,
k8sNodeEntity,
k8sPodEntity,
k8sReplicaSetEntity,
k8sStatefulSetEntity,
k8sServiceEntity,
k8sContainerEntity,
},
};

View file

@ -23,6 +23,7 @@ export function k8sClusterJobEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'cluster',
'entity.type': 'cluster',
'orchestrator.cluster.name': name,
'entity.id': entityId,
@ -31,6 +32,7 @@ export function k8sClusterJobEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'cluster',
'entity.type': 'cluster',
'k8s.cluster.uid': name,
'entity.id': entityId,

View file

@ -23,6 +23,7 @@ export function k8sContainerEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'container',
'entity.type': 'container',
'kubernetes.container.id': id,
'entity.id': entityId,
@ -31,6 +32,7 @@ export function k8sContainerEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'container',
'entity.type': 'container',
'container.id': id,
'entity.id': entityId,

View file

@ -27,9 +27,9 @@ export function k8sCronJobEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'entity.definition_id': 'cron_job',
'entity.type': 'cronjob',
'kubernetes.cronjob.name': name,
'kubernetes.cronjob.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,7 +37,8 @@ export function k8sCronJobEntity({
}
return new K8sEntity(schema, {
'entity.type': 'cron_job',
'entity.definition_id': 'cron_job',
'entity.type': 'cronjob',
'k8s.cronjob.name': name,
'k8s.cronjob.uid': uid,
'k8s.cluster.name': clusterName,

View file

@ -27,7 +27,8 @@ export function k8sDaemonSetEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'entity.definition_id': 'daemon_set',
'entity.type': 'daemonset',
'kubernetes.daemonset.name': name,
'kubernetes.daemonset.uid': uid,
'kubernetes.namespace': clusterName,
@ -37,7 +38,8 @@ export function k8sDaemonSetEntity({
}
return new K8sEntity(schema, {
'entity.type': 'daemon_set',
'entity.definition_id': 'daemon_set',
'entity.type': 'daemonset',
'k8s.daemonset.name': name,
'k8s.daemonset.uid': uid,
'k8s.cluster.name': clusterName,

View file

@ -27,9 +27,9 @@ export function k8sDeploymentEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'deployment',
'entity.type': 'deployment',
'kubernetes.deployment.name': name,
'kubernetes.deployment.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,6 +37,7 @@ export function k8sDeploymentEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'deployment',
'entity.type': 'deployment',
'k8s.deployment.name': name,
'k8s.deployment.uid': uid,

View file

@ -12,27 +12,28 @@ import { Serializable } from '../../serializable';
const identityFieldsMap: Record<Schema, Record<string, string[]>> = {
ecs: {
pod: ['kubernetes.pod.name'],
pod: ['kubernetes.pod.uid'],
cluster: ['orchestrator.cluster.name'],
cron_job: ['kubernetes.cronjob.name'],
daemon_set: ['kubernetes.daemonset.name'],
cronjob: ['kubernetes.cronjob.name'],
daemonset: ['kubernetes.daemonset.name'],
deployment: ['kubernetes.deployment.name'],
job: ['kubernetes.job.name'],
node: ['kubernetes.node.name'],
replica_set: ['kubernetes.replicaset.name'],
stateful_set: ['kubernetes.statefulset.name'],
replicaset: ['kubernetes.replicaset.name'],
statefulset: ['kubernetes.statefulset.name'],
service: ['kubernetes.service.name'],
container: ['kubernetes.container.id'],
},
semconv: {
pod: ['k8s.pod.name'],
otel: {
pod: ['k8s.pod.uid'],
cluster: ['k8s.cluster.uid'],
cron_job: ['k8s.cronjob.name'],
daemon_set: ['k8s.daemonset.name'],
deployment: ['k8s.deployment.name'],
job: ['k8s.job.name'],
cronjob: ['k8s.cronjob.uid'],
daemonset: ['k8s.daemonset.uid'],
deployment: ['k8s.deployment.uid'],
job: ['k8s.job.uid'],
node: ['k8s.node.uid'],
replica_set: ['k8s.replicaset.name'],
stateful_set: ['k8s.statefulset.name'],
replicaset: ['k8s.replicaset.uid'],
statefulset: ['k8s.statefulset.uid'],
container: ['container.id'],
},
};
@ -41,10 +42,17 @@ export class K8sEntity extends Serializable<EntityFields> {
constructor(schema: Schema, fields: EntityFields) {
const entityType = fields['entity.type'];
if (entityType === undefined) {
throw new Error(`Entity type not defined: ${entityType}`);
throw new Error(`Entity type not defined`);
}
const entityTypeWithSchema = `kubernetes_${entityType}_${schema}`;
const entityDefinitionId = fields['entity.definition_id'];
if (entityDefinitionId === undefined) {
throw new Error(`Entity definition id not defined`);
}
const entityDefinitionWithSchema = `kubernetes_${entityDefinitionId}_${
schema === 'ecs' ? schema : 'semconv'
}`;
const identityFields = identityFieldsMap[schema][entityType];
if (identityFields === undefined || identityFields.length === 0) {
throw new Error(
@ -54,8 +62,8 @@ export class K8sEntity extends Serializable<EntityFields> {
super({
...fields,
'entity.type': entityTypeWithSchema,
'entity.definition_id': `builtin_${entityTypeWithSchema}`,
'entity.type': `k8s.${entityType}.${schema}`,
'entity.definition_id': `builtin_${entityDefinitionWithSchema}`,
'entity.identity_fields': identityFields,
'entity.display_name': getDisplayName({ identityFields, fields }),
'entity.definition_version': '1.0.0',

View file

@ -10,7 +10,7 @@
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sJobSetEntity({
export function k8sJobEntity({
schema,
name,
uid,
@ -27,9 +27,9 @@ export function k8sJobSetEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'job',
'entity.type': 'job',
'kubernetes.job.name': name,
'kubernetes.job.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,6 +37,7 @@ export function k8sJobSetEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'job',
'entity.type': 'job',
'k8s.job.name': name,
'k8s.job.uid': uid,

View file

@ -27,9 +27,9 @@ export function k8sNodeEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'node',
'entity.type': 'node',
'kubernetes.node.name': name,
'kubernetes.node.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,6 +37,7 @@ export function k8sNodeEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'node',
'entity.type': 'node',
'k8s.node.uid': uid,
'k8s.cluster.name': clusterName,

View file

@ -27,6 +27,7 @@ export function k8sPodEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.definition_id': 'pod',
'entity.type': 'pod',
'kubernetes.pod.name': name,
'kubernetes.pod.uid': uid,
@ -37,6 +38,7 @@ export function k8sPodEntity({
}
return new K8sEntity(schema, {
'entity.definition_id': 'pod',
'entity.type': 'pod',
'k8s.pod.name': name,
'k8s.pod.uid': uid,

View file

@ -27,9 +27,9 @@ export function k8sReplicaSetEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'replica_set',
'entity.definition_id': 'replica_set',
'entity.type': 'replicaset',
'kubernetes.replicaset.name': name,
'kubernetes.replicaset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,7 +37,8 @@ export function k8sReplicaSetEntity({
}
return new K8sEntity(schema, {
'entity.type': 'replica_set',
'entity.definition_id': 'replica_set',
'entity.type': 'replicaset',
'k8s.replicaset.name': name,
'k8s.replicaset.uid': uid,
'k8s.cluster.name': clusterName,

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Schema } from '..';
import { K8sEntity } from '.';
export function k8sServiceEntity({
schema,
name,
uid,
clusterName,
entityId,
...others
}: {
schema: Schema;
name: string;
uid?: string;
clusterName?: string;
entityId: string;
[key: string]: any;
}) {
if (schema !== 'ecs') {
throw new Error('Schema not supported for service entity: ' + schema);
}
return new K8sEntity(schema, {
'entity.definition_id': 'service',
'entity.type': 'service',
'kubernetes.service.name': name,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
});
}

View file

@ -27,9 +27,9 @@ export function k8sStatefulSetEntity({
}) {
if (schema === 'ecs') {
return new K8sEntity(schema, {
'entity.type': 'stateful_set',
'entity.definition_id': 'stateful_set',
'entity.type': 'statefulset',
'kubernetes.statefulset.name': name,
'kubernetes.statefulset.uid': uid,
'kubernetes.namespace': clusterName,
'entity.id': entityId,
...others,
@ -37,7 +37,8 @@ export function k8sStatefulSetEntity({
}
return new K8sEntity(schema, {
'entity.type': 'stateful_set',
'entity.definition_id': 'stateful_set',
'entity.type': 'statefulset',
'k8s.statefulset.name': name,
'k8s.statefulset.uid': uid,
'k8s.cluster.name': clusterName,

View file

@ -75,15 +75,13 @@ function getRoutingTransform() {
return new Transform({
objectMode: true,
transform(document: ESDocumentWithOperation<EntityFields>, encoding, callback) {
const entityType: string | undefined = document['entity.type'];
if (entityType === undefined) {
throw new Error(`entity.type was not defined: ${JSON.stringify(document)}`);
const definitionId: string | undefined = document['entity.definition_id'];
if (definitionId === undefined) {
throw new Error(`entity.definition_id was not defined: ${JSON.stringify(document)}`);
}
const entityIndexName = `${entityType}s`;
document._action = {
index: {
_index:
`.entities.v1.latest.builtin_${entityIndexName}_from_ecs_data`.toLocaleLowerCase(),
_index: `.entities.v1.latest.${definitionId}`.toLocaleLowerCase(),
_id: document['entity.id'],
},
};

View file

@ -31,6 +31,7 @@ const CRON_JOB_ENTITY_ID = generateShortId();
const CRON_JOB_UID = generateShortId();
const NODE_ENTITY_ID = generateShortId();
const NODE_UID = generateShortId();
const SERVICE_UID = generateShortId();
const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
const { logger } = runOptions;
@ -45,7 +46,7 @@ const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
.interval('1m')
.rate(1)
.generator((timestamp) => {
return [
const commonEntities = [
entities.k8s
.k8sClusterJobEntity({
schema,
@ -99,7 +100,7 @@ const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
})
.timestamp(timestamp),
entities.k8s
.k8sJobSetEntity({
.k8sJobEntity({
clusterName: CLUSTER_NAME,
name: 'job_set_foo',
schema,
@ -133,10 +134,26 @@ const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
})
.timestamp(timestamp),
];
if (schema === 'ecs') {
return [
...commonEntities,
entities.k8s
.k8sServiceEntity({
schema,
clusterName: CLUSTER_NAME,
name: 'my_service',
entityId: SERVICE_UID,
})
.timestamp(timestamp),
];
}
return commonEntities;
});
const ecsEntities = getK8sEntitiesEvents('ecs');
const otelEntities = getK8sEntitiesEvents('semconv');
const otelEntities = getK8sEntitiesEvents('otel');
return [
withClient(

View file

@ -100,7 +100,7 @@ const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
})
.timestamp(timestamp),
entities.k8s
.k8sJobSetEntity({
.k8sJobEntity({
clusterName: CLUSTER_NAME,
name: 'job_set_foo',
schema,
@ -137,7 +137,7 @@ const scenario: Scenario<Partial<EntityFields>> = async (runOptions) => {
});
const ecsEntities = getK8sEntitiesEvents('ecs');
const otelEntities = getK8sEntitiesEvents('semconv');
const otelEntities = getK8sEntitiesEvents('otel');
const synthJavaTraces = entities.serviceEntity({
serviceName: 'synth_java',
agentName: ['java'],

View file

@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_cron_job_ecs`,
filter: 'kubernetes.cronjob.uid : *',
filter: 'kubernetes.cronjob.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes CronJob from ECS data',
@ -21,7 +21,7 @@ export const builtInKubernetesCronJobEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes cron job entities from the Kubernetes integration data streams',
type: 'k8s.cronjob.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.cronjob.uid'],
identityFields: ['kubernetes.cronjob.name'],
displayNameTemplate: '{{kubernetes.cronjob.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesDaemonSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_daemon_set_ecs`,
filter: 'kubernetes.daemonset.uid : *',
filter: 'kubernetes.daemonset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes DaemonSet from ECS data',

View file

@ -13,7 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_deployment_ecs`,
filter: 'kubernetes.deployment.uid : *',
filter: 'kubernetes.deployment.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Deployment from ECS data',
@ -21,7 +21,7 @@ export const builtInKubernetesDeploymentEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes deployment entities from the Kubernetes integration data streams',
type: 'k8s.deployment.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.deployment.uid'],
identityFields: ['kubernetes.deployment.name'],
displayNameTemplate: '{{kubernetes.deployment.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_job_ecs`,
filter: 'kubernetes.job.uid : *',
filter: 'kubernetes.job.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Job from ECS data',
@ -21,7 +21,7 @@ export const builtInKubernetesJobEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes job entities from the Kubernetes integration data streams',
type: 'k8s.job.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.job.uid'],
identityFields: ['kubernetes.job.name'],
displayNameTemplate: '{{kubernetes.job.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -13,7 +13,7 @@ import { commonEcsMetadata } from '../common/ecs_metadata';
export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_node_ecs`,
filer: 'kubernetes.node.uid : *',
filer: 'kubernetes.node.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes Node from ECS data',
@ -21,7 +21,7 @@ export const builtInKubernetesNodeEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes node entities from the Kubernetes integration data streams',
type: 'k8s.node.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.node.uid'],
identityFields: ['kubernetes.node.name'],
displayNameTemplate: '{{kubernetes.node.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -21,7 +21,7 @@ export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes pod entities from the Kubernetes integration data streams',
type: 'k8s.pod.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.pod.name'],
identityFields: ['kubernetes.pod.uid'],
displayNameTemplate: '{{kubernetes.pod.name}}',
latest: {
timestampField: '@timestamp',
@ -30,5 +30,12 @@ export const builtInKubernetesPodEcsEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonEcsMetadata,
metadata: [
...commonEcsMetadata,
{
source: 'kubernetes.pod.name',
destination: 'kubernetes.pod.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -13,6 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesReplicaSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_replica_set_ecs`,
filer: 'kubernetes.replicaset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes ReplicaSet from ECS data',
@ -20,7 +21,7 @@ export const builtInKubernetesReplicaSetEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes replica set entities from the Kubernetes integration data streams',
type: 'k8s.replicaset.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.replicaset.uid'],
identityFields: ['kubernetes.replicaset.name'],
displayNameTemplate: '{{kubernetes.replicaset.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -13,7 +13,7 @@ import { commonEcsIndexPatterns } from '../common/ecs_index_patterns';
export const builtInKubernetesStatefulSetEcsEntityDefinition: EntityDefinition =
entityDefinitionSchema.parse({
id: `${BUILT_IN_ID_PREFIX}kubernetes_stateful_set_ecs`,
filter: 'kubernetes.statefulset.uid : *',
filter: 'kubernetes.statefulset.name : *',
managed: true,
version: '0.1.0',
name: 'Kubernetes StatefulSet from ECS data',
@ -21,7 +21,7 @@ export const builtInKubernetesStatefulSetEcsEntityDefinition: EntityDefinition =
'This definition extracts Kubernetes stateful set entities from the Kubernetes integration data streams',
type: 'k8s.statefulset.ecs',
indexPatterns: commonEcsIndexPatterns,
identityFields: ['kubernetes.statefulset.uid'],
identityFields: ['kubernetes.statefulset.name'],
displayNameTemplate: '{{kubernetes.statefulset.name}}',
latest: {
timestampField: '@timestamp',

View file

@ -30,5 +30,12 @@ export const builtInKubernetesClusterSemConvEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.cluster.name',
destination: 'k8s.cluster.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesCronJobSemConvEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.cronjob.name',
destination: 'k8s.cronjob.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesDaemonSetSemConvEntityDefinition: EntityDefinition
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.daemonset.name',
destination: 'k8s.daemonset.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesDeploymentSemConvEntityDefinition: EntityDefinitio
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.deployment.name',
destination: 'k8s.deployment.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesJobSemConvEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.job.name',
destination: 'k8s.job.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -22,7 +22,7 @@ export const builtInKubernetesNodeSemConvEntityDefinition: EntityDefinition =
type: 'k8s.node.otel',
indexPatterns: commonOtelIndexPatterns,
identityFields: ['k8s.node.uid'],
displayNameTemplate: '{{k8s.node.uid}}',
displayNameTemplate: '{{k8s.node.name}}',
latest: {
timestampField: '@timestamp',
lookbackPeriod: '10m',
@ -30,5 +30,12 @@ export const builtInKubernetesNodeSemConvEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.node.name',
destination: 'k8s.node.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesPodSemConvEntityDefinition: EntityDefinition =
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.pod.name',
destination: 'k8s.pod.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -19,9 +19,9 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio
name: 'Kubernetes ReplicaSet from SemConv data',
description:
'This definition extracts Kubernetes replica set entities using data collected with OpenTelemetry',
type: 'kubernetes_replica_set_semconv',
type: 'k8s.replicaset.otel',
indexPatterns: commonOtelIndexPatterns,
identityFields: ['k8s.replicaset.name'],
identityFields: ['k8s.replicaset.uid'],
displayNameTemplate: '{{k8s.replicaset.name}}',
latest: {
timestampField: '@timestamp',
@ -30,5 +30,12 @@ export const builtInKubernetesReplicaSetSemConvEntityDefinition: EntityDefinitio
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.replicaset.name',
destination: 'k8s.replicaset.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -30,5 +30,12 @@ export const builtInKubernetesStatefulSetSemConvEntityDefinition: EntityDefiniti
frequency: '5m',
},
},
metadata: commonOtelMetadata,
metadata: [
...commonOtelMetadata,
{
source: 'k8s.statefulset.name',
destination: 'k8s.statefulset.name',
aggregation: { type: 'top_value', sort: { '@timestamp': 'desc' } },
},
],
});

View file

@ -6,10 +6,16 @@
*/
import * as z from '@kbn/zod';
import { EntityDataStreamType, ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import {
EntityDataStreamType,
BUILT_IN_ENTITY_TYPES,
} from '@kbn/observability-shared-plugin/common';
import { useFetcher } from '../../../hooks/use_fetcher';
const EntityTypeSchema = z.union([z.literal(ENTITY_TYPES.HOST), z.literal(ENTITY_TYPES.CONTAINER)]);
const EntityTypeSchema = z.union([
z.literal(BUILT_IN_ENTITY_TYPES.HOST),
z.literal(BUILT_IN_ENTITY_TYPES.CONTAINER),
]);
const EntityDataStreamSchema = z.union([
z.literal(EntityDataStreamType.METRICS),
z.literal(EntityDataStreamType.LOGS),

View file

@ -23,7 +23,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import { getFieldByType } from '@kbn/metrics-data-access-plugin/common';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import useLocalStorage from 'react-use/lib/useLocalStorage';
import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { useSourceContext } from '../../../../containers/metrics_source';
import { isPending, useFetcher } from '../../../../hooks/use_fetcher';
import { parseSearchString } from './parse_search_string';
@ -58,7 +58,7 @@ export const Processes = () => {
const { request$ } = useRequestObservable();
const { isActiveTab } = useTabSwitcherContext();
const { dataStreams, status: dataStreamsStatus } = useEntitySummary({
entityType: ENTITY_TYPES.HOST,
entityType: BUILT_IN_ENTITY_TYPES.HOST,
entityId: asset.name,
});
const addMetricsCalloutId: AddMetricsCalloutKey = 'hostProcesses';

View file

@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema';
import { METRICS_APP_ID } from '@kbn/deeplinks-observability/constants';
import { entityCentricExperience } from '@kbn/observability-plugin/common';
import { createObservabilityEsClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
import { ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { BUILT_IN_ENTITY_TYPES } from '@kbn/observability-shared-plugin/common';
import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
import { InfraBackendLibs } from '../../lib/infra_types';
import { getDataStreamTypes } from './get_data_stream_types';
@ -24,8 +24,8 @@ export const initEntitiesConfigurationRoutes = (libs: InfraBackendLibs) => {
validate: {
params: schema.object({
entityType: schema.oneOf([
schema.literal(ENTITY_TYPES.HOST),
schema.literal(ENTITY_TYPES.CONTAINER),
schema.literal(BUILT_IN_ENTITY_TYPES.HOST),
schema.literal(BUILT_IN_ENTITY_TYPES.CONTAINER),
]),
entityId: schema.string(),
}),

View file

@ -45,7 +45,7 @@ export function EntityIcon({ entity }: EntityIconProps) {
return <AgentIcon agentName={castArray(entity.agent?.name)[0]} role="presentation" />;
}
if (entity.entityType.startsWith('kubernetes')) {
if (entity.entityType.startsWith('k8s')) {
return <EuiIcon type="logoKubernetes" size="l" />;
}

View file

@ -10,7 +10,7 @@ import { useDetailViewRedirect } from './use_detail_view_redirect';
import { useKibana } from './use_kibana';
import {
CONTAINER_ID,
ENTITY_TYPES,
BUILT_IN_ENTITY_TYPES,
HOST_NAME,
SERVICE_NAME,
} from '@kbn/observability-shared-plugin/common';
@ -134,15 +134,30 @@ describe('useDetailViewRedirect', () => {
});
[
[ENTITY_TYPES.KUBERNETES.CLUSTER.ecs, 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c'],
[ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'],
[ENTITY_TYPES.KUBERNETES.CRONJOB.ecs, 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs, 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs, 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'],
[ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs, 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs,
'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv, 'kubernetes_otel-cluster-overview'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs,
'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs,
'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs,
'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs, 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013'],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs, 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013'],
[BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs, 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013'],
[
BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs,
'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
],
].forEach(([entityType, dashboardId]) => {
it(`getEntityRedirectUrl should return the correct URL for ${entityType} entity`, () => {
const entity: InventoryEntity = {

View file

@ -6,7 +6,7 @@
*/
import {
ASSET_DETAILS_LOCATOR_ID,
ENTITY_TYPES,
BUILT_IN_ENTITY_TYPES,
SERVICE_OVERVIEW_LOCATOR_ID,
type AssetDetailsLocatorParams,
type ServiceOverviewParams,
@ -20,15 +20,19 @@ import type { InventoryEntity } from '../../common/entities';
import { useKibana } from './use_kibana';
const KUBERNETES_DASHBOARDS_IDS: Record<string, string> = {
[ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
[ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview',
[ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]: 'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]: 'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013',
[ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]: 'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.ecs]: 'kubernetes-f4dc26db-1b53-4ea2-a78b-1bfab8ea267c',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CLUSTER.semconv]: 'kubernetes_otel-cluster-overview',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.CRONJOB.ecs]: 'kubernetes-0a672d50-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.DAEMONSET.ecs]:
'kubernetes-85879010-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.DEPLOYMENT.ecs]:
'kubernetes-5be46210-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.JOB.ecs]: 'kubernetes-9bf990a0-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.NODE.ecs]: 'kubernetes-b945b7b0-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.POD.ecs]: 'kubernetes-3d4d9290-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.SERVICE.ecs]: 'kubernetes-ff1b3850-bcb1-11ec-b64f-7dd6e8e82013',
[BUILT_IN_ENTITY_TYPES.KUBERNETES.STATEFULSET.ecs]:
'kubernetes-21694370-bcb2-11ec-b64f-7dd6e8e82013',
};
export const useDetailViewRedirect = () => {

View file

@ -6,11 +6,11 @@
*/
const createKubernetesEntity = <T extends string>(base: T) => ({
ecs: `kubernetes_${base}_ecs` as const,
semconv: `kubernetes_${base}_semconv` as const,
ecs: `k8s.${base}.ecs` as const,
semconv: `k8s.${base}.semconv` as const,
});
export const ENTITY_TYPES = {
export const BUILT_IN_ENTITY_TYPES = {
HOST: 'host',
CONTAINER: 'container',
SERVICE: 'service',
@ -18,12 +18,13 @@ export const ENTITY_TYPES = {
CLUSTER: createKubernetesEntity('cluster'),
CONTAINER: createKubernetesEntity('container'),
CRONJOB: createKubernetesEntity('cron_job'),
DAEMONSET: createKubernetesEntity('daemon_set'),
DAEMONSET: createKubernetesEntity('daemonset'),
DEPLOYMENT: createKubernetesEntity('deployment'),
JOB: createKubernetesEntity('job'),
NAMESPACE: createKubernetesEntity('namespace'),
NODE: createKubernetesEntity('node'),
POD: createKubernetesEntity('pod'),
STATEFULSET: createKubernetesEntity('stateful_set'),
SERVICE: createKubernetesEntity('service'),
STATEFULSET: createKubernetesEntity('statefulset'),
},
} as const;

View file

@ -5,5 +5,5 @@
* 2.0.
*/
export { ENTITY_TYPES } from './entity_types';
export { BUILT_IN_ENTITY_TYPES } from './entity_types';
export { EntityDataStreamType } from './entity_data_stream_types';

View file

@ -219,4 +219,4 @@ export {
export { COMMON_OBSERVABILITY_GROUPING } from './embeddable_grouping';
export { ENTITY_TYPES, EntityDataStreamType } from './entity';
export { BUILT_IN_ENTITY_TYPES, EntityDataStreamType } from './entity';