github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset quality] Dedicated column for namespace (#173087)

Browse source 
## Changes

1. Splits `datasetname + namespace` from Dataset name column.
2. Creates a dedicated column for `namespace`.
3. Bring the human readable name from datasets.


#### Before
<img width="2411" alt="image"
src="0df47bb9-3109-4670-af4a-e917bc75deb1">


#### After
<img width="2414" alt="image"
src="4f2c9db7-b8b4-4520-89b2-41ba0326524a">
This commit is contained in:
Yngrid Coello 2023-12-18 14:52:45 +01:00 • committed by GitHub
parent e3ef0c8c6c
commit 9a653373e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 224 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

9
x-pack/plugins/dataset_quality/common/api_types.ts
View file

@ -7,7 +7,7 @@
import * as rt from 'io-ts';
export const datasetStatRt = rt.intersection([
export const dataStreamStatRt = rt.intersection([
rt.type({
name: rt.string,
}),
@ -19,6 +19,8 @@ export const datasetStatRt = rt.intersection([
}),
]);
export type DataStreamStat = rt.TypeOf<typeof dataStreamStatRt>;
export const integrationIconRt = rt.intersection([
rt.type({
path: rt.string,
@ -39,9 +41,12 @@ export const integrationRt = rt.intersection([
title: rt.string,
version: rt.string,
icons: rt.array(integrationIconRt),
datasets: rt.record(rt.string, rt.string),
}),
]);
export type Integration = rt.TypeOf<typeof integrationRt>;
export const degradedDocsRt = rt.type({
dataset: rt.string,
percentage: rt.number,
@ -52,7 +57,7 @@ export type DegradedDocs = rt.TypeOf<typeof degradedDocsRt>;
export const getDataStreamsStatsResponseRt = rt.exact(
rt.intersection([
rt.type({
dataStreamsStats: rt.array(datasetStatRt),
dataStreamsStats: rt.array(dataStreamStatRt),
}),
rt.type({
integrations: rt.array(integrationRt),

13
x-pack/plugins/dataset_quality/common/data_streams_stats/data_stream_stat.ts
View file

@ -10,16 +10,18 @@ import { DataStreamStatType, IntegrationType } from './types';
export class DataStreamStat {
name: DataStreamStatType['name'];
namespace: string;
title: string;
size?: DataStreamStatType['size'];
sizeBytes?: DataStreamStatType['size_bytes'];
lastActivity?: DataStreamStatType['last_activity'];
sizeBytes?: DataStreamStatType['sizeBytes'];
lastActivity?: DataStreamStatType['lastActivity'];
integration?: IntegrationType;
degradedDocs?: number;
private constructor(dataStreamStat: DataStreamStat) {
this.name = dataStreamStat.name;
this.title = dataStreamStat.title ?? dataStreamStat.name;
this.namespace = dataStreamStat.namespace;
this.size = dataStreamStat.size;
this.sizeBytes = dataStreamStat.sizeBytes;
this.lastActivity = dataStreamStat.lastActivity;
@ -32,10 +34,11 @@ export class DataStreamStat {
const dataStreamStatProps = {
name: dataStreamStat.name,
title: `${dataset}-${namespace}`,
title: dataStreamStat.integration?.datasets?.[dataset] ?? dataset,
namespace,
size: dataStreamStat.size,
sizeBytes: dataStreamStat.size_bytes,
lastActivity: dataStreamStat.last_activity,
sizeBytes: dataStreamStat.sizeBytes,
lastActivity: dataStreamStat.lastActivity,
integration: dataStreamStat.integration
? Integration.create(dataStreamStat.integration)
: undefined,

13
x-pack/plugins/dataset_quality/public/components/dataset_quality/columns.tsx
View file

@ -7,6 +7,7 @@
import React from 'react';
import {
EuiBadge,
EuiBasicTableColumn,
EuiCode,
EuiFlexGroup,
@ -33,6 +34,10 @@ const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', {
defaultMessage: 'Dataset Name',
});
const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumnName', {
defaultMessage: 'Namespace',
});
const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', {
defaultMessage: 'Size',
});
@ -122,6 +127,14 @@ export const getDatasetQualitTableColumns = ({
);
},
},
{
name: namespaceColumnName,
field: 'namespace',
sortable: true,
render: (_, dataStreamStat: DataStreamStat) => (
<EuiBadge color="hollow">{dataStreamStat.namespace}</EuiBadge>
),
},
{
name: sizeColumnName,
field: 'size',

2
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts
View file

@ -6,8 +6,8 @@
*/
import type { ElasticsearchClient } from '@kbn/core/server';
import { DataStreamTypes } from '../../../types/default_api_types';
import { dataStreamService } from '../../../services';
import { DataStreamTypes } from '../../../types/data_stream';
export async function getDataStreams(options: {
esClient: ElasticsearchClient;

20
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts
View file

@ -79,32 +79,32 @@ describe('getDataStreams', () => {
{
name: 'logs-elastic_agent-default',
size: '1gb',
size_bytes: 1170805528,
last_activity: 1698916071000,
sizeBytes: 1170805528,
lastActivity: 1698916071000,
},
{
name: 'logs-elastic_agent.filebeat-default',
size: '1.3mb',
size_bytes: 1459100,
last_activity: 1698902209996,
sizeBytes: 1459100,
lastActivity: 1698902209996,
},
{
name: 'logs-elastic_agent.fleet_server-default',
size: '2.9mb',
size_bytes: 3052148,
last_activity: 1698914110010,
sizeBytes: 3052148,
lastActivity: 1698914110010,
},
{
name: 'logs-elastic_agent.metricbeat-default',
size: '1.6mb',
size_bytes: 1704807,
last_activity: 1698672046707,
sizeBytes: 1704807,
lastActivity: 1698672046707,
},
{
name: 'logs-test.test-default',
size: '6.2mb',
size_bytes: 6570447,
last_activity: 1698913802000,
sizeBytes: 6570447,
lastActivity: 1698913802000,
},
]);
});

8
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts
View file

@ -6,8 +6,8 @@
*/
import type { ElasticsearchClient } from '@kbn/core/server';
import { DataStreamTypes } from '../../../types/default_api_types';
import { dataStreamService } from '../../../services';
import { DataStreamTypes } from '../../../types/data_stream';
export async function getDataStreamsStats(options: {
esClient: ElasticsearchClient;
@ -24,9 +24,9 @@ export async function getDataStreamsStats(options: {
const mappedDataStreams = matchingDataStreamsStats.map((dataStream) => {
return {
name: dataStream.data_stream,
size: dataStream.store_size,
size_bytes: dataStream.store_size_bytes,
last_activity: dataStream.maximum_timestamp,
size: dataStream.store_size?.toString(),
sizeBytes: dataStream.store_size_bytes,
lastActivity: dataStream.maximum_timestamp,
};
});

2
x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts
View file

@ -14,7 +14,7 @@ import {
DATA_STREAM_TYPE,
_IGNORED,
} from '../../../common/es_fields';
import { DataStreamTypes } from '../../types/data_stream';
import { DataStreamTypes } from '../../types/default_api_types';
import { createDatasetQualityESClient, wildcardQuery } from '../../utils';
export async function getDegradedDocsPaginated(options: {

53
x-pack/plugins/dataset_quality/server/routes/data_streams/get_integrations.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PackageClient } from '@kbn/fleet-plugin/server';
import { DataStreamStat, Integration } from '../../../common/api_types';
export async function getIntegrations(options: {
packageClient: PackageClient;
dataStreams: DataStreamStat[];
}): Promise<Integration[]> {
const { packageClient, dataStreams } = options;
const packages = await packageClient.getPackages();
const installedPackages = dataStreams.map((item) => item.integration);
return Promise.all(
packages
.filter((pkg) => installedPackages.includes(pkg.name))
.map(async (p) => ({
name: p.name,
title: p.title,
version: p.version,
icons: p.icons,
datasets: await getDatasets({
packageClient,
name: p.name,
version: p.version,
}),
}))
);
}
const getDatasets = async (options: {
packageClient: PackageClient;
name: string;
version: string;
}) => {
const { packageClient, name, version } = options;
const pkg = await packageClient.getPackage(name, version);
return pkg.packageInfo.data_streams?.reduce(
(acc, curr) => ({
...acc,
[curr.dataset]: curr.title,
}),
{}
);
};

25
x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts
View file

@ -7,20 +7,19 @@
import * as t from 'io-ts';
import { keyBy, merge, values } from 'lodash';
import { DataStreamStat } from '../../types/data_stream';
import { dataStreamTypesRt, rangeRt } from '../../types/default_api_types';
import { Integration } from '../../types/integration';
import { typeRt, rangeRt } from '../../types/default_api_types';
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
import { getDataStreams } from './get_data_streams';
import { getDataStreamsStats } from './get_data_streams_stats';
import { getDegradedDocsPaginated } from './get_degraded_docs';
import { DegradedDocs } from '../../../common/api_types';
import { DegradedDocs, DataStreamStat, Integration } from '../../../common/api_types';
import { getIntegrations } from './get_integrations';
const statsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/stats',
params: t.type({
query: t.intersection([
dataStreamTypesRt,
typeRt,
t.partial({
datasetQuery: t.string,
}),
@ -41,7 +40,6 @@ const statsRoute = createDatasetQualityServerRoute({
const fleetPluginStart = await plugins.fleet.start();
const packageClient = fleetPluginStart.packageService.asInternalUser;
const packages = await packageClient.getPackages();
const [dataStreams, dataStreamsStats] = await Promise.all([
getDataStreams({
@ -52,22 +50,11 @@ const statsRoute = createDatasetQualityServerRoute({
getDataStreamsStats({ esClient, ...params.query }),
]);
const installedPackages = dataStreams.items.map((item) => item.integration);
const integrations = packages
.filter((pkg) => installedPackages.includes(pkg.name))
.map((p) => ({
name: p.name,
title: p.title,
version: p.version,
icons: p.icons,
}));
return {
dataStreamsStats: values(
merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name'))
),
integrations,
integrations: await getIntegrations({ packageClient, dataStreams: dataStreams.items }),
};
},
});
@ -77,7 +64,7 @@ const degradedDocsRoute = createDatasetQualityServerRoute({
params: t.type({
query: t.intersection([
rangeRt,
dataStreamTypesRt,
typeRt,
t.partial({
datasetQuery: t.string,
}),

18
x-pack/plugins/dataset_quality/server/types/data_stream.ts
View file

@ -1,18 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ByteSize } from '@elastic/elasticsearch/lib/api/types';
import { Integration } from './integration';
export interface DataStreamStat {
name: string;
size?: ByteSize;
size_bytes?: number;
last_activity?: number;
integration?: Integration;
}
export type DataStreamTypes = 'logs' | 'metrics' | 'traces' | 'synthetics' | 'profiling';

21
x-pack/plugins/dataset_quality/server/types/default_api_types.ts
View file

@ -8,16 +8,21 @@
import * as t from 'io-ts';
import { isoToEpochRt } from '@kbn/io-ts-utils';
export const dataStreamTypesRt = t.partial({
type: t.union([
t.literal('logs'),
t.literal('metrics'),
t.literal('traces'),
t.literal('synthetics'),
t.literal('profiling'),
]),
// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals
export const dataStreamTypesRt = t.keyof({
logs: null,
metrics: null,
traces: null,
synthetics: null,
profiling: null,
});
export const typeRt = t.partial({
type: dataStreamTypesRt,
});
export type DataStreamTypes = t.TypeOf<typeof dataStreamTypesRt>;
export const rangeRt = t.type({
start: isoToEpochRt,
end: isoToEpochRt,

21
x-pack/plugins/dataset_quality/server/types/integration.ts
View file

@ -1,21 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface Integration {
name: string;
title?: string;
version?: string;
icons?: IntegrationIcon[];
}
export interface IntegrationIcon {
path: string;
src: string;
title?: string;
size?: string;
type?: string;
}

98
x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts Normal file
View file

@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { DatasetQualityApiClientKey } from '../../common/config';
import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const synthtrace = getService('logSynthtraceEsClient');
const datasetQualityApiClient = getService('datasetQualityApiClient');
const start = '2023-12-11T18:00:00.000Z';
const end = '2023-12-11T18:01:00.000Z';
async function callApiAs(user: DatasetQualityApiClientKey) {
return await datasetQualityApiClient[user]({
endpoint: 'GET /internal/dataset_quality/data_streams/degraded_docs',
params: {
query: {
type: 'logs',
start,
end,
},
},
});
}
registry.when('Degraded docs', { config: 'basic' }, () => {
describe('and there are log documents', () => {
before(async () => {
await synthtrace.index([
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset('synth.1')
.defaults({
'log.file.path': '/my-service.log',
})
),
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset('synth.2')
.logLevel(
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'
)
.defaults({
'log.file.path': '/my-service.log',
})
),
]);
});
it('returns stats correctly', async () => {
const stats = await callApiAs('datasetQualityLogsUser');
expect(stats.body.degradedDocs.length).to.be(2);
const percentages = stats.body.degradedDocs.reduce(
(acc, curr) => ({
...acc,
[curr.dataset]: curr.percentage,
}),
{} as Record<string, number>
);
expect(percentages['logs-synth.1-default']).to.be(0);
expect(percentages['logs-synth.2-default']).to.be(100);
});
after(async () => {
await synthtrace.clean();
});
});
describe('and there are not log documents', () => {
it('returns stats correctly', async () => {
const stats = await callApiAs('datasetQualityLogsUser');
expect(stats.body.degradedDocs.length).to.be(0);
});
});
});
}

0
x-pack/test/dataset_quality_api_integration/tests/es_utils.ts → x-pack/test/dataset_quality_api_integration/tests/data_streams/es_utils.ts
View file

20
x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts → x-pack/test/dataset_quality_api_integration/tests/data_streams/stats.spec.ts
View file

@ -7,10 +7,10 @@
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { DatasetQualityApiClientKey } from '../common/config';
import { DatasetQualityApiError } from '../common/dataset_quality_api_supertest';
import { FtrProviderContext } from '../common/ftr_provider_context';
import { expectToReject } from '../utils';
import { DatasetQualityApiClientKey } from '../../common/config';
import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { expectToReject } from '../../utils';
import { cleanLogIndexTemplate, addIntegrationToLogIndexTemplate } from './es_utils';
export default function ApiTest({ getService }: FtrProviderContext) {
@ -43,7 +43,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
describe('when required privileges are set', () => {
describe('and uncategorized datastreams', () => {
describe('and categorized datastreams', () => {
const integration = 'my-custom-integration';
before(async () => {
@ -67,8 +67,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(stats.body.dataStreamsStats.length).to.be(1);
expect(stats.body.dataStreamsStats[0].integration).to.be(integration);
expect(stats.body.dataStreamsStats[0].size).not.empty();
expect(stats.body.dataStreamsStats[0].size_bytes).greaterThan(0);
expect(stats.body.dataStreamsStats[0].last_activity).greaterThan(0);
expect(stats.body.dataStreamsStats[0].sizeBytes).greaterThan(0);
expect(stats.body.dataStreamsStats[0].lastActivity).greaterThan(0);
});
after(async () => {
@ -77,7 +77,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});
describe('and categorized datastreams', () => {
describe('and uncategorized datastreams', () => {
before(async () => {
await synthtrace.index([
timerange('2023-11-20T15:00:00.000Z', '2023-11-20T15:01:00.000Z')
@ -97,8 +97,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
expect(stats.body.dataStreamsStats.length).to.be(1);
expect(stats.body.dataStreamsStats[0].integration).not.ok();
expect(stats.body.dataStreamsStats[0].size).not.empty();
expect(stats.body.dataStreamsStats[0].size_bytes).greaterThan(0);
expect(stats.body.dataStreamsStats[0].last_activity).greaterThan(0);
expect(stats.body.dataStreamsStats[0].sizeBytes).greaterThan(0);
expect(stats.body.dataStreamsStats[0].lastActivity).greaterThan(0);
});
after(async () => {