github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset quality] Including integration information within stats (#171663)

Browse source 
### Changes
- `sortOrder` was removed as a param
- `GET /internal/dataset_quality/data_streams/stats` now includes
information related to integrations (name, title, version, icons)

Response now looks like
```
{
  "items": [
    {
      "name": "logs-elastic_agent-default",
      "size": "1.5mb",
      "size_bytes": 1645068,
      "last_activity": 1700378618104
    },
    {
      "name": "logs-elastic_agent.filebeat-default",
      "size": "3.6mb",
      "size_bytes": 3860281,
      "last_activity": 1700472909107
    },
    {
      "name": "logs-elastic_agent.metricbeat-default",
      "size": "1.6mb",
      "size_bytes": 1773572,
      "last_activity": 1700142440834
    },
    {
      "name": "logs-system.auth-default",
      "integration": "system",
      "size": "6.8mb",
      "size_bytes": 7226358,
      "last_activity": 1700472604014
    },
    {
      "name": "logs-system.syslog-default",
      "integration": "system",
      "size": "1.2gb",
      "size_bytes": 1302089724,
      "last_activity": 1700473867000
    }
  ],
  "integrations": [
    {
      "name": "system",
      "title": "System",
      "version": "1.49.0",
      "icons": [
        {
          "src": "/img/system.svg",
          "path": "/package/system/1.49.0/img/system.svg",
          "title": "system",
          "size": "1000x1000",
          "type": "image/svg+xml"
        }
      ]
    }
  ]
}
```
This commit is contained in:
Yngrid Coello 2023-11-22 18:27:04 +01:00 • committed by GitHub
parent 1bc5b6f43e
commit 74dea1e2c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 88 additions and 129 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/dataset_quality/kibana.jsonc
View file

@ -8,7 +8,7 @@
"server": true,
"browser": true,
"configPath": ["xpack", "datasetQuality"],
"requiredPlugins": ["data", "kibanaReact", "kibanaUtils", "controls", "embeddable", "share"],
"requiredPlugins": ["data", "kibanaReact", "kibanaUtils", "controls", "embeddable", "share", "fleet"],
"optionalPlugins": [],
"requiredBundles": [],
"extraPublicDirs": ["common"]

50
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/get_data_streams.test.ts
View file

@ -114,7 +114,6 @@ describe('getDataStreams', () => {
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'asc',
uncategorisedOnly: true,
});
expect(dataStreamService.getMatchingDataStreams).toHaveBeenCalledWith(expect.anything(), {
@ -129,7 +128,6 @@ describe('getDataStreams', () => {
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'asc',
uncategorisedOnly: true,
});
expect(results.items.length).toBe(1);
@ -140,62 +138,36 @@ describe('getDataStreams', () => {
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'asc',
uncategorisedOnly: false,
});
expect(results.items.length).toBe(5);
});
});
describe('Can be sorted', () => {
it('Ascending', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreams({
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'asc',
uncategorisedOnly: false,
});
expect(results.items[0].name).toBe('logs-elastic_agent-default');
});
it('Descending', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreams({
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'desc',
uncategorisedOnly: false,
});
expect(results.items[0].name).toBe('logs-test.test-default');
});
});
it('Formats the items correctly', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreams({
esClient: esClientMock,
type: 'logs',
sortOrder: 'desc',
uncategorisedOnly: false,
});
expect(results.items).toEqual([
{ name: 'logs-test.test-default' },
expect(results.items.sort()).toEqual([
{
name: 'logs-elastic_agent.metricbeat-default',
integration: { name: 'elastic_agent', managed_by: 'fleet' },
},
{
name: 'logs-elastic_agent.fleet_server-default',
integration: { name: 'elastic_agent', managed_by: 'fleet' },
name: 'logs-elastic_agent-default',
integration: 'elastic_agent',
},
{
name: 'logs-elastic_agent.filebeat-default',
integration: { name: 'elastic_agent', managed_by: 'fleet' },
integration: 'elastic_agent',
},
{
name: 'logs-elastic_agent-default',
integration: { name: 'elastic_agent', managed_by: 'fleet' },
name: 'logs-elastic_agent.fleet_server-default',
integration: 'elastic_agent',
},
{
name: 'logs-elastic_agent.metricbeat-default',
integration: 'elastic_agent',
},
{ name: 'logs-test.test-default' },
]);
});
});

22
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams/index.ts
View file

@ -13,10 +13,9 @@ export async function getDataStreams(options: {
esClient: ElasticsearchClient;
type?: DataStreamTypes;
datasetQuery?: string;
sortOrder: 'asc' | 'desc';
uncategorisedOnly: boolean;
}) {
const { esClient, type, datasetQuery, uncategorisedOnly, sortOrder } = options;
const { esClient, type, datasetQuery, uncategorisedOnly } = options;
const allDataStreams = await dataStreamService.getMatchingDataStreams(esClient, {
type: type ?? '*',
@ -31,25 +30,10 @@ export async function getDataStreams(options: {
const mappedDataStreams = filteredDataStreams.map((dataStream) => ({
name: dataStream.name,
...(dataStream._meta
? {
integration: {
name: dataStream._meta?.package?.name,
managed_by: dataStream._meta?.managed_by,
},
}
: {}),
integration: dataStream._meta?.package?.name,
}));
const sortedDataStreams = mappedDataStreams.sort((a, b) => {
if (sortOrder === 'desc') {
return b.name.localeCompare(a.name);
}
return a.name.localeCompare(b.name);
});
return {
items: sortedDataStreams,
items: mappedDataStreams,
};
}

64
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/get_data_streams_stats.test.ts
View file

@ -63,58 +63,24 @@ describe('getDataStreams', () => {
esClient: esClientMock,
type: 'logs',
datasetQuery: 'nginx',
sortOrder: 'asc',
});
expect(dataStreamService.getMatchingDataStreamsStats).toHaveBeenCalledWith(expect.anything(), {
type: 'logs',
dataset: '*nginx*',
});
});
describe('Can be sorted', () => {
it('Ascending', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreamsStats({
esClient: esClientMock,
type: 'logs',
sortOrder: 'asc',
});
expect(results.items[0].name).toBe('logs-elastic_agent-default');
});
it('Descending', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreamsStats({
esClient: esClientMock,
type: 'logs',
sortOrder: 'desc',
});
expect(results.items[0].name).toBe('logs-test.test-default');
});
});
it('Formats the items correctly', async () => {
const esClientMock = elasticsearchServiceMock.createElasticsearchClient();
const results = await getDataStreamsStats({
esClient: esClientMock,
type: 'logs',
sortOrder: 'desc',
});
expect(results.items).toEqual([
expect(results.items.sort()).toEqual([
{
name: 'logs-test.test-default',
size: '6.2mb',
size_bytes: 6570447,
last_activity: 1698913802000,
},
{
name: 'logs-elastic_agent.metricbeat-default',
size: '1.6mb',
size_bytes: 1704807,
last_activity: 1698672046707,
},
{
name: 'logs-elastic_agent.fleet_server-default',
size: '2.9mb',
size_bytes: 3052148,
last_activity: 1698914110010,
name: 'logs-elastic_agent-default',
size: '1gb',
size_bytes: 1170805528,
last_activity: 1698916071000,
},
{
name: 'logs-elastic_agent.filebeat-default',
@ -123,10 +89,22 @@ describe('getDataStreams', () => {
last_activity: 1698902209996,
},
{
name: 'logs-elastic_agent-default',
size: '1gb',
size_bytes: 1170805528,
last_activity: 1698916071000,
name: 'logs-elastic_agent.fleet_server-default',
size: '2.9mb',
size_bytes: 3052148,
last_activity: 1698914110010,
},
{
name: 'logs-elastic_agent.metricbeat-default',
size: '1.6mb',
size_bytes: 1704807,
last_activity: 1698672046707,
},
{
name: 'logs-test.test-default',
size: '6.2mb',
size_bytes: 6570447,
last_activity: 1698913802000,
},
]);
});

13
x-pack/plugins/dataset_quality/server/routes/data_streams/get_data_streams_stats/index.ts
View file

@ -13,9 +13,8 @@ export async function getDataStreamsStats(options: {
esClient: ElasticsearchClient;
type?: DataStreamTypes;
datasetQuery?: string;
sortOrder: 'asc' | 'desc';
}) {
const { esClient, type, datasetQuery, sortOrder } = options;
const { esClient, type, datasetQuery } = options;
const matchingDataStreamsStats = await dataStreamService.getMatchingDataStreamsStats(esClient, {
type: type ?? '*',
@ -31,15 +30,7 @@ export async function getDataStreamsStats(options: {
};
});
const sortedDataStreams = mappedDataStreams.sort((a, b) => {
if (sortOrder === 'desc') {
return b.name.localeCompare(a.name);
}
return a.name.localeCompare(b.name);
});
return {
items: sortedDataStreams,
items: mappedDataStreams,
};
}

21
x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts
View file

@ -7,7 +7,7 @@
import * as t from 'io-ts';
import { keyBy, merge, values } from 'lodash';
import { dataStreamTypesRt, sortOrderRt } from '../../types/api_types';
import { dataStreamTypesRt } from '../../types/api_types';
import { DataStreamsStatResponse } from '../../types/data_stream';
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
import { getDataStreams } from './get_data_streams';
@ -21,19 +21,22 @@ const statsRoute = createDatasetQualityServerRoute({
t.partial({
datasetQuery: t.string,
}),
sortOrderRt,
]),
}),
options: {
tags: [],
},
async handler(resources): Promise<DataStreamsStatResponse> {
const { context, params } = resources;
const { context, params, plugins } = resources;
const coreContext = await context.core;
// Query datastreams as the current user as the Kibana internal user may not have all the required permissions
const esClient = coreContext.elasticsearch.client.asCurrentUser;
const fleetPluginStart = await plugins.fleet.start();
const packageClient = fleetPluginStart.packageService.asInternalUser;
const packages = await packageClient.getPackages();
const [dataStreams, dataStreamsStats] = await Promise.all([
getDataStreams({
esClient,
@ -43,8 +46,20 @@ const statsRoute = createDatasetQualityServerRoute({
getDataStreamsStats({ esClient, ...params.query }),
]);
const installedPackages = dataStreams.items.map((item) => item.integration);
const integrations = packages
.filter((pkg) => installedPackages.includes(pkg.name))
.map((p) => ({
name: p.name,
title: p.title,
version: p.version,
icons: p.icons,
}));
return {
items: values(merge(keyBy(dataStreams.items, 'name'), keyBy(dataStreamsStats.items, 'name'))),
integrations,
};
},
});

11
x-pack/plugins/dataset_quality/server/types.ts
View file

@ -6,12 +6,15 @@
*/
import { CustomRequestHandlerContext } from '@kbn/core/server';
import { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface DatasetQualityPluginSetupDependencies {}
export interface DatasetQualityPluginSetupDependencies {
fleet: FleetSetupContract;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface DatasetQualityPluginStartDependencies {}
export interface DatasetQualityPluginStartDependencies {
fleet: FleetStartContract;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface DatasetQualityPluginSetup {}

4
x-pack/plugins/dataset_quality/server/types/api_types.ts
View file

@ -16,7 +16,3 @@ export const dataStreamTypesRt = t.partial({
t.literal('profiling'),
]),
});
export const sortOrderRt = t.type({
sortOrder: t.union([t.literal('asc'), t.literal('desc')]),
});

7
x-pack/plugins/dataset_quality/server/types/data_stream.ts
View file

@ -6,9 +6,11 @@
*/
import { ByteSize } from '@elastic/elasticsearch/lib/api/types';
import { Integration } from './integration';
export interface DataStreamsStatResponse {
items: DataStreamStat[];
integrations: Integration[];
}
export interface DataStreamStat {
@ -16,10 +18,7 @@ export interface DataStreamStat {
size?: ByteSize;
size_bytes?: number;
last_activity?: number;
integration?: {
name?: string;
managed_by?: string;
};
integration?: Integration;
}
export type DataStreamTypes = 'logs' | 'metrics' | 'traces' | 'synthetics' | 'profiling';

21
x-pack/plugins/dataset_quality/server/types/integration.ts Normal file
View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface Integration {
name: string;
title?: string;
version?: string;
icons?: IntegrationIcon[];
}
export interface IntegrationIcon {
path: string;
src: string;
title?: string;
size?: string;
type?: string;
}

1
x-pack/plugins/dataset_quality/tsconfig.json
View file

@ -15,6 +15,7 @@
"@kbn/core-plugins-server",
"@kbn/core-elasticsearch-server-mocks",
"@kbn/std",
"@kbn/fleet-plugin",
],
"exclude": [
"target/**/*",

1
x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts
View file

@ -21,7 +21,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
params: {
query: {
type: 'logs',
sortOrder: 'asc',
},
},
});