[Data Usage] process autoops mock data (#195640)

- validates autoOps response data using mock data and new type
- processes autoOps data to return an object of {x,y} values from our
API instead of array of [timestamp, value]. updates UI accordingly
This commit is contained in:
Sandra G 2024-10-10 15:59:48 -04:00 committed by GitHub
parent 6eed6298af
commit 3ec190823f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 153 additions and 178 deletions

View file

@ -10,48 +10,29 @@ import { UsageMetricsRequestSchema } from './usage_metrics';
describe('usage_metrics schemas', () => {
it('should accept valid request query', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['storage_retained'],
})
).not.toThrow();
});
it('should accept a single `metricTypes` in request query', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: 'ingest_rate',
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
})
).not.toThrow();
});
it('should accept multiple `metricTypes` in request query', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['ingest_rate', 'storage_retained', 'index_rate'],
})
).not.toThrow();
});
it('should accept a single string as `dataStreams` in request query', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: 'storage_retained',
dataStreams: 'data_stream_1',
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
})
).not.toThrow();
});
it('should accept `dataStream` list', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['storage_retained'],
@ -62,74 +43,76 @@ describe('usage_metrics schemas', () => {
it('should error if `dataStream` list is empty', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['storage_retained'],
dataStreams: [],
})
).toThrowError('expected value of type [string] but got [Array]');
).toThrowError('[dataStreams]: array size is [0], but cannot be smaller than [1]');
});
it('should error if `dataStream` is given an empty string', () => {
it('should error if `dataStream` is given type not array', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['storage_retained'],
dataStreams: ' ',
})
).toThrow('[dataStreams] must have at least one value');
).toThrow('[dataStreams]: could not parse array value from json input');
});
it('should error if `dataStream` is given an empty item in the list', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: ['storage_retained'],
dataStreams: ['ds_1', ' '],
})
).toThrow('[dataStreams] list can not contain empty values');
).toThrow('[dataStreams]: [dataStreams] list cannot contain empty values');
});
it('should error if `metricTypes` is empty string', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ' ',
})
).toThrow();
});
it('should error if `metricTypes` is empty item', () => {
it('should error if `metricTypes` contains an empty item', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
metricTypes: [' ', 'storage_retained'],
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: [' ', 'storage_retained'], // First item is invalid
})
).toThrow('[metricTypes] list can not contain empty values');
).toThrowError(/list cannot contain empty values/);
});
it('should error if `metricTypes` is not a valid value', () => {
it('should error if `metricTypes` is not a valid type', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: 'foo',
})
).toThrow(
'[metricTypes] must be one of storage_retained, ingest_rate, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate'
);
).toThrow('[metricTypes]: could not parse array value from json input');
});
it('should error if `metricTypes` is not a valid list', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: new Date().toISOString(),
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ['storage_retained', 'foo'],
})
).toThrow(
@ -139,9 +122,10 @@ describe('usage_metrics schemas', () => {
it('should error if `from` is not a valid input', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: 1010,
to: new Date().toISOString(),
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ['storage_retained', 'foo'],
})
).toThrow('[from]: expected value of type [string] but got [number]');
@ -149,9 +133,10 @@ describe('usage_metrics schemas', () => {
it('should error if `to` is not a valid input', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: 1010,
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ['storage_retained', 'foo'],
})
).toThrow('[to]: expected value of type [string] but got [number]');
@ -159,9 +144,10 @@ describe('usage_metrics schemas', () => {
it('should error if `from` is empty string', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: ' ',
to: new Date().toISOString(),
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ['storage_retained', 'foo'],
})
).toThrow('[from]: Date ISO string must not be empty');
@ -169,9 +155,10 @@ describe('usage_metrics schemas', () => {
it('should error if `to` is empty string', () => {
expect(() =>
UsageMetricsRequestSchema.query.validate({
UsageMetricsRequestSchema.validate({
from: new Date().toISOString(),
to: ' ',
dataStreams: ['data_stream_1', 'data_stream_2', 'data_stream_3'],
metricTypes: ['storage_retained', 'foo'],
})
).toThrow('[to]: Date ISO string must not be empty');

View file

@ -37,51 +37,31 @@ const metricTypesSchema = schema.oneOf(
// @ts-expect-error TS2769: No overload matches this call
METRIC_TYPE_VALUES.map((metricType) => schema.literal(metricType)) // Create a oneOf schema for the keys
);
export const UsageMetricsRequestSchema = {
query: schema.object({
from: DateSchema,
to: DateSchema,
metricTypes: schema.oneOf([
schema.arrayOf(schema.string(), {
minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[metricTypes] list can not contain empty values';
} else if (values.map((v) => v.trim()).some((v) => !isValidMetricType(v))) {
return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
}
},
}),
schema.string({
validate: (v) => {
if (!v.trim().length) {
return '[metricTypes] must have at least one value';
} else if (!isValidMetricType(v)) {
return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
}
},
}),
]),
dataStreams: schema.maybe(
schema.oneOf([
schema.arrayOf(schema.string(), {
minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[dataStreams] list can not contain empty values';
}
},
}),
schema.string({
validate: (v) =>
v.trim().length ? undefined : '[dataStreams] must have at least one value',
}),
])
),
export const UsageMetricsRequestSchema = schema.object({
from: DateSchema,
to: DateSchema,
metricTypes: schema.arrayOf(schema.string(), {
minSize: 1,
validate: (values) => {
const trimmedValues = values.map((v) => v.trim());
if (trimmedValues.some((v) => !v.length)) {
return '[metricTypes] list cannot contain empty values';
} else if (trimmedValues.some((v) => !isValidMetricType(v))) {
return `[metricTypes] must be one of ${METRIC_TYPE_VALUES.join(', ')}`;
}
},
}),
};
dataStreams: schema.arrayOf(schema.string(), {
minSize: 1,
validate: (values) => {
if (values.map((v) => v.trim()).some((v) => !v.length)) {
return '[dataStreams] list cannot contain empty values';
}
},
}),
});
export type UsageMetricsRequestSchemaQueryParams = TypeOf<typeof UsageMetricsRequestSchema.query>;
export type UsageMetricsRequestSchemaQueryParams = TypeOf<typeof UsageMetricsRequestSchema>;
export const UsageMetricsResponseSchema = {
body: () =>
@ -92,11 +72,40 @@ export const UsageMetricsResponseSchema = {
schema.object({
name: schema.string(),
data: schema.arrayOf(
schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 }) // Each data point is an array of 2 numbers
schema.object({
x: schema.number(),
y: schema.number(),
})
),
})
)
),
}),
};
export type UsageMetricsResponseSchemaBody = TypeOf<typeof UsageMetricsResponseSchema.body>;
export type UsageMetricsResponseSchemaBody = Omit<
TypeOf<typeof UsageMetricsResponseSchema.body>,
'metrics'
> & {
metrics: Partial<Record<MetricTypes, MetricSeries[]>>;
};
export type MetricSeries = TypeOf<
typeof UsageMetricsResponseSchema.body
>['metrics'][MetricTypes][number];
export const UsageMetricsAutoOpsResponseSchema = {
body: () =>
schema.object({
metrics: schema.recordOf(
metricTypesSchema,
schema.arrayOf(
schema.object({
name: schema.string(),
data: schema.arrayOf(schema.arrayOf(schema.number(), { minSize: 2, maxSize: 2 })),
})
)
),
}),
};
export type UsageMetricsAutoOpsResponseSchemaBody = TypeOf<
typeof UsageMetricsAutoOpsResponseSchema.body
>;

View file

@ -19,8 +19,7 @@ import {
} from '@elastic/charts';
import { i18n } from '@kbn/i18n';
import { LegendAction } from './legend_action';
import { MetricTypes } from '../../../common/rest_types';
import { MetricSeries } from '../types';
import { MetricTypes, MetricSeries } from '../../../common/rest_types';
// TODO: Remove this when we have a title for each metric type
type ChartKey = Extract<MetricTypes, 'ingest_rate' | 'storage_retained'>;
@ -50,7 +49,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
}) => {
const theme = useEuiTheme();
const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d[0]));
const chartTimestamps = series.flatMap((stream) => stream.data.map((d) => d.x));
const [minTimestamp, maxTimestamp] = [Math.min(...chartTimestamps), Math.max(...chartTimestamps)];
@ -72,6 +71,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
},
[idx, popoverOpen, togglePopover]
);
return (
<EuiFlexItem grow={false} key={metricType}>
<EuiPanel hasShadow={false} hasBorder={true}>
@ -94,9 +94,9 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
data={stream.data}
xScaleType={ScaleType.Time}
yScaleType={ScaleType.Linear}
xAccessor={0} // x is the first element in the tuple
yAccessors={[1]} // y is the second element in the tuple
stackAccessors={[0]}
xAccessor="x"
yAccessors={['y']}
stackAccessors={['x']}
/>
))}
@ -118,6 +118,7 @@ export const ChartPanel: React.FC<ChartPanelProps> = ({
</EuiFlexItem>
);
};
const formatBytes = (bytes: number) => {
return numeral(bytes).format('0.0 b');
};

View file

@ -6,11 +6,11 @@
*/
import React, { useCallback, useState } from 'react';
import { EuiFlexGroup } from '@elastic/eui';
import { MetricsResponse } from '../types';
import { MetricTypes } from '../../../common/rest_types';
import { ChartPanel } from './chart_panel';
import { UsageMetricsResponseSchemaBody } from '../../../common/rest_types';
interface ChartsProps {
data: MetricsResponse;
data: UsageMetricsResponseSchemaBody;
}
export const Charts: React.FC<ChartsProps> = ({ data }) => {

View file

@ -26,7 +26,6 @@ import { PLUGIN_NAME } from '../../common';
import { useGetDataUsageMetrics } from '../hooks/use_get_usage_metrics';
import { DEFAULT_DATE_RANGE_OPTIONS, useDateRangePicker } from './hooks/use_date_picker';
import { useDataUsageMetricsUrlParams } from './hooks/use_charts_url_params';
import { MetricsResponse } from './types';
export const DataUsage = () => {
const {
@ -42,37 +41,37 @@ export const DataUsage = () => {
setUrlDateRangeFilter,
} = useDataUsageMetricsUrlParams();
const [queryParams, setQueryParams] = useState<UsageMetricsRequestSchemaQueryParams>({
const [metricsFilters, setMetricsFilters] = useState<UsageMetricsRequestSchemaQueryParams>({
metricTypes: ['storage_retained', 'ingest_rate'],
dataStreams: [],
// TODO: Replace with data streams from /data_streams api
dataStreams: [
'.alerts-ml.anomaly-detection-health.alerts-default',
'.alerts-stack.alerts-default',
],
from: DEFAULT_DATE_RANGE_OPTIONS.startDate,
to: DEFAULT_DATE_RANGE_OPTIONS.endDate,
});
useEffect(() => {
if (!metricTypesFromUrl) {
setUrlMetricTypesFilter(
typeof queryParams.metricTypes !== 'string'
? queryParams.metricTypes.join(',')
: queryParams.metricTypes
);
setUrlMetricTypesFilter(metricsFilters.metricTypes.join(','));
}
if (!startDateFromUrl || !endDateFromUrl) {
setUrlDateRangeFilter({ startDate: queryParams.from, endDate: queryParams.to });
setUrlDateRangeFilter({ startDate: metricsFilters.from, endDate: metricsFilters.to });
}
}, [
endDateFromUrl,
metricTypesFromUrl,
queryParams.from,
queryParams.metricTypes,
queryParams.to,
metricsFilters.from,
metricsFilters.metricTypes,
metricsFilters.to,
setUrlDateRangeFilter,
setUrlMetricTypesFilter,
startDateFromUrl,
]);
useEffect(() => {
setQueryParams((prevState) => ({
setMetricsFilters((prevState) => ({
...prevState,
metricTypes: metricTypesFromUrl?.length ? metricTypesFromUrl : prevState.metricTypes,
dataStreams: dataStreamsFromUrl?.length ? dataStreamsFromUrl : prevState.dataStreams,
@ -89,7 +88,7 @@ export const DataUsage = () => {
refetch: refetchDataUsageMetrics,
} = useGetDataUsageMetrics(
{
...queryParams,
...metricsFilters,
from: dateRangePickerState.startDate,
to: dateRangePickerState.endDate,
},
@ -140,7 +139,7 @@ export const DataUsage = () => {
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size="l" />
{isFetched && data ? <Charts data={data as MetricsResponse} /> : <EuiLoadingElastic />}
{isFetched && data ? <Charts data={data} /> : <EuiLoadingElastic />}
</EuiPageSection>
</>
);

View file

@ -1,24 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MetricTypes } from '../../common/rest_types';
export type DataPoint = [number, number]; // [timestamp, value]
export interface MetricSeries {
name: string; // Name of the data stream
data: DataPoint[]; // Array of data points in tuple format [timestamp, value]
}
// Use MetricTypes dynamically as keys for the Metrics interface
export type Metrics = Partial<Record<MetricTypes, MetricSeries[]>>;
export interface MetricsResponse {
metrics: Metrics;
}
export interface MetricsResponse {
metrics: Metrics;
}

View file

@ -21,24 +21,24 @@ interface ErrorType {
}
export const useGetDataUsageMetrics = (
query: UsageMetricsRequestSchemaQueryParams,
body: UsageMetricsRequestSchemaQueryParams,
options: UseQueryOptions<UsageMetricsResponseSchemaBody, IHttpFetchError<ErrorType>> = {}
): UseQueryResult<UsageMetricsResponseSchemaBody, IHttpFetchError<ErrorType>> => {
const http = useKibanaContextForPlugin().services.http;
return useQuery<UsageMetricsResponseSchemaBody, IHttpFetchError<ErrorType>>({
queryKey: ['get-data-usage-metrics', query],
queryKey: ['get-data-usage-metrics', body],
...options,
keepPreviousData: true,
queryFn: async () => {
return http.get<UsageMetricsResponseSchemaBody>(DATA_USAGE_METRICS_API_ROUTE, {
return http.post<UsageMetricsResponseSchemaBody>(DATA_USAGE_METRICS_API_ROUTE, {
version: '1',
query: {
from: query.from,
to: query.to,
metricTypes: query.metricTypes,
dataStreams: query.dataStreams,
},
body: JSON.stringify({
from: body.from,
to: body.to,
metricTypes: body.metricTypes,
dataStreams: body.dataStreams,
}),
});
},
});

View file

@ -17,7 +17,7 @@ export const registerUsageMetricsRoute = (
) => {
if (dataUsageContext.serverConfig.enabled) {
router.versioned
.get({
.post({
access: 'internal',
path: DATA_USAGE_METRICS_API_ROUTE,
})
@ -25,7 +25,9 @@ export const registerUsageMetricsRoute = (
{
version: '1',
validate: {
request: UsageMetricsRequestSchema,
request: {
body: UsageMetricsRequestSchema,
},
response: {
200: UsageMetricsResponseSchema,
},

View file

@ -9,8 +9,10 @@ import { RequestHandler } from '@kbn/core/server';
import { IndicesGetDataStreamResponse } from '@elastic/elasticsearch/lib/api/types';
import {
MetricTypes,
UsageMetricsAutoOpsResponseSchema,
UsageMetricsAutoOpsResponseSchemaBody,
UsageMetricsRequestSchemaQueryParams,
UsageMetricsResponseSchema,
UsageMetricsResponseSchemaBody,
} from '../../../common/rest_types';
import { DataUsageContext, DataUsageRequestHandlerContext } from '../../types';
@ -34,45 +36,26 @@ export const getUsageMetricsHandler = (
const core = await context.core;
const esClient = core.elasticsearch.client.asCurrentUser;
// @ts-ignore
const { from, to, metricTypes, dataStreams: dsNames, size } = request.query;
const { from, to, metricTypes, dataStreams: requestDsNames } = request.query;
logger.debug(`Retrieving usage metrics`);
const { data_streams: dataStreamsResponse }: IndicesGetDataStreamResponse =
await esClient.indices.getDataStream({
name: '*',
name: requestDsNames,
expand_wildcards: 'all',
});
const hasDataStreams = dataStreamsResponse.length > 0;
let userDsNames: string[] = [];
if (dsNames?.length) {
userDsNames = typeof dsNames === 'string' ? [dsNames] : dsNames;
} else if (!userDsNames.length && hasDataStreams) {
userDsNames = dataStreamsResponse.map((ds) => ds.name);
}
// If no data streams are found, return an empty response
if (!userDsNames.length) {
return response.ok({
body: {
metrics: {},
},
});
}
const metrics = await fetchMetricsFromAutoOps({
from,
to,
metricTypes: formatStringParams(metricTypes) as MetricTypes[],
dataStreams: formatStringParams(userDsNames),
dataStreams: formatStringParams(dataStreamsResponse.map((ds) => ds.name)),
});
const processedMetrics = transformMetricsData(metrics);
return response.ok({
body: {
metrics,
},
body: processedMetrics,
});
} catch (error) {
logger.error(`Error retrieving usage metrics: ${error.message}`);
@ -94,7 +77,7 @@ const fetchMetricsFromAutoOps = async ({
}) => {
// TODO: fetch data from autoOps using userDsNames
/*
const response = await axios.post('https://api.auto-ops.{region}.{csp}.cloud.elastic.co/monitoring/serverless/v1/projects/{project_id}/metrics', {
const response = await axios.post({AUTOOPS_URL}, {
from: Date.parse(from),
to: Date.parse(to),
metric_types: metricTypes,
@ -231,7 +214,25 @@ const fetchMetricsFromAutoOps = async ({
},
};
// Make sure data is what we expect
const validatedData = UsageMetricsResponseSchema.body().validate(mockData);
const validatedData = UsageMetricsAutoOpsResponseSchema.body().validate(mockData);
return validatedData.metrics;
return validatedData;
};
function transformMetricsData(
data: UsageMetricsAutoOpsResponseSchemaBody
): UsageMetricsResponseSchemaBody {
return {
metrics: Object.fromEntries(
Object.entries(data.metrics).map(([metricType, series]) => [
metricType,
series.map((metricSeries) => ({
name: metricSeries.name,
data: (metricSeries.data as Array<[number, number]>).map(([timestamp, value]) => ({
x: timestamp,
y: value,
})),
})),
])
),
};
}