[Dataset quality] Filters for timeRange, integrations and datasetQuery (#176611)

Closes https://github.com/elastic/kibana/issues/170242

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Yngrid Coello 2024-02-19 16:16:47 +01:00 committed by GitHub
parent cd374d2336
commit c54dfc3622
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 784 additions and 107 deletions

1
.github/CODEOWNERS vendored
View file

@ -813,6 +813,7 @@ x-pack/examples/third_party_vis_lens_example @elastic/kibana-visualizations
x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations
x-pack/plugins/timelines @elastic/security-threat-hunting-investigations
packages/kbn-timelion-grammar @elastic/kibana-visualizations
packages/kbn-timerange @elastic/obs-ux-logs-team
packages/kbn-tinymath @elastic/kibana-visualizations
packages/kbn-tooling-log @elastic/kibana-operations
x-pack/plugins/transform @elastic/ml-ui

View file

@ -805,6 +805,7 @@
"@kbn/threat-intelligence-plugin": "link:x-pack/plugins/threat_intelligence",
"@kbn/timelines-plugin": "link:x-pack/plugins/timelines",
"@kbn/timelion-grammar": "link:packages/kbn-timelion-grammar",
"@kbn/timerange": "link:packages/kbn-timerange",
"@kbn/tinymath": "link:packages/kbn-tinymath",
"@kbn/transform-plugin": "link:x-pack/plugins/transform",
"@kbn/translations-plugin": "link:x-pack/plugins/translations",

View file

@ -0,0 +1,25 @@
# @kbn/timerange
This package shares a set of utilities for working with timeranges.
## Utils
### getDateRange
This function return a timestamp for `startDate` and `endDate` of a given date range.
```tsx
import { getDateRange } from '@kbn/timerange';
const { startDate, endDate } = getDateRange({ from: 'now-24h', to: 'now' });
```
### getDateISORange
This function return an ISO string for `startDate` and `endDate` of a given date range.
```tsx
import { getDateRange } from '@kbn/timerange';
const { startDate, endDate } = getDateISORange({ from: 'now-24h', to: 'now' });
```

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
export { getDateRange, getDateISORange } from './src';

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
module.exports = {
preset: '@kbn/test',
rootDir: '../..',
roots: ['<rootDir>/packages/kbn-timerange'],
};

View file

@ -0,0 +1,5 @@
{
"type": "shared-common",
"id": "@kbn/timerange",
"owner": "@elastic/obs-ux-logs-team"
}

View file

@ -0,0 +1,6 @@
{
"name": "@kbn/timerange",
"private": true,
"version": "1.0.0",
"license": "SSPL-1.0 OR Elastic License 2.0"
}

View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import datemath from '@kbn/datemath';
function getParsedDate(rawDate?: string, options = {}) {
if (rawDate) {
const parsed = datemath.parse(rawDate, options);
if (parsed && parsed.isValid()) {
return parsed.toDate();
}
}
}
function getRanges({ from, to }: { from: string; to: string }) {
const start = getParsedDate(from);
const end = getParsedDate(to, { roundUp: true });
if (!start || !end || start > end) {
throw new Error(`Invalid Dates: from: ${from}, to: ${to}`);
}
const startDate = start.toISOString();
const endDate = end.toISOString();
return {
startDate,
endDate,
};
}
export function getDateRange({ from, to }: { from: string; to: string }) {
const { startDate, endDate } = getRanges({ from, to });
return {
startDate: new Date(startDate).getTime(),
endDate: new Date(endDate).getTime(),
};
}
export function getDateISORange({ from, to }: { from: string; to: string }) {
const { startDate, endDate } = getRanges({ from, to });
return {
startDate,
endDate,
};
}

View file

@ -0,0 +1,21 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"types": [
"jest",
"node",
"react"
]
},
"include": [
"**/*.ts",
"**/*.tsx",
],
"exclude": [
"target/**/*"
],
"kbn_references": [
"@kbn/datemath",
]
}

View file

@ -1620,6 +1620,8 @@
"@kbn/timelines-plugin/*": ["x-pack/plugins/timelines/*"],
"@kbn/timelion-grammar": ["packages/kbn-timelion-grammar"],
"@kbn/timelion-grammar/*": ["packages/kbn-timelion-grammar/*"],
"@kbn/timerange": ["packages/kbn-timerange"],
"@kbn/timerange/*": ["packages/kbn-timerange/*"],
"@kbn/tinymath": ["packages/kbn-tinymath"],
"@kbn/tinymath/*": ["packages/kbn-tinymath/*"],
"@kbn/tooling-log": ["packages/kbn-tooling-log"],

View file

@ -10,5 +10,10 @@ export const DEFAULT_DATASET_TYPE = 'logs';
export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3;
export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0;
export const DEFAULT_SORT_FIELD = 'title';
export const DEFAULT_SORT_DIRECTION = 'asc';
export const NONE = 'none';
export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' };

View file

@ -7,13 +7,17 @@
import { APIClientRequestParamsOf, APIReturnType } from '../rest';
import { DataStreamStat } from './data_stream_stat';
import { Integration } from './integration';
export type GetDataStreamsStatsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/stats`>['params'];
export type GetDataStreamsStatsQuery = GetDataStreamsStatsParams['query'];
export type GetDataStreamsStatsResponse =
APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>;
export type DataStreamStatServiceResponse = DataStreamStat[];
export interface DataStreamStatServiceResponse {
dataStreamStats: DataStreamStat[];
integrations: Integration[];
}
export type IntegrationType = GetDataStreamsStatsResponse['integrations'][0];
export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0] & {
integration?: IntegrationType;

View file

@ -16,7 +16,7 @@ export const noDatasetsDescription = i18n.translate('xpack.datasetQuality.noData
});
export const noDatasetsTitle = i18n.translate('xpack.datasetQuality.noDatasetsTitle', {
defaultMessage: 'There is no data to display.',
defaultMessage: 'No matching data streams found',
});
export const loadingDatasetsText = i18n.translate('xpack.datasetQuality.loadingDatasetsText', {

View file

@ -8,6 +8,7 @@
import { EuiIcon } from '@elastic/eui';
import { PackageIcon } from '@kbn/fleet-plugin/public';
import React from 'react';
import { NONE } from '../../../common/constants';
import { Integration } from '../../../common/data_streams_stats/integration';
import loggingIcon from '../../icons/logging.svg';
@ -16,7 +17,7 @@ interface IntegrationIconProps {
}
export const IntegrationIcon = ({ integration }: IntegrationIconProps) => {
return integration ? (
return integration && integration.name !== NONE ? (
<PackageIcon
packageName={integration.name}
version={integration.version!}

View file

@ -56,18 +56,22 @@ export const createDatasetQuality = ({
};
const Header = dynamic(() => import('./header'));
const Table = dynamic(() => import('./table'));
const Table = dynamic(() => import('./table/table'));
const Filters = dynamic(() => import('./filters/filters'));
const SummaryPanel = dynamic(() => import('./summary_panel/summary_panel'));
function DatasetQuality() {
return (
<EuiFlexGroup direction="column" gutterSize="m">
<EuiFlexGroup direction="column" gutterSize="l">
<EuiFlexItem grow={false}>
<Header />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<SummaryPanel />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<Filters />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<Table />
</EuiFlexItem>

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { ChangeEvent, useCallback } from 'react';
import { EuiFieldSearch } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
const placeholder = i18n.translate('xpack.datasetQuality.filterBar.placeholder', {
defaultMessage: 'Filter datasets',
});
export interface FilterBarComponentProps {
query?: string;
onQueryChange: (query: string) => void;
}
export const FilterBar = ({ query, onQueryChange }: FilterBarComponentProps) => {
const onChange = useCallback(
(event: ChangeEvent<HTMLInputElement>) => {
onQueryChange(event.target.value);
},
[onQueryChange]
);
return (
<EuiFieldSearch
fullWidth
placeholder={placeholder}
value={query ?? ''}
onChange={onChange}
isClearable={true}
aria-label={placeholder}
/>
);
};

View file

@ -0,0 +1,78 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import { EuiSuperDatePicker } from '@elastic/eui';
import { UI_SETTINGS } from '@kbn/data-service';
import { TimePickerQuickRange } from '@kbn/observability-shared-plugin/public/hooks/use_quick_time_ranges';
import React, { useMemo } from 'react';
import { useDatasetQualityFilters } from '../../../hooks/use_dataset_quality_filters';
import { useKibanaContextForPlugin } from '../../../utils/use_kibana';
import { FilterBar } from './filter_bar';
import { IntegrationsSelector } from './integrations_selector';
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function Filters() {
const {
timeRange,
onTimeChange,
onRefresh,
onRefreshChange,
isLoading,
integrations,
onIntegrationsChange,
selectedQuery,
onQueryChange,
} = useDatasetQualityFilters();
const {
services: { uiSettings },
} = useKibanaContextForPlugin();
const timePickerQuickRanges = uiSettings.get<TimePickerQuickRange[]>(
UI_SETTINGS.TIMEPICKER_QUICK_RANGES
);
const commonlyUsedRanges = useMemo(
() =>
timePickerQuickRanges.map(({ from, to, display }) => ({
start: from,
end: to,
label: display,
})),
[timePickerQuickRanges]
);
return (
<EuiFlexGroup gutterSize="s">
<EuiFlexItem>
<FilterBar query={selectedQuery} onQueryChange={onQueryChange} />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<IntegrationsSelector
isLoading={isLoading}
integrations={integrations}
onIntegrationsChange={onIntegrationsChange}
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiSuperDatePicker
start={timeRange.from}
end={timeRange.to}
onTimeChange={onTimeChange}
onRefresh={onRefresh}
onRefreshChange={onRefreshChange}
commonlyUsedRanges={commonlyUsedRanges}
showUpdateButton={true}
isPaused={timeRange.refresh.isPaused}
refreshInterval={timeRange.refresh.interval}
/>
</EuiFlexItem>
</EuiFlexGroup>
);
}

View file

@ -0,0 +1,138 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
EuiFilterButton,
EuiFilterGroup,
EuiFlexGroup,
EuiFlexItem,
EuiPopover,
EuiPopoverTitle,
EuiSelectable,
EuiText,
} from '@elastic/eui';
import React, { useState } from 'react';
import type { EuiSelectableOptionCheckedType } from '@elastic/eui/src/components/selectable/selectable_option';
import { i18n } from '@kbn/i18n';
import { Integration } from '../../../../common/data_streams_stats/integration';
import { IntegrationIcon } from '../../common';
const integrationsSelectorLabel = i18n.translate('xpack.datasetQuality.integrationsSelectorLabel', {
defaultMessage: 'Integrations',
});
const integrationsSelectorLoading = i18n.translate(
'xpack.datasetQuality.integrationsSelectorLoading',
{
defaultMessage: 'Loading integrations',
}
);
const integrationsSelectorSearchPlaceholder = i18n.translate(
'xpack.datasetQuality.integrationsSelectorSearchPlaceholder',
{
defaultMessage: 'Filter integrations',
}
);
const integrationsSelectorNoneAvailable = i18n.translate(
'xpack.datasetQuality.integrationsSelectorNoneAvailable',
{
defaultMessage: 'No integrations available',
}
);
const integrationsSelectorNoneMatching = i18n.translate(
'xpack.datasetQuality.integrationsSelectorNoneMatching',
{
defaultMessage: 'No integrations found',
}
);
interface IntegrationsSelectorProps {
isLoading: boolean;
integrations: IntegrationItem[];
onIntegrationsChange: (integrations: IntegrationItem[]) => void;
}
export interface IntegrationItem extends Integration {
label: string;
checked?: EuiSelectableOptionCheckedType;
}
export function IntegrationsSelector({
isLoading,
integrations,
onIntegrationsChange,
}: IntegrationsSelectorProps) {
const [isPopoverOpen, setIsPopoverOpen] = useState(false);
const onButtonClick = () => {
setIsPopoverOpen(!isPopoverOpen);
};
const closePopover = () => {
setIsPopoverOpen(false);
};
const renderOption = (integration: IntegrationItem) => (
<EuiFlexGroup alignItems="center" gutterSize="s">
<EuiFlexItem grow={false}>
<IntegrationIcon integration={integration} />
</EuiFlexItem>
<EuiText size="s">{integration.title}</EuiText>
</EuiFlexGroup>
);
const button = (
<EuiFilterButton
iconType="arrowDown"
badgeColor="success"
onClick={onButtonClick}
isSelected={isPopoverOpen}
numFilters={integrations.length}
hasActiveFilters={!!integrations.find((item) => item.checked === 'on')}
numActiveFilters={integrations.filter((item) => item.checked === 'on').length}
>
{integrationsSelectorLabel}
</EuiFilterButton>
);
return (
<EuiFilterGroup>
<EuiPopover
button={button}
isOpen={isPopoverOpen}
closePopover={closePopover}
panelPaddingSize="none"
>
<EuiSelectable
searchable
searchProps={{
placeholder: integrationsSelectorSearchPlaceholder,
compressed: true,
}}
aria-label={integrationsSelectorLabel}
options={integrations}
onChange={onIntegrationsChange}
isLoading={isLoading}
loadingMessage={integrationsSelectorLoading}
emptyMessage={integrationsSelectorNoneAvailable}
noMatchesMessage={integrationsSelectorNoneMatching}
renderOption={(option) => renderOption(option)}
>
{(list, search) => (
<div style={{ width: 300 }}>
<EuiPopoverTitle paddingSize="s">{search}</EuiPopoverTitle>
{list}
</div>
)}
</EuiSelectable>
</EuiPopover>
</EuiFilterGroup>
);
}

View file

@ -27,12 +27,12 @@ import { css } from '@emotion/react';
import {
DEGRADED_QUALITY_MINIMUM_PERCENTAGE,
POOR_QUALITY_MINIMUM_PERCENTAGE,
} from '../../../common/constants';
import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat';
import { QualityIndicator, QualityPercentageIndicator } from '../quality_indicator';
import { IntegrationIcon } from '../common';
import { useLinkToLogsExplorer } from '../../hooks';
import { FlyoutDataset } from '../../state_machines/dataset_quality_controller';
} from '../../../../common/constants';
import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
import { QualityIndicator, QualityPercentageIndicator } from '../../quality_indicator';
import { IntegrationIcon } from '../../common';
import { useLinkToLogsExplorer } from '../../../hooks';
import { FlyoutDataset } from '../../../state_machines/dataset_quality_controller';
const expandDatasetAriaLabel = i18n.translate('xpack.datasetQuality.expandLabel', {
defaultMessage: 'Expand',
@ -189,11 +189,13 @@ export const getDatasetQualityTableColumns = ({
render: (_, dataStreamStat: DataStreamStat) => (
<EuiBadge color="hollow">{dataStreamStat.namespace}</EuiBadge>
),
width: '160px',
},
{
name: sizeColumnName,
field: 'size',
sortable: true,
width: '100px',
},
{
name: (
@ -219,6 +221,7 @@ export const getDatasetQualityTableColumns = ({
</EuiFlexGroup>
</EuiSkeletonRectangle>
),
width: '140px',
},
{
name: lastActivityColumnName,
@ -239,6 +242,7 @@ export const getDatasetQualityTableColumns = ({
.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
.convert(timestamp);
},
width: '300px',
sortable: true,
},
{

View file

@ -23,11 +23,11 @@ import {
inactiveDatasetsLabel,
loadingDatasetsText,
noDatasetsTitle,
} from '../../../common/translations';
import { useDatasetQualityTable } from '../../hooks';
import { DescriptiveSwitch } from '../common/descriptive_switch';
} from '../../../../common/translations';
import { useDatasetQualityTable } from '../../../hooks';
import { DescriptiveSwitch } from '../../common/descriptive_switch';
const Flyout = dynamic(() => import('../flyout/flyout'));
const Flyout = dynamic(() => import('../../flyout/flyout'));
export const Table = () => {
const {

View file

@ -26,7 +26,7 @@ export type DatasetQualityTableOptions = Partial<
export type DatasetQualityFlyoutOptions = Omit<WithFlyoutOptions['flyout'], 'datasetDetails'>;
export type DatasetQualityFilterOptions = Partial<Omit<WithFilters['filters'], 'timeRange'>>;
export type DatasetQualityFilterOptions = Partial<WithFilters['filters']>;
export interface DatasetQualityPublicState {
table: DatasetQualityTableOptions;

View file

@ -0,0 +1,108 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { OnRefreshChangeProps } from '@elastic/eui';
import { useSelector } from '@xstate/react';
import { useCallback, useMemo } from 'react';
import { useDatasetQualityContext } from '../components/dataset_quality/context';
import { IntegrationItem } from '../components/dataset_quality/filters/integrations_selector';
export const useDatasetQualityFilters = () => {
const { service } = useDatasetQualityContext();
const isLoading = useSelector(service, (state) => state.matches('datasets.fetching'));
const {
timeRange,
integrations: selectedIntegrations,
query: selectedQuery,
} = useSelector(service, (state) => state.context.filters);
const integrations = useSelector(service, (state) => state.context.integrations);
const onTimeChange = useCallback(
(selectedTime: { start: string; end: string; isInvalid: boolean }) => {
if (selectedTime.isInvalid) {
return;
}
service.send({
type: 'UPDATE_TIME_RANGE',
timeRange: {
...timeRange,
from: selectedTime.start,
to: selectedTime.end,
},
});
},
[service, timeRange]
);
const onRefresh = useCallback(() => {
service.send({
type: 'REFRESH_DATA',
});
}, [service]);
const onRefreshChange = useCallback(
({ refreshInterval, isPaused }: OnRefreshChangeProps) => {
service.send({
type: 'UPDATE_TIME_RANGE',
timeRange: {
...timeRange,
refresh: {
isPaused,
interval: refreshInterval,
},
},
});
},
[service, timeRange]
);
const integrationItems: IntegrationItem[] = useMemo(
() =>
integrations.map((integration) => ({
...integration,
label: integration.title,
checked: selectedIntegrations.includes(integration.name) ? 'on' : undefined,
})),
[integrations, selectedIntegrations]
);
const onIntegrationsChange = useCallback(
(newIntegrationItems: IntegrationItem[]) => {
service.send({
type: 'UPDATE_INTEGRATIONS',
integrations: newIntegrationItems
.filter((integration) => integration.checked === 'on')
.map((integration) => integration.name),
});
},
[service]
);
const onQueryChange = useCallback(
(query: string) => {
service.send({
type: 'UPDATE_QUERY',
query,
});
},
[service]
);
return {
timeRange,
onTimeChange,
onRefresh,
onRefreshChange,
integrations: integrationItems,
onIntegrationsChange,
isLoading,
selectedQuery,
onQueryChange,
};
};

View file

@ -8,10 +8,10 @@
import { useSelector } from '@xstate/react';
import { orderBy } from 'lodash';
import React, { useCallback, useMemo } from 'react';
import { DEFAULT_SORT_DIRECTION, DEFAULT_SORT_FIELD } from '../../common/constants';
import { DEFAULT_SORT_DIRECTION, DEFAULT_SORT_FIELD, NONE } from '../../common/constants';
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
import { tableSummaryAllText, tableSummaryOfText } from '../../common/translations';
import { getDatasetQualityTableColumns } from '../components/dataset_quality/columns';
import { getDatasetQualityTableColumns } from '../components/dataset_quality/table/columns';
import { useDatasetQualityContext } from '../components/dataset_quality/context';
import { FlyoutDataset } from '../state_machines/dataset_quality_controller';
import { useKibanaContextForPlugin } from '../utils';
@ -38,6 +38,8 @@ export const useDatasetQualityTable = () => {
inactive: showInactiveDatasets,
fullNames: showFullDatasetNames,
timeRange,
integrations,
query,
} = useSelector(service, (state) => state.context.filters);
const flyout = useSelector(service, (state) => state.context.flyout);
@ -115,10 +117,26 @@ export const useDatasetQualityTable = () => {
]
);
const filteredItems = useMemo(
() => (showInactiveDatasets ? datasets : filterInactiveDatasets({ datasets, timeRange })),
[showInactiveDatasets, datasets, timeRange]
);
const filteredItems = useMemo(() => {
const visibleDatasets = showInactiveDatasets
? datasets
: filterInactiveDatasets({ datasets, timeRange });
const filteredByIntegrations =
integrations.length > 0
? visibleDatasets.filter((dataset) => {
if (!dataset.integration && integrations.includes(NONE)) {
return true;
}
return dataset.integration && integrations.includes(dataset.integration.name);
})
: visibleDatasets;
return query
? filteredByIntegrations.filter((dataset) => dataset.rawName.includes(query))
: filteredByIntegrations;
}, [showInactiveDatasets, datasets, timeRange, integrations, query]);
const pagination = {
pageIndex: page,
@ -155,7 +173,7 @@ export const useDatasetQualityTable = () => {
}, [sort.field, sort.direction, filteredItems, page, rowsPerPage]);
const resultsCount = useMemo(() => {
const startNumberItemsOnPage = rowsPerPage * page + 1;
const startNumberItemsOnPage = rowsPerPage * page + (renderedItems.length ? 1 : 0);
const endNumberItemsOnPage = rowsPerPage * page + renderedItems.length;
return rowsPerPage === 0 ? (

View file

@ -10,7 +10,9 @@ import {
SingleDatasetLocatorParams,
} from '@kbn/deeplinks-observability';
import { getRouterLinkProps } from '@kbn/router-utils';
import { useSelector } from '@xstate/react';
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
import { useDatasetQualityContext } from '../components/dataset_quality/context';
import { FlyoutDataset } from '../state_machines/dataset_quality_controller';
import { useKibanaContextForPlugin } from '../utils';
@ -23,11 +25,16 @@ export const useLinkToLogsExplorer = ({
services: { share },
} = useKibanaContextForPlugin();
const { service } = useDatasetQualityContext();
const {
timeRange: { from, to },
} = useSelector(service, (state) => state.context.filters);
const params: SingleDatasetLocatorParams = {
dataset: dataStreamStat.name,
timeRange: {
from: 'now-1d',
to: 'now',
from,
to,
},
integration: dataStreamStat.integration?.name,
filterControls: {

View file

@ -8,13 +8,14 @@
import { HttpStart } from '@kbn/core/public';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import { find, merge } from 'lodash';
import { Integration } from '../../../common/data_streams_stats/integration';
import {
getDataStreamsDegradedDocsStatsResponseRt,
getDataStreamsStatsResponseRt,
getDataStreamsDetailsResponseRt,
getDataStreamsEstimatedDataInBytesResponseRt,
} from '../../../common/api_types';
import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
import { DEFAULT_DATASET_TYPE, NONE } from '../../../common/constants';
import {
DataStreamStatServiceResponse,
GetDataStreamsDegradedDocsStatsQuery,
@ -57,7 +58,15 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
return merge({}, statsItem, { integration });
});
return mergedDataStreamsStats.map(DataStreamStat.create);
const uncategorizedDatasets = dataStreamsStats.some((dataStream) => !dataStream.integration);
return {
dataStreamStats: mergedDataStreamsStats.map(DataStreamStat.create),
integrations: (uncategorizedDatasets
? [...integrations, { name: NONE, title: 'None' }]
: integrations
).map(Integration.create),
};
}
public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) {

View file

@ -5,11 +5,17 @@
* 2.0.
*/
import { getDefaultTimeRange } from '../../../utils';
import { DEFAULT_SORT_DIRECTION, DEFAULT_SORT_FIELD } from '../../../../common/constants';
import {
DEFAULT_DATASET_TYPE,
DEFAULT_SORT_DIRECTION,
DEFAULT_SORT_FIELD,
} from '../../../../common/constants';
import { DefaultDatasetQualityControllerState } from './types';
const ONE_MINUTE_IN_MS = 60000;
export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
type: DEFAULT_DATASET_TYPE,
table: {
page: 0,
rowsPerPage: 10,
@ -21,8 +27,17 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
filters: {
inactive: true,
fullNames: false,
timeRange: getDefaultTimeRange(),
timeRange: {
from: 'now-24h',
to: 'now',
refresh: {
isPaused: true,
interval: ONE_MINUTE_IN_MS,
},
},
integrations: [],
},
flyout: {},
datasets: [],
integrations: [],
};

View file

@ -6,27 +6,31 @@
*/
import { IToasts } from '@kbn/core/public';
import { getDateISORange } from '@kbn/timerange';
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
import { mergeDegradedStatsIntoDataStreams } from '../../../utils/merge_degraded_docs_into_datastreams';
import { DataStreamDetails } from '../../../../common/data_streams_stats';
import {
DataStreamDetails,
DataStreamStatServiceResponse,
GetDataStreamsStatsQuery,
} from '../../../../common/data_streams_stats';
import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat';
import { DataStreamType } from '../../../../common/types';
import { dataStreamPartsToIndexName } from '../../../../common/utils';
import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
import { IDataStreamsStatsClient } from '../../../services/data_streams_stats';
import { mergeDegradedStatsIntoDataStreams } from '../../../utils';
import { DEFAULT_CONTEXT } from './defaults';
import {
DatasetQualityControllerContext,
DatasetQualityControllerEvent,
DatasetQualityControllerTypeState,
FlyoutDataset,
} from './types';
import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat';
import {
fetchDatasetDetailsFailedNotifier,
fetchDatasetStatsFailedNotifier,
fetchDegradedStatsFailedNotifier,
noDatasetSelected,
} from './notifications';
import {
DatasetQualityControllerContext,
DatasetQualityControllerEvent,
DatasetQualityControllerTypeState,
FlyoutDataset,
} from './types';
export const createPureDatasetQualityControllerStateMachine = (
initialContext: DatasetQualityControllerContext
@ -76,6 +80,22 @@ export const createPureDatasetQualityControllerStateMachine = (
},
},
},
on: {
UPDATE_TIME_RANGE: {
target: 'datasets.fetching',
actions: ['storeTimeRange'],
},
REFRESH_DATA: {
target: 'datasets.fetching',
},
UPDATE_INTEGRATIONS: {
target: 'datasets.loaded',
actions: ['storeIntegrations'],
},
UPDATE_QUERY: {
actions: ['storeQuery'],
},
},
},
degradedDocs: {
initial: 'fetching',
@ -95,6 +115,15 @@ export const createPureDatasetQualityControllerStateMachine = (
},
loaded: {},
},
on: {
UPDATE_TIME_RANGE: {
target: 'degradedDocs.fetching',
actions: ['storeTimeRange'],
},
REFRESH_DATA: {
target: 'degradedDocs.fetching',
},
},
},
flyout: {
initial: 'closed',
@ -173,6 +202,36 @@ export const createPureDatasetQualityControllerStateMachine = (
},
};
}),
storeTimeRange: assign((context, event) => {
return 'timeRange' in event
? {
filters: {
...context.filters,
timeRange: event.timeRange,
},
}
: {};
}),
storeIntegrations: assign((context, event) => {
return 'integrations' in event
? {
filters: {
...context.filters,
integrations: event.integrations,
},
}
: {};
}),
storeQuery: assign((context, event) => {
return 'query' in event
? {
filters: {
...context.filters,
query: event.query,
},
}
: {};
}),
storeFlyoutOptions: assign((context, event) => {
return 'dataset' in event
? {
@ -187,7 +246,8 @@ export const createPureDatasetQualityControllerStateMachine = (
storeDataStreamStats: assign((_context, event) => {
return 'data' in event
? {
dataStreamStats: event.data as DataStreamStat[],
dataStreamStats: (event.data as DataStreamStatServiceResponse).dataStreamStats,
integrations: (event.data as DataStreamStatServiceResponse).integrations,
}
: {};
}),
@ -245,12 +305,21 @@ export const createDatasetQualityControllerStateMachine = ({
fetchDatasetDetailsFailedNotifier(toasts, event.data),
},
services: {
loadDataStreamStats: (_context) => dataStreamStatsClient.getDataStreamsStats(),
loadDegradedDocs: (context) =>
dataStreamStatsClient.getDataStreamsDegradedStats({
start: context.filters.timeRange.from,
end: context.filters.timeRange.to,
loadDataStreamStats: (context) =>
dataStreamStatsClient.getDataStreamsStats({
type: context.type as GetDataStreamsStatsQuery['type'],
datasetQuery: context.filters.query,
}),
loadDegradedDocs: (context) => {
const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange);
return dataStreamStatsClient.getDataStreamsDegradedStats({
type: context.type as GetDataStreamsStatsQuery['type'],
datasetQuery: context.filters.query,
start,
end,
});
},
loadDataStreamDetails: (context) => {
if (!context.flyout.dataset) {
fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected));

View file

@ -6,6 +6,7 @@
*/
import { DoneInvokeEvent } from 'xstate';
import { Integration } from '../../../../common/data_streams_stats/integration';
import { Direction, SortField } from '../../../hooks';
import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat';
import {
@ -29,13 +30,21 @@ interface TableCriteria {
};
}
export interface TimeRangeConfig {
from: string;
to: string;
refresh: {
isPaused: boolean;
interval: number;
};
}
interface FiltersCriteria {
inactive: boolean;
fullNames: boolean;
timeRange: {
from: string;
to: string;
};
timeRange: TimeRangeConfig;
integrations: string[];
query?: string;
}
export interface WithTableOptions {
@ -65,12 +74,17 @@ export interface WithDatasets {
datasets: DataStreamStat[];
}
export type DefaultDatasetQualityControllerState = WithTableOptions &
export interface WithIntegrations {
integrations: Integration[];
}
export type DefaultDatasetQualityControllerState = { type: string } & WithTableOptions &
Partial<WithDataStreamStats> &
Partial<WithDegradedDocs> &
WithFlyoutOptions &
WithDatasets &
WithFilters;
WithFilters &
WithIntegrations;
type DefaultDatasetQualityStateContext = DefaultDatasetQualityControllerState &
Partial<WithFlyoutOptions>;
@ -129,6 +143,21 @@ export type DatasetQualityControllerEvent =
| {
type: 'TOGGLE_FULL_DATASET_NAMES';
}
| {
type: 'UPDATE_TIME_RANGE';
timeRange: TimeRangeConfig;
}
| {
type: 'REFRESH_DATA';
}
| {
type: 'UPDATE_INTEGRATIONS';
integrations: string[];
}
| {
type: 'UPDATE_QUERY';
query: string;
}
| DoneInvokeEvent<DataStreamDegradedDocsStatServiceResponse>
| DoneInvokeEvent<DataStreamStatServiceResponse>
| DoneInvokeEvent<Error>;

View file

@ -6,11 +6,17 @@
*/
import { IToasts } from '@kbn/core/public';
import { getDateISORange } from '@kbn/timerange';
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
import { getDefaultTimeRange } from '../../../utils';
import { filterInactiveDatasets } from '../../../utils/filter_inactive_datasets';
import { DEFAULT_TIME_RANGE } from '../../../../common/constants';
import { IDataStreamsStatsClient } from '../../../services/data_streams_stats';
import { filterInactiveDatasets } from '../../../utils/filter_inactive_datasets';
import { defaultContext, MAX_RETRIES, RETRY_DELAY_IN_MS } from './defaults';
import {
fetchDatasetsActivityFailedNotifier,
fetchDatasetsEstimatedDataFailedNotifier,
fetchDatasetsQualityFailedNotifier,
} from './notifications';
import {
DatasetsActivityDetails,
DatasetsQuality,
@ -21,11 +27,6 @@ import {
EstimatedDataDetails,
Retries,
} from './types';
import {
fetchDatasetsEstimatedDataFailedNotifier,
fetchDatasetsActivityFailedNotifier,
fetchDatasetsQualityFailedNotifier,
} from './notifications';
export const createPureDatasetsSummaryPanelStateMachine = (
initialContext: DatasetsSummaryPanelContext
@ -212,20 +213,20 @@ export const createDatasetsSummaryPanelStateMachine = ({
return { percentages };
},
loadDatasetsActivity: async (_context) => {
const dataStreamsStats = await dataStreamStatsClient.getDataStreamsStats();
const activeDataStreams = filterInactiveDatasets({ datasets: dataStreamsStats });
const { dataStreamStats } = await dataStreamStatsClient.getDataStreamsStats();
const activeDataStreams = filterInactiveDatasets({ datasets: dataStreamStats });
return {
total: dataStreamsStats.length,
total: dataStreamStats.length,
active: activeDataStreams.length,
};
},
loadEstimatedData: async (_context) => {
const { from: start, to: end } = getDefaultTimeRange();
const { startDate, endDate } = getDateISORange(DEFAULT_TIME_RANGE);
return dataStreamStatsClient.getDataStreamsEstimatedDataInBytes({
query: {
type: 'logs',
start,
end,
start: startDate,
end: endDate,
},
});
},

View file

@ -8,6 +8,8 @@
import { ComponentType } from 'react';
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
import { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
import { DataPublicPluginStart } from '@kbn/data-plugin/public';
import { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
import { CreateDatasetQualityController } from './controller';
import { DatasetQualityProps } from './components/dataset_quality';
@ -20,8 +22,10 @@ export interface DatasetQualityPluginStart {
}
export interface DatasetQualityStartDeps {
data: DataPublicPluginStart;
share: SharePluginStart;
fieldFormats: FieldFormatsStart;
unifiedSearch: UnifiedSearchPublicPluginStart;
}
export interface DatasetQualitySetupDeps {

View file

@ -1,17 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
const ONE_DAY_IN_MILLISECONDS = 24 * 3600000;
export const getDefaultTimeRange = () => {
const now = Date.now();
return {
from: new Date(now - ONE_DAY_IN_MILLISECONDS).toISOString(),
to: new Date(now).toISOString(),
};
};

View file

@ -5,8 +5,9 @@
* 2.0.
*/
import { getDateRange } from '@kbn/timerange';
import { DEFAULT_TIME_RANGE } from '../../common/constants';
import { DataStreamStat } from '../../common/data_streams_stats';
import { getDefaultTimeRange } from './default_timerange';
interface FilterInactiveDatasetsOptions {
datasets: DataStreamStat[];
@ -18,15 +19,14 @@ interface FilterInactiveDatasetsOptions {
export const filterInactiveDatasets = ({
datasets,
timeRange = getDefaultTimeRange(),
timeRange = DEFAULT_TIME_RANGE,
}: FilterInactiveDatasetsOptions) => {
const { from, to } = timeRange;
const startDate = new Date(from).getTime();
const endDate = new Date(to).getTime();
const { startDate, endDate } = getDateRange(timeRange);
return datasets.filter((dataset) =>
dataset.lastActivity ? isActive(dataset.lastActivity, startDate, endDate) : false
dataset.lastActivity
? isActive(dataset.lastActivity, startDate as number, endDate as number)
: false
);
};
@ -39,13 +39,8 @@ interface IsActiveDatasetOptions {
}
export const isActiveDataset = (options: IsActiveDatasetOptions) => {
const {
lastActivity,
timeRange: { from, to },
} = options;
const startDate = new Date(from).getTime();
const endDate = new Date(to).getTime();
const { lastActivity, timeRange } = options;
const { startDate, endDate } = getDateRange(timeRange);
return isActive(lastActivity, startDate, endDate);
};

View file

@ -5,5 +5,6 @@
* 2.0.
*/
export * from './filter_inactive_datasets';
export * from './merge_degraded_docs_into_datastreams';
export * from './use_kibana';
export * from './default_timerange';

View file

@ -6,6 +6,7 @@
*/
import { PackageClient } from '@kbn/fleet-plugin/server';
import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors';
import { DataStreamStat, Integration } from '../../../common/api_types';
export async function getIntegrations(options: {
@ -39,15 +40,24 @@ const getDatasets = async (options: {
name: string;
version: string;
}) => {
const { packageClient, name, version } = options;
try {
const { packageClient, name, version } = options;
const pkg = await packageClient.getPackage(name, version);
const pkg = await packageClient.getPackage(name, version);
return pkg.packageInfo.data_streams?.reduce(
(acc, curr) => ({
...acc,
[curr.dataset]: curr.title,
}),
{}
);
return pkg.packageInfo.data_streams?.reduce(
(acc, curr) => ({
...acc,
[curr.dataset]: curr.title,
}),
{}
);
} catch (error) {
// Custom integration
if (error instanceof PackageNotFoundError) {
return {};
}
throw error;
}
};

View file

@ -31,7 +31,12 @@
"@kbn/shared-ux-utility",
"@kbn/ui-theme",
"@kbn/core-notifications-browser",
"@kbn/formatters"
"@kbn/formatters",
"@kbn/data-service",
"@kbn/observability-shared-plugin",
"@kbn/data-plugin",
"@kbn/unified-search-plugin",
"@kbn/timerange"
],
"exclude": [
"target/**/*",

View file

@ -56,6 +56,16 @@ export const filtersRT = rt.exact(
rt.partial({
inactive: rt.boolean,
fullNames: rt.boolean,
timeRange: rt.strict({
from: rt.string,
to: rt.string,
refresh: rt.strict({
isPaused: rt.boolean,
interval: rt.number,
}),
}),
integrations: rt.array(rt.string),
query: rt.string,
})
);

View file

@ -6309,6 +6309,10 @@
version "0.0.0"
uid ""
"@kbn/timerange@link:packages/kbn-timerange":
version "0.0.0"
uid ""
"@kbn/tinymath@link:packages/kbn-tinymath":
version "0.0.0"
uid ""