github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset Quality] Implement Dataset Quality Enhancements Telemetry (#188284)

Browse source 
closes https://github.com/elastic/logs-dev/issues/135

## 📝  Summary

This PR implements the last remaining Telemetry point in the above
ticket.

> Users using the `Chart Breakdown` feature including the chosen field

We are also now masking any non `ECS` field with a placeholder `<custom
field>` for PII restrictions.

## 🎥 Demo


https://github.com/user-attachments/assets/12e4a972-c9a0-4199-a198-062a0c666dc1

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
mohamedhamed-ahmed 2024-07-18 19:01:22 +03:00 • committed by GitHub
parent d7c22a0e58
commit f6914d8428
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 231 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/observability_solution/dataset_quality/common/constants.ts
View file

@ -38,3 +38,6 @@ export const BYTE_NUMBER_FORMAT = '0.0 b';
export const MAX_HOSTS_METRIC_VALUE = 50;
export const MAX_DEGRADED_FIELDS = 1000;
export const MASKED_FIELD_PLACEHOLDER = '<custom field>';
export const UNKOWN_FIELD_PLACEHOLDER = '<unkwon>';

3
x-pack/plugins/observability_solution/dataset_quality/kibana.jsonc
View file

@ -22,7 +22,8 @@
"fleet",
"fieldFormats",
"dataViews",
"lens"
"lens",
"fieldsMetadata"
],
"optionalPlugins": [],
"requiredBundles": ["unifiedHistogram", "discover"],

2
x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_docs_trend/degraded_docs.tsx
View file

@ -56,7 +56,7 @@ export function DegradedDocs({
if (breakdown.dataViewField && !breakdown.fieldSupportsBreakdown) {
// TODO: If needed, notify user that the field is not breakable
}
}, [setBreakdownDataViewField, breakdown.dataViewField, breakdown.fieldSupportsBreakdown]);
}, [setBreakdownDataViewField, breakdown]);
return (
<EuiPanel hasBorder grow={false}>

5
x-pack/plugins/observability_solution/dataset_quality/public/controller/create_controller.ts
View file

@ -16,6 +16,7 @@ import {
createDatasetQualityControllerStateMachine,
DEFAULT_CONTEXT,
} from '../state_machines/dataset_quality_controller';
import { DatasetQualityStartDeps } from '../types';
import { getContextFromPublicState, getPublicStateFromContext } from './public_state';
import { DatasetQualityController, DatasetQualityPublicStateUpdate } from './types';
@ -23,12 +24,13 @@ type InitialState = DatasetQualityPublicStateUpdate;
interface Dependencies {
core: CoreStart;
plugins: DatasetQualityStartDeps;
dataStreamStatsClient: IDataStreamsStatsClient;
dataStreamDetailsClient: IDataStreamDetailsClient;
}
export const createDatasetQualityControllerFactory =
({ core, dataStreamStatsClient, dataStreamDetailsClient }: Dependencies) =>
({ core, plugins, dataStreamStatsClient, dataStreamDetailsClient }: Dependencies) =>
async ({
initialState = DEFAULT_CONTEXT,
}: {
@ -38,6 +40,7 @@ export const createDatasetQualityControllerFactory =
const machine = createDatasetQualityControllerStateMachine({
initialContext,
plugins,
toasts: core.notifications.toasts,
dataStreamStatsClient,
dataStreamDetailsClient,

28
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.tsx
View file

@ -8,7 +8,7 @@
import { useCallback, useState, useMemo, useEffect } from 'react';
import { Action } from '@kbn/ui-actions-plugin/public';
import { fieldSupportsBreakdown } from '@kbn/unified-histogram-plugin/public';
import { useSelector } from '@xstate/react';
import { i18n } from '@kbn/i18n';
import { useEuiTheme } from '@elastic/eui';
import { type DataView, DataViewField } from '@kbn/data-views-plugin/common';
@ -53,11 +53,29 @@ export const useDegradedDocsChart = ({ dataStream }: DegradedDocsChartDeps) => {
services: { lens },
} = useKibanaContextForPlugin();
const { service } = useDatasetQualityContext();
const { trackDetailsNavigated, navigationTargets, navigationSources } =
useDatasetDetailsTelemetry();
const {
trackDatasetDetailsBreakdownFieldChanged,
trackDetailsNavigated,
navigationTargets,
navigationSources,
} = useDatasetDetailsTelemetry();
const { dataStreamStat, timeRange, breakdownField } = useDatasetQualityFlyout();
const isBreakdownFieldEcs = useSelector(
service,
(state) => state.context.flyout.isBreakdownFieldEcs
);
const isBreakdownFieldEcsAsserted = useSelector(service, (state) => {
return (
state.matches('flyout.initializing.assertBreakdownFieldIsEcs.done') &&
state.history?.matches('flyout.initializing.assertBreakdownFieldIsEcs.fetching') &&
isBreakdownFieldEcs !== null
);
});
const [isChartLoading, setIsChartLoading] = useState<boolean | undefined>(undefined);
const [attributes, setAttributes] = useState<ReturnType<typeof getLensAttributes> | undefined>(
undefined
@ -86,6 +104,10 @@ export const useDegradedDocsChart = ({ dataStream }: DegradedDocsChartDeps) => {
[service]
);
useEffect(() => {
if (isBreakdownFieldEcsAsserted) trackDatasetDetailsBreakdownFieldChanged();
}, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldEcsAsserted]);
useEffect(() => {
const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW;

33
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_telemetry.tsx
View file

@ -11,6 +11,7 @@ import { useSelector } from '@xstate/react';
import { getDateISORange } from '@kbn/timerange';
import { AggregateQuery, Query } from '@kbn/es-query';
import { MASKED_FIELD_PLACEHOLDER, UNKOWN_FIELD_PLACEHOLDER } from '../../common/constants';
import { DataStreamStat } from '../../common/data_streams_stats';
import { DataStreamDetails } from '../../common/api_types';
import { mapPercentageToQuality } from '../../common/utils';
@ -143,10 +144,13 @@ export const useDatasetDetailsTelemetry = () => {
insightsTimeRange,
breakdownField,
isNonAggregatable,
isBreakdownFieldEcs,
} = useSelector(service, (state) => state.context.flyout) ?? {};
const loadingState = useSelector(service, (state) => ({
dataStreamDetailsLoading: state.matches('flyout.initializing.dataStreamDetails.fetching'),
dataStreamDetailsLoading:
state.matches('flyout.initializing.dataStreamDetails.fetching') ||
state.matches('flyout.initializing.assertBreakdownFieldIsEcs.fetching'),
}));
const canUserAccessDashboards = useSelector(
@ -173,6 +177,7 @@ export const useDatasetDetailsTelemetry = () => {
isNonAggregatable ?? false,
canUserViewIntegrations,
canUserAccessDashboards,
isBreakdownFieldEcs,
breakdownField
);
}
@ -186,6 +191,7 @@ export const useDatasetDetailsTelemetry = () => {
isNonAggregatable,
canUserViewIntegrations,
canUserAccessDashboards,
isBreakdownFieldEcs,
breakdownField,
]);
@ -225,6 +231,19 @@ export const useDatasetDetailsTelemetry = () => {
[ebtProps, telemetryClient]
);
const trackDatasetDetailsBreakdownFieldChanged = useCallback(() => {
const datasetDetailsTrackingState = telemetryClient.getDatasetDetailsTrackingState();
if (
(datasetDetailsTrackingState === 'opened' || datasetDetailsTrackingState === 'navigated') &&
ebtProps
) {
telemetryClient.trackDatasetDetailsBreakdownFieldChanged({
...ebtProps,
breakdown_field: ebtProps.breakdown_field,
});
}
}, [ebtProps, telemetryClient]);
const wrapLinkPropsForTelemetry = useCallback(
(
props: RouterLinkProps,
@ -251,6 +270,7 @@ export const useDatasetDetailsTelemetry = () => {
wrapLinkPropsForTelemetry,
navigationTargets: NavigationTarget,
navigationSources: NavigationSource,
trackDatasetDetailsBreakdownFieldChanged,
};
};
@ -318,6 +338,7 @@ function getDatasetDetailsEbtProps(
isNonAggregatable: boolean,
canUserViewIntegrations: boolean,
canUserAccessDashboards: boolean,
isBreakdownFieldEcs: boolean | null,
breakdownField?: string
): DatasetDetailsEbtProps {
const indexName = flyoutDataset.rawName;
@ -347,6 +368,14 @@ function getDatasetDetailsEbtProps(
to,
degraded_percentage: degradedPercentage,
integration: flyoutDataset.integration?.name,
breakdown_field: breakdownField,
breakdown_field: breakdownField
? isBreakdownFieldEcs === null
? UNKOWN_FIELD_PLACEHOLDER
: getMaskedBreakdownField(breakdownField, isBreakdownFieldEcs)
: breakdownField,
};
}
function getMaskedBreakdownField(breakdownField: string, isBreakdownFieldEcs: boolean) {
return isBreakdownFieldEcs ? breakdownField : MASKED_FIELD_PLACEHOLDER;
}

1
x-pack/plugins/observability_solution/dataset_quality/public/plugin.tsx
View file

@ -49,6 +49,7 @@ export class DatasetQualityPlugin
const createDatasetQualityController = createDatasetQualityControllerLazyFactory({
core,
plugins,
dataStreamStatsClient,
dataStreamDetailsClient,
});

7
x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/telemetry_client.ts
View file

@ -55,4 +55,11 @@ export class TelemetryClient implements ITelemetryClient {
tracking_id: this.datasetDetailsTrackingId,
});
};
public trackDatasetDetailsBreakdownFieldChanged = (eventProps: DatasetDetailsEbtProps) => {
this.analytics.reportEvent(DatasetQualityTelemetryEventTypes.BREAKDOWN_FIELD_CHANGED, {
...eventProps,
tracking_id: this.datasetDetailsTrackingId,
});
};
}

21
x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/telemetry_events.ts
View file

@ -254,8 +254,29 @@ const datasetDetailsNavigatedEventType: DatasetQualityTelemetryEvent = {
},
};
const datasetDetailsBreakdownFieldChangedEventType: DatasetQualityTelemetryEvent = {
eventType: DatasetQualityTelemetryEventTypes.BREAKDOWN_FIELD_CHANGED,
schema: {
...datasetCommonSchema,
tracking_id: {
type: 'keyword',
_meta: {
description: `Locally generated session tracking ID for funnel analysis`,
},
},
breakdown_field: {
type: 'keyword',
_meta: {
description: 'Field used for chart breakdown, if any',
optional: true,
},
},
},
};
export const datasetQualityEbtEvents = {
datasetNavigatedEventType,
datasetDetailsOpenedEventType,
datasetDetailsNavigatedEventType,
datasetDetailsBreakdownFieldChangedEventType,
};

16
x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/telemetry_service.test.ts
View file

@ -174,4 +174,20 @@ describe('TelemetryService', () => {
'example_field'
);
});
it('should report dataset details breakdown field change event', async () => {
const telemetry = service.start();
const exampleEventData: DatasetDetailsEbtProps = {
...defaultEbtProps,
breakdown_field: 'service.name',
};
telemetry.trackDatasetDetailsBreakdownFieldChanged(exampleEventData);
expect(mockCoreStart.analytics.reportEvent).toHaveBeenCalledTimes(1);
expect(mockCoreStart.analytics.reportEvent).toHaveBeenCalledWith(
datasetQualityEbtEvents.datasetDetailsBreakdownFieldChangedEventType.eventType,
expect.objectContaining(exampleEventData)
);
});
});

3
x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/telemetry_service.ts
View file

@ -22,6 +22,9 @@ export class TelemetryService {
analytics.registerEventType(datasetQualityEbtEvents.datasetNavigatedEventType);
analytics.registerEventType(datasetQualityEbtEvents.datasetDetailsOpenedEventType);
analytics.registerEventType(datasetQualityEbtEvents.datasetDetailsNavigatedEventType);
analytics.registerEventType(
datasetQualityEbtEvents.datasetDetailsBreakdownFieldChangedEventType
);
}
public start(): ITelemetryClient {

6
x-pack/plugins/observability_solution/dataset_quality/public/services/telemetry/types.ts
View file

@ -101,12 +101,14 @@ export interface ITelemetryClient {
getDatasetDetailsTrackingState: () => DatasetDetailsTrackingState;
trackDatasetDetailsOpened: (eventProps: DatasetDetailsEbtProps) => void;
trackDatasetDetailsNavigated: (eventProps: DatasetDetailsNavigatedEbtProps) => void;
trackDatasetDetailsBreakdownFieldChanged: (eventProps: DatasetDetailsEbtProps) => void;
}
export enum DatasetQualityTelemetryEventTypes {
NAVIGATED = 'Dataset Quality Navigated',
DETAILS_OPENED = 'Dataset Quality Dataset Details Opened',
DETAILS_NAVIGATED = 'Dataset Quality Dataset Details Navigated',
BREAKDOWN_FIELD_CHANGED = 'Dataset Quality Dataset Details Breakdown Field Changed',
}
export type DatasetQualityTelemetryEvent =
@ -121,4 +123,8 @@ export type DatasetQualityTelemetryEvent =
| {
eventType: DatasetQualityTelemetryEventTypes.DETAILS_NAVIGATED;
schema: RootSchema<DatasetDetailsNavigatedEbtProps & WithTrackingId>;
}
| {
eventType: DatasetQualityTelemetryEventTypes.BREAKDOWN_FIELD_CHANGED;
schema: RootSchema<DatasetDetailsEbtProps & WithTrackingId>;
};

1
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts
View file

@ -58,6 +58,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
},
},
},
isBreakdownFieldEcs: null,
},
datasets: [],
isSizeStatsAvailable: true,

9
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts
View file

@ -93,3 +93,12 @@ export const noDatasetSelected = i18n.translate(
defaultMessage: 'No data set have been selected',
}
);
export const assertBreakdownFieldEcsFailedNotifier = (toasts: IToasts, error: Error) => {
toasts.addDanger({
title: i18n.translate('xpack.datasetQuality. assertBreakdownFieldEcsFailed', {
defaultMessage: "We couldn't retrieve breakdown field metadata.",
}),
text: error.message,
});
};

101
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts
View file

@ -8,6 +8,7 @@
import { IToasts } from '@kbn/core/public';
import { getDateISORange } from '@kbn/timerange';
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
import { DatasetQualityStartDeps } from '../../../types';
import { Dashboard, DataStreamStat, DegradedFieldResponse } from '../../../../common/api_types';
import { Integration } from '../../../../common/data_streams_stats/integration';
import { IDataStreamDetailsClient } from '../../../services/data_stream_details';
@ -18,6 +19,7 @@ import {
GetIntegrationsParams,
GetNonAggregatableDataStreamsParams,
GetNonAggregatableDataStreamsResponse,
DataStreamStatServiceResponse,
} from '../../../../common/data_streams_stats';
import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat';
import { DataStreamType } from '../../../../common/types';
@ -35,6 +37,7 @@ import {
noDatasetSelected,
fetchNonAggregatableDatasetsFailedNotifier,
fetchDataStreamIntegrationFailedNotifier,
assertBreakdownFieldEcsFailedNotifier,
} from './notifications';
import {
DatasetQualityControllerContext,
@ -354,6 +357,8 @@ export const createPureDatasetQualityControllerStateMachine = (
actions: ['storeFlyoutOptions'],
},
BREAKDOWN_FIELD_CHANGE: {
target:
'#DatasetQualityController.flyout.initializing.assertBreakdownFieldIsEcs.fetching',
actions: ['storeFlyoutOptions'],
},
},
@ -389,6 +394,25 @@ export const createPureDatasetQualityControllerStateMachine = (
},
},
},
assertBreakdownFieldIsEcs: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'assertBreakdownFieldIsEcs',
onDone: {
target: 'done',
actions: ['storeBreakdownFieldIsEcs'],
},
onError: {
target: 'done',
actions: ['notifyAssertBreakdownFieldEcsFailed'],
},
},
},
done: {},
},
},
},
onDone: {
target: '#DatasetQualityController.flyout.loaded',
@ -552,24 +576,35 @@ export const createPureDatasetQualityControllerStateMachine = (
},
};
}),
resetFlyoutOptions: assign((_context, _event) => ({ flyout: DEFAULT_CONTEXT.flyout })),
storeDataStreamStats: assign((_context, event) => {
if ('data' in event && 'dataStreamsStats' in event.data) {
const dataStreamStats = event.data.dataStreamsStats as DataStreamStat[];
const datasetUserPrivileges = event.data.datasetUserPrivileges;
// Check if any DataStreamStat has null; to check for serverless
const isSizeStatsAvailable =
!dataStreamStats.length || dataStreamStats.some((stat) => stat.totalDocs !== null);
return {
dataStreamStats,
isSizeStatsAvailable,
datasetUserPrivileges,
};
}
return {};
storeBreakdownFieldIsEcs: assign((context, event: DoneInvokeEvent<boolean | null>) => {
return {
flyout: {
...context.flyout,
isBreakdownFieldEcs:
'data' in event && typeof event.data === 'boolean' ? event.data : null,
},
};
}),
resetFlyoutOptions: assign((_context, _event) => ({ flyout: DEFAULT_CONTEXT.flyout })),
storeDataStreamStats: assign(
(_context, event: DoneInvokeEvent<DataStreamStatServiceResponse>) => {
if ('data' in event && 'dataStreamsStats' in event.data) {
const dataStreamStats = event.data.dataStreamsStats as DataStreamStat[];
const datasetUserPrivileges = event.data.datasetUserPrivileges;
// Check if any DataStreamStat has null; to check for serverless
const isSizeStatsAvailable =
!dataStreamStats.length || dataStreamStats.some((stat) => stat.totalDocs !== null);
return {
dataStreamStats,
isSizeStatsAvailable,
datasetUserPrivileges,
};
}
return {};
}
),
storeDegradedDocStats: assign((_context, event) => {
return 'data' in event
? {
@ -689,7 +724,12 @@ export const createPureDatasetQualityControllerStateMachine = (
},
guards: {
checkIfActionForbidden: (context, event) => {
return 'data' in event && 'statusCode' in event.data && event.data.statusCode === 403;
return (
'data' in event &&
typeof event.data === 'object' &&
'statusCode' in event.data! &&
event.data.statusCode === 403
);
},
},
}
@ -697,6 +737,7 @@ export const createPureDatasetQualityControllerStateMachine = (
export interface DatasetQualityControllerStateMachineDependencies {
initialContext?: DatasetQualityControllerContext;
plugins: DatasetQualityStartDeps;
toasts: IToasts;
dataStreamStatsClient: IDataStreamsStatsClient;
dataStreamDetailsClient: IDataStreamDetailsClient;
@ -704,6 +745,7 @@ export interface DatasetQualityControllerStateMachineDependencies {
export const createDatasetQualityControllerStateMachine = ({
initialContext = DEFAULT_CONTEXT,
plugins,
toasts,
dataStreamStatsClient,
dataStreamDetailsClient,
@ -728,6 +770,8 @@ export const createDatasetQualityControllerStateMachine = ({
const integrationName = context.flyout.datasetSettings?.integration;
return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName);
},
notifyAssertBreakdownFieldEcsFailed: (_context, event: DoneInvokeEvent<Error>) =>
assertBreakdownFieldEcsFailedNotifier(toasts, event.data),
},
services: {
loadDataStreamStats: (context) =>
@ -861,6 +905,27 @@ export const createDatasetQualityControllerStateMachine = ({
}),
});
},
assertBreakdownFieldIsEcs: async (context) => {
if (context.flyout.breakdownField) {
const allowedFieldSources = ['ecs', 'metadata'];
// This timeout is to avoid a runtime error that randomly happens on breakdown field change
// TypeError: Cannot read properties of undefined (reading 'timeFieldName')
await new Promise((res) => setTimeout(res, 300));
const client = await plugins.fieldsMetadata.getClient();
const { fields } = await client.find({
attributes: ['source'],
fieldNames: [context.flyout.breakdownField],
});
const breakdownFieldSource = fields[context.flyout.breakdownField]?.source;
return !!(breakdownFieldSource && allowedFieldSources.includes(breakdownFieldSource));
}
return null;
},
},
});

14
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts
View file

@ -76,6 +76,7 @@ export interface WithFlyoutOptions {
degradedFields: DegradedFields;
isNonAggregatable?: boolean;
integration?: DataStreamIntegrations;
isBreakdownFieldEcs: boolean | null;
};
}
@ -166,6 +167,18 @@ export type DatasetQualityControllerTypeState =
value: 'flyout.initializing.dataStreamDetails.fetching';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'flyout.initializing.dataStreamDetails.done';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'flyout.initializing.assertBreakdownFieldIsEcs.fetching';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'flyout.initializing.assertBreakdownFieldIsEcs.done';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'flyout.initializing.dataStreamDegradedFields.fetching';
context: DefaultDatasetQualityStateContext;
@ -244,4 +257,5 @@ export type DatasetQualityControllerEvent =
| DoneInvokeEvent<DataStreamSettings>
| DoneInvokeEvent<DataStreamStatServiceResponse>
| DoneInvokeEvent<Integration>
| DoneInvokeEvent<boolean | null>
| DoneInvokeEvent<Error>;

2
x-pack/plugins/observability_solution/dataset_quality/public/types.ts
View file

@ -13,6 +13,7 @@ import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/
import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
import type { LensPublicStart } from '@kbn/lens-plugin/public';
import type { ObservabilitySharedPluginSetup } from '@kbn/observability-shared-plugin/public';
import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
import type { CreateDatasetQualityController } from './controller';
import type { DatasetQualityProps } from './components/dataset_quality';
@ -33,6 +34,7 @@ export interface DatasetQualityStartDeps {
lens: LensPublicStart;
dataViews: DataViewsPublicPluginStart;
observabilityShared: ObservabilitySharedPluginSetup;
fieldsMetadata: FieldsMetadataPublicStart;
}
export interface DatasetQualitySetupDeps {

3
x-pack/plugins/observability_solution/dataset_quality/tsconfig.json
View file

@ -52,7 +52,8 @@
"@kbn/shared-ux-prompt-no-data-views-types",
"@kbn/core-analytics-server",
"@kbn/ebt",
"@kbn/ebt-tools"
"@kbn/ebt-tools",
"@kbn/fields-metadata-plugin"
],
"exclude": [
"target/**/*"