github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset Quality] Change logic to identify integrations (#198692)

Browse source 
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Felix Stürmer <felix.stuermer@elastic.co>
This commit is contained in:
Achyut Jhunjhunwala 2024-12-02 19:37:31 +01:00 • committed by GitHub
parent 281269f6a0
commit 86a044484d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 1021 additions and 322 deletions
Download patch file Download diff file Expand all files Collapse all files

5
x-pack/plugins/data_quality/public/routes/dataset_quality_details/context.tsx
View file

@ -74,7 +74,10 @@ export function DatasetQualityDetailsContextProvider({
urlStateStorageContainer,
datasetQualityDetailsState: state,
});
const breadcrumbValue = getBreadcrumbValue(state.dataStream, state.integration);
const breadcrumbValue = getBreadcrumbValue(
state.dataStream,
state.integration?.integration
);
setBreadcrumbs([{ text: breadcrumbValue }]);
}
);

1
x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
View file

@ -17,6 +17,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
fetchFindLatestPackage: jest.fn(),
readBundledPackage: jest.fn(),
getAgentPolicyConfigYAML: jest.fn(),
getLatestPackageInfo: jest.fn(),
getPackage: jest.fn(),
getPackageFieldsMetadata: jest.fn(),
getPackages: jest.fn(),

24
x-pack/plugins/fleet/server/services/epm/package_service.test.ts
View file

@ -43,6 +43,7 @@ const testKeys = [
'getInstallation',
'ensureInstalledPackage',
'fetchFindLatestPackage',
'getLatestPackageInfo',
'getPackage',
'getPackageFieldsMetadata',
'reinstallEsAssets',
@ -112,6 +113,23 @@ function getTest(
};
break;
case testKeys[3]:
test = {
method: mocks.packageClient.getLatestPackageInfo.bind(mocks.packageClient),
args: ['package name'],
spy: jest.spyOn(epmPackagesGet, 'getPackageInfo'),
spyArgs: [
{
pkgName: 'package name',
pkgVersion: '',
savedObjectsClient: mocks.soClient,
prerelease: undefined,
},
],
spyResponse: { name: 'getLatestPackageInfo test' },
expectedReturnValue: { name: 'getLatestPackageInfo test' },
};
break;
case testKeys[4]:
test = {
method: mocks.packageClient.getPackage.bind(mocks.packageClient),
args: ['package name', '8.0.0'],
@ -127,7 +145,7 @@ function getTest(
},
};
break;
case testKeys[4]:
case testKeys[5]:
test = {
method: mocks.packageClient.getPackageFieldsMetadata.bind(mocks.packageClient),
args: [{ packageName: 'package_name', datasetName: 'dataset_name' }],
@ -141,7 +159,7 @@ function getTest(
},
};
break;
case testKeys[5]:
case testKeys[6]:
const pkg: InstallablePackage = {
format_version: '1.0.0',
name: 'package name',
@ -187,7 +205,7 @@ function getTest(
],
};
break;
case testKeys[6]:
case testKeys[7]:
const bundledPackage = {
name: 'package name',
version: '8.0.0',

16
x-pack/plugins/fleet/server/services/epm/package_service.ts
View file

@ -56,6 +56,7 @@ import {
getPackages,
installPackage,
getTemplateInputs,
getPackageInfo,
} from './packages';
import { generatePackageInfoFromArchiveBuffer } from './archive';
import { getEsPackage } from './archive/storage';
@ -113,6 +114,11 @@ export interface PackageClient {
options?: Parameters<typeof getPackageFieldsMetadata>['1']
): ReturnType<typeof getPackageFieldsMetadata>;
getLatestPackageInfo(
packageName: string,
prerelease?: boolean
): ReturnType<typeof getPackageInfo>;
getPackages(params?: {
excludeInstallStatus?: false;
category?: CategoryId;
@ -328,6 +334,16 @@ class PackageClientImpl implements PackageClient {
return getPackageFieldsMetadata(params, options);
}
public async getLatestPackageInfo(packageName: string, prerelease?: boolean) {
await this.#runPreflight(READ_PACKAGE_INFO_AUTHZ);
return getPackageInfo({
savedObjectsClient: this.internalSoClient,
pkgName: packageName,
pkgVersion: '',
prerelease,
});
}
public async getPackages(params?: {
excludeInstallStatus?: false;
category?: CategoryId;

13
x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts
View file

@ -95,13 +95,24 @@ export const integrationRt = rt.intersection([
export type IntegrationType = rt.TypeOf<typeof integrationRt>;
export const checkAndLoadIntegrationResponseRt = rt.union([
rt.type({ isIntegration: rt.literal(false), areAssetsAvailable: rt.boolean }),
rt.type({
isIntegration: rt.literal(true),
areAssetsAvailable: rt.literal(true),
integration: integrationRt,
}),
]);
export type CheckAndLoadIntegrationResponse = rt.TypeOf<typeof checkAndLoadIntegrationResponseRt>;
export const getIntegrationsResponseRt = rt.exact(
rt.type({
integrations: rt.array(integrationRt),
})
);
export type IntegrationResponse = rt.TypeOf<typeof getIntegrationsResponseRt>;
export type IntegrationsResponse = rt.TypeOf<typeof getIntegrationsResponseRt>;
export const degradedFieldRt = rt.type({
name: rt.string,

14
x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/types.ts
View file

@ -5,9 +5,7 @@
* 2.0.
*/
export interface GetDataStreamIntegrationParams {
integrationName: string;
}
import { Integration } from '../data_streams_stats/integration';
export interface AnalyzeDegradedFieldsParams {
dataStream: string;
@ -19,3 +17,13 @@ export interface UpdateFieldLimitParams {
dataStream: string;
newFieldLimit: number;
}
export interface CheckAndLoadIntegrationParams {
dataStream: string;
}
export interface IntegrationType {
isIntegration: boolean;
areAssetsAvailable: boolean;
integration?: Integration;
}

8
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/field_mapping_limit.tsx
View file

@ -29,7 +29,11 @@ import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit';
import { FieldLimitDocLink } from './field_limit_documentation_link';
import { MessageCallout } from './message_callout';
export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean }) {
export function FieldMappingLimit({
areIntegrationAssetsAvailable,
}: {
areIntegrationAssetsAvailable: boolean;
}) {
const accordionId = useGeneratedHtmlId({
prefix: increaseFieldMappingLimitTitle,
});
@ -66,7 +70,7 @@ export function FieldMappingLimit({ isIntegration }: { isIntegration: boolean })
</ul>
</EuiText>
<EuiHorizontalRule margin="s" />
{isIntegration && (
{areIntegrationAssetsAvailable && (
<>
<IncreaseFieldMappingLimit
totalFieldLimit={degradedFieldAnalysis?.totalFieldLimit ?? 0}

2
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/field_limit/increase_field_mapping_limit.tsx
View file

@ -69,7 +69,7 @@ export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit
<EuiFlexItem grow={false}>
<EuiFormRow hasEmptyLabelSpace>
<EuiButton
data-test-subj="datasetQualityIncreaseFieldMappingLimitButtonButton"
data-test-subj="datasetQualityIncreaseFieldMappingLimitButton"
disabled={isInvalid}
onClick={() => updateNewFieldLimit(newFieldLimit)}
isLoading={isMitigationInProgress}

4
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/index.tsx
View file

@ -15,7 +15,7 @@ import { PossibleMitigationTitle } from './title';
export function PossibleMitigations() {
const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields();
const { integrationDetails } = useDatasetQualityDetailsState();
const isIntegration = Boolean(integrationDetails?.integration);
const areIntegrationAssetsAvailable = !!integrationDetails?.integration?.areAssetsAvailable;
return (
!isAnalysisInProgress && (
@ -24,7 +24,7 @@ export function PossibleMitigations() {
<EuiSpacer size="m" />
{degradedFieldAnalysis?.isFieldLimitIssue && (
<>
<FieldMappingLimit isIntegration={isIntegration} />
<FieldMappingLimit areIntegrationAssetsAvailable={areIntegrationAssetsAvailable} />
<EuiSpacer size="m" />
</>
)}

8
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/component_template_link.tsx
View file

@ -13,7 +13,11 @@ import { useDatasetQualityDetailsState } from '../../../../../hooks';
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name';
import { otherMitigationsCustomComponentTemplate } from '../../../../../../common/translations';
export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegration: boolean }) {
export function CreateEditComponentTemplateLink({
areIntegrationAssetsAvailable,
}: {
areIntegrationAssetsAvailable: boolean;
}) {
const {
services: {
application,
@ -54,7 +58,7 @@ export function CreateEditComponentTemplateLink({ isIntegration }: { isIntegrati
name,
]);
const templateUrl = isIntegration ? componentTemplatePath : indexTemplatePath;
const templateUrl = areIntegrationAssetsAvailable ? componentTemplatePath : indexTemplatePath;
const onClickHandler = useCallback(async () => {
const options = {

24
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/index.tsx
View file

@ -13,27 +13,27 @@ import { CreateEditPipelineLink } from './pipeline_link';
import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations';
export function ManualMitigations() {
const { integrationDetails, loadingState, dataStreamSettings } = useDatasetQualityDetailsState();
const isIntegrationPresentInSettings = dataStreamSettings?.integration;
const isIntegration = !!integrationDetails?.integration;
const { dataStreamSettingsLoading, integrationDetailsLoadings } = loadingState;
const hasIntegrationCheckCompleted =
!dataStreamSettingsLoading &&
((isIntegrationPresentInSettings && !integrationDetailsLoadings) ||
!isIntegrationPresentInSettings);
const {
integrationDetails,
loadingState: { integrationDetailsLoaded },
} = useDatasetQualityDetailsState();
const areIntegrationAssetsAvailable = !!integrationDetails?.integration?.areAssetsAvailable;
return (
<EuiSkeletonRectangle
isLoading={!hasIntegrationCheckCompleted}
isLoading={!integrationDetailsLoaded}
contentAriaLabel={otherMitigationsLoadingAriaText}
width="100%"
height={300}
borderRadius="none"
data-test-subj="datasetQualityDetailsFlyoutManualMitigationsLoading"
className="datasetQualityDetailsFlyoutManualMitigationsLoading"
>
<CreateEditComponentTemplateLink isIntegration={isIntegration} />
<CreateEditComponentTemplateLink
areIntegrationAssetsAvailable={areIntegrationAssetsAvailable}
/>
<EuiSpacer size="s" />
<CreateEditPipelineLink isIntegration={isIntegration} />
<CreateEditPipelineLink areIntegrationAssetsAvailable={areIntegrationAssetsAvailable} />
</EuiSkeletonRectangle>
);
}

13
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/degraded_field_flyout/possible_mitigations/manual/pipeline_link.tsx
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import React, { useCallback, useMemo } from 'react';
import React, { useCallback } from 'react';
import { FormattedMessage } from '@kbn/i18n-react';
import { i18n } from '@kbn/i18n';
import {
@ -34,7 +34,11 @@ const AccordionTitle = () => (
</EuiTitle>
);
export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boolean }) {
export function CreateEditPipelineLink({
areIntegrationAssetsAvailable,
}: {
areIntegrationAssetsAvailable: boolean;
}) {
const {
services: {
share: {
@ -50,10 +54,7 @@ export function CreateEditPipelineLink({ isIntegration }: { isIntegration: boole
const { datasetDetails } = useDatasetQualityDetailsState();
const { type, name } = datasetDetails;
const pipelineName = useMemo(
() => (isIntegration ? `${type}-${name}@custom` : `${type}@custom`),
[isIntegration, type, name]
);
const pipelineName = areIntegrationAssetsAvailable ? `${type}-${name}@custom` : `${type}@custom`;
const ingestPipelineLocator = locators.get('INGEST_PIPELINES_APP_LOCATOR');

32
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/details/dataset_summary.tsx
View file

@ -7,7 +7,7 @@
import React, { Fragment } from 'react';
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
import { EuiBadge, EuiFlexGroup, EuiPanel, EuiText } from '@elastic/eui';
import { EuiBadge, EuiFlexGroup, EuiPanel, EuiSkeletonRectangle, EuiText } from '@elastic/eui';
import { css } from '@emotion/react';
import { IntegrationActionsMenu } from './integration_actions_menu';
import {
@ -29,7 +29,8 @@ export function DatasetSummary() {
const {
dataStreamDetailsLoading,
dataStreamSettingsLoading,
integrationDetailsLoadings,
integrationDetailsLoading,
integrationDetailsLoaded,
integrationDashboardsLoading,
} = loadingState;
const formattedLastActivity = dataStreamDetails?.lastActivity
@ -39,12 +40,19 @@ export function DatasetSummary() {
? dataFormatter.convert(dataStreamSettings.createdOn)
: '-';
return (
return !integrationDetailsLoaded ? (
<EuiSkeletonRectangle
width="100%"
height="200px"
data-test-subj="datasetQualityDetailsDetailsSectionLoading"
className="datasetQualityDetailsDetailsSectionLoading"
/>
) : (
<EuiPanel hasBorder={false} hasShadow={false} paddingSize="none">
<Fragment>
<FieldsList
fields={[
...(integrationDetails?.integration
...(integrationDetails?.integration?.integration
? [
{
fieldTitle: integrationNameText,
@ -56,24 +64,28 @@ export function DatasetSummary() {
`}
>
<EuiFlexGroup gutterSize="xs" alignItems="center">
<IntegrationIcon integration={integrationDetails.integration} />
<EuiText size="s">{integrationDetails.integration?.name}</EuiText>
<IntegrationIcon
integration={integrationDetails.integration.integration}
/>
<EuiText size="s">
{integrationDetails.integration.integration?.name}
</EuiText>
</EuiFlexGroup>
</EuiBadge>
),
actionsMenu: (
<IntegrationActionsMenu
integration={integrationDetails.integration}
integration={integrationDetails.integration.integration}
dashboards={integrationDetails.dashboard}
dashboardsLoading={integrationDashboardsLoading}
/>
),
isLoading: integrationDetailsLoadings,
isLoading: integrationDetailsLoading,
},
{
fieldTitle: integrationVersionText,
fieldValue: integrationDetails.integration?.version,
isLoading: integrationDetailsLoadings,
fieldValue: integrationDetails.integration.integration?.version,
isLoading: integrationDetailsLoading,
},
]
: []),

5
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/header.tsx
View file

@ -44,7 +44,8 @@ export function Header() {
sendTelemetry,
});
const pageTitle = integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? title;
const pageTitle =
integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ?? title;
return !loadingState.integrationDetailsLoaded ? (
<EuiSkeletonTitle
@ -67,7 +68,7 @@ export function Header() {
border-radius: ${euiTheme.size.xxs};
`}
>
<IntegrationIcon integration={integrationDetails?.integration} />
<IntegrationIcon integration={integrationDetails?.integration?.integration} />
</div>
</EuiFlexGroup>
<p>

4
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_details_telemetry.ts
View file

@ -42,7 +42,7 @@ export function useDatasetDetailsTelemetry() {
breakdownField,
isNonAggregatable,
isBreakdownFieldEcs,
integration: integrationDetails.integration,
integration: integrationDetails.integration?.integration,
});
}
@ -57,7 +57,7 @@ export function useDatasetDetailsTelemetry() {
breakdownField,
isNonAggregatable,
isBreakdownFieldEcs,
integrationDetails.integration,
integrationDetails.integration?.integration,
]);
const startTracking = useCallback(() => {

35
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_details_state.ts
View file

@ -51,7 +51,9 @@ export const useDatasetQualityDetailsState = () => {
);
const dataStreamSettings = useSelector(service, (state) =>
state.matches('initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields')
state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields') ||
state.matches('initializing.dataStreamSettings.doneFetchingDegradedFields') ||
state.matches('initializing.dataStreamSettings.errorFetchingDegradedFields')
? state.context.dataStreamSettings
: undefined
);
@ -59,15 +61,17 @@ export const useDatasetQualityDetailsState = () => {
const integrationDetails = {
integration: useSelector(service, (state) =>
state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
)
'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards'
) ||
state.matches(
'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards'
) ||
state.matches('initializing.checkAndLoadIntegrationAndDashboards.done')
? state.context.integration
: undefined
),
dashboard: useSelector(service, (state) =>
state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'
)
state.matches('initializing.checkAndLoadIntegrationAndDashboards.done')
? state.context.integrationDashboards
: undefined
),
@ -77,7 +81,7 @@ export const useDatasetQualityDetailsState = () => {
service,
(state) =>
!state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'
'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards'
)
);
@ -106,14 +110,19 @@ export const useDatasetQualityDetailsState = () => {
dataStreamSettingsLoading: state.matches(
'initializing.dataStreamSettings.fetchingDataStreamSettings'
),
integrationDetailsLoadings: state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching'
),
integrationDetailsLoaded: state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
integrationDetailsLoading: state.matches(
'initializing.checkAndLoadIntegrationAndDashboards.checkingAndLoadingIntegration'
),
integrationDetailsLoaded:
state.matches(
'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards'
) ||
state.matches(
'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards'
) ||
state.matches('initializing.checkAndLoadIntegrationAndDashboards.done'),
integrationDashboardsLoading: state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching'
'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards'
),
}));

5
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_docs_chart.ts
View file

@ -81,7 +81,8 @@ export const useDegradedDocsChart = () => {
useEffect(() => {
const dataStreamName = dataStream ?? DEFAULT_LOGS_DATA_VIEW;
const datasetTitle =
integrationDetails?.integration?.datasets?.[datasetDetails.name] ?? datasetDetails.name;
integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ??
datasetDetails.name;
const lensAttributes = getLensAttributes({
color: euiTheme.colors.danger,
@ -95,7 +96,7 @@ export const useDegradedDocsChart = () => {
euiTheme.colors.danger,
setAttributes,
dataStream,
integrationDetails?.integration?.datasets,
integrationDetails?.integration?.integration?.datasets,
datasetDetails.name,
]);

9
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_degraded_fields.ts
View file

@ -77,10 +77,11 @@ export function useDegradedFields() {
return renderedItems.find((item) => item.name === expandedDegradedField);
}, [expandedDegradedField, renderedItems]);
const isDegradedFieldsLoading = useSelector(service, (state) =>
state.matches(
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'
)
const isDegradedFieldsLoading = useSelector(
service,
(state) =>
state.matches('initializing.dataStreamSettings.fetchingDataStreamSettings') ||
state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields')
);
const closeDegradedFieldFlyout = useCallback(

53
x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts
View file

@ -8,6 +8,8 @@
import { HttpStart } from '@kbn/core/public';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import {
CheckAndLoadIntegrationResponse,
checkAndLoadIntegrationResponseRt,
DataStreamRolloverResponse,
dataStreamRolloverResponseRt,
DegradedFieldAnalysis,
@ -17,10 +19,8 @@ import {
getDataStreamDegradedFieldsResponseRt,
getDataStreamsDetailsResponseRt,
getDataStreamsSettingsResponseRt,
getIntegrationsResponseRt,
IntegrationDashboardsResponse,
integrationDashboardsRT,
IntegrationResponse,
UpdateFieldLimitResponse,
updateFieldLimitResponseRt,
} from '../../../common/api_types';
@ -40,7 +40,7 @@ import { IDataStreamDetailsClient } from './types';
import { Integration } from '../../../common/data_streams_stats/integration';
import {
AnalyzeDegradedFieldsParams,
GetDataStreamIntegrationParams,
CheckAndLoadIntegrationParams,
UpdateFieldLimitParams,
} from '../../../common/data_stream_details/types';
import { DatasetQualityError } from '../../../common/errors';
@ -139,6 +139,32 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient {
)(response);
}
public async checkAndLoadIntegration({ dataStream }: CheckAndLoadIntegrationParams) {
const response = await this.http
.get<CheckAndLoadIntegrationResponse>(
`/internal/dataset_quality/data_streams/${dataStream}/integration/check`
)
.catch((error) => {
throw new DatasetQualityError(
`Failed to check if data stream belongs to an integration": ${error}`,
error
);
});
const decodedResponse = decodeOrThrow(
checkAndLoadIntegrationResponseRt,
(message: string) =>
new DatasetQualityError(`Failed to decode integration check response: ${message}"`)
)(response);
return {
...decodedResponse,
integration: decodedResponse.isIntegration
? Integration.create(decodedResponse.integration)
: undefined,
};
}
public async getIntegrationDashboards({ integration }: GetIntegrationDashboardsParams) {
const response = await this.http
.get<IntegrationDashboardsResponse>(
@ -157,27 +183,6 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient {
return dashboards;
}
public async getDataStreamIntegration(
params: GetDataStreamIntegrationParams
): Promise<Integration | undefined> {
const { integrationName } = params;
const response = await this.http
.get<IntegrationResponse>('/internal/dataset_quality/integrations')
.catch((error) => {
throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error);
});
const { integrations } = decodeOrThrow(
getIntegrationsResponseRt,
(message: string) =>
new DatasetQualityError(`Failed to decode integrations response: ${message}`)
)(response);
const integration = integrations.find((i) => i.name === integrationName);
if (integration) return Integration.create(integration);
}
public async analyzeDegradedField({
dataStream,
degradedField,

8
x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts
View file

@ -6,7 +6,6 @@
*/
import { HttpStart } from '@kbn/core/public';
import { Integration } from '../../../common/data_streams_stats/integration';
import {
GetDataStreamSettingsParams,
DataStreamSettings,
@ -19,7 +18,8 @@ import {
} from '../../../common/data_streams_stats';
import {
AnalyzeDegradedFieldsParams,
GetDataStreamIntegrationParams,
IntegrationType,
CheckAndLoadIntegrationParams,
UpdateFieldLimitParams,
} from '../../../common/data_stream_details/types';
import {
@ -49,10 +49,8 @@ export interface IDataStreamDetailsClient {
getDataStreamDegradedFieldValues(
params: GetDataStreamDegradedFieldValuesPathParams
): Promise<DegradedFieldValues>;
checkAndLoadIntegration(params: CheckAndLoadIntegrationParams): Promise<IntegrationType>;
getIntegrationDashboards(params: GetIntegrationDashboardsParams): Promise<Dashboard[]>;
getDataStreamIntegration(
params: GetDataStreamIntegrationParams
): Promise<Integration | undefined>;
analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise<DegradedFieldAnalysis>;
setNewFieldLimit(params: UpdateFieldLimitParams): Promise<UpdateFieldLimitResponse>;
rolloverDataStream(params: { dataStream: string }): Promise<DataStreamRolloverResponse>;

4
x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts
View file

@ -17,7 +17,7 @@ import {
getDataStreamTotalDocsResponseRt,
getIntegrationsResponseRt,
getNonAggregatableDatasetsRt,
IntegrationResponse,
IntegrationsResponse,
NonAggregatableDatasets,
} from '../../../common/api_types';
import {
@ -132,7 +132,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
public async getIntegrations(): Promise<Integration[]> {
const response = await this.http
.get<IntegrationResponse>('/internal/dataset_quality/integrations')
.get<IntegrationsResponse>('/internal/dataset_quality/integrations')
.catch((error) => {
throw new DatasetQualityError(`Failed to fetch integrations: ${error}`, error);
});

11
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/notifications.ts
View file

@ -44,17 +44,10 @@ export const fetchIntegrationDashboardsFailedNotifier = (toasts: IToasts, error:
});
};
export const fetchDataStreamIntegrationFailedNotifier = (
toasts: IToasts,
error: Error,
integrationName?: string
) => {
export const fetchDataStreamIntegrationFailedNotifier = (toasts: IToasts, error: Error) => {
toasts.addDanger({
title: i18n.translate('xpack.datasetQuality.details.fetchIntegrationsFailed', {
defaultMessage: "We couldn't get {integrationName} integration info.",
values: {
integrationName,
},
defaultMessage: "We couldn't get integration info.",
}),
text: error.message,
});

210
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/state_machine.ts
View file

@ -28,6 +28,8 @@ import {
UpdateFieldLimitResponse,
} from '../../../common/api_types';
import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications';
import { IntegrationType } from '../../../common/data_stream_details';
import {
fetchDataStreamDetailsFailedNotifier,
assertBreakdownFieldEcsFailedNotifier,
@ -37,7 +39,6 @@ import {
updateFieldLimitFailedNotifier,
rolloverDataStreamFailedNotifier,
} from './notifications';
import { Integration } from '../../../common/data_streams_stats/integration';
export const createPureDatasetQualityDetailsControllerStateMachine = (
initialContext: DatasetQualityDetailsControllerContext
@ -151,7 +152,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
invoke: {
src: 'loadDataStreamSettings',
onDone: {
target: 'loadingIntegrationsAndDegradedFields',
target: 'fetchingDataStreamDegradedFields',
actions: ['storeDataStreamSettings'],
},
onError: [
@ -160,111 +161,49 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
cond: 'isIndexNotFoundError',
},
{
target: 'done',
target: 'errorFetchingDataStreamSettings',
actions: ['notifyFetchDataStreamSettingsFailed'],
},
],
},
},
loadingIntegrationsAndDegradedFields: {
type: 'parallel',
states: {
dataStreamDegradedFields: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadDegradedFields',
onDone: {
target: 'done',
actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'],
},
onError: [
{
target: '#DatasetQualityDetailsController.indexNotFound',
cond: 'isIndexNotFoundError',
},
{
target: 'done',
},
],
},
},
done: {
on: {
UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: {
target: 'done',
actions: ['storeDegradedFieldTableOptions'],
},
OPEN_DEGRADED_FIELD_FLYOUT: {
target:
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
actions: [
'storeExpandedDegradedField',
'resetFieldLimitServerResponse',
],
},
TOGGLE_CURRENT_QUALITY_ISSUES: {
target: 'fetching',
actions: ['toggleCurrentQualityIssues'],
},
},
},
},
errorFetchingDataStreamSettings: {},
fetchingDataStreamDegradedFields: {
invoke: {
src: 'loadDegradedFields',
onDone: {
target: 'doneFetchingDegradedFields',
actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'],
},
integrationDetails: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadDataStreamIntegration',
onDone: {
target: 'done',
actions: ['storeDataStreamIntegration'],
},
onError: {
target: 'done',
actions: ['notifyFetchDatasetIntegrationsFailed'],
},
},
},
done: {},
onError: [
{
target: '#DatasetQualityDetailsController.indexNotFound',
cond: 'isIndexNotFoundError',
},
},
integrationDashboards: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadIntegrationDashboards',
onDone: {
target: 'done',
actions: ['storeIntegrationDashboards'],
},
onError: [
{
target: 'unauthorized',
cond: 'checkIfActionForbidden',
},
{
target: 'done',
actions: ['notifyFetchIntegrationDashboardsFailed'],
},
],
},
},
done: {},
unauthorized: {
type: 'final',
},
{
target: 'errorFetchingDegradedFields',
},
},
},
onDone: {
target: 'done',
],
},
},
done: {},
doneFetchingDegradedFields: {
on: {
UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: {
target: 'doneFetchingDegradedFields',
actions: ['storeDegradedFieldTableOptions'],
},
OPEN_DEGRADED_FIELD_FLYOUT: {
target:
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
actions: ['storeExpandedDegradedField', 'resetFieldLimitServerResponse'],
},
TOGGLE_CURRENT_QUALITY_ISSUES: {
target: 'fetchingDataStreamDegradedFields',
actions: ['toggleCurrentQualityIssues'],
},
},
},
errorFetchingDegradedFields: {},
},
on: {
UPDATE_TIME_RANGE: {
@ -272,6 +211,54 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
},
},
},
checkAndLoadIntegrationAndDashboards: {
initial: 'checkingAndLoadingIntegration',
states: {
checkingAndLoadingIntegration: {
invoke: {
src: 'checkAndLoadIntegration',
onDone: [
{
target: 'loadingIntegrationDashboards',
actions: 'storeDataStreamIntegration',
cond: 'isDataStreamIsPartOfIntegration',
},
{
actions: 'storeDataStreamIntegration',
target: 'done',
},
],
onError: {
target: 'done',
actions: ['notifyFetchDatasetIntegrationsFailed'],
},
},
},
loadingIntegrationDashboards: {
invoke: {
src: 'loadIntegrationDashboards',
onDone: {
target: 'done',
actions: ['storeIntegrationDashboards'],
},
onError: [
{
target: 'unauthorizedToLoadDashboards',
cond: 'checkIfActionForbidden',
},
{
target: 'done',
actions: ['notifyFetchIntegrationDashboardsFailed'],
},
],
},
},
unauthorizedToLoadDashboards: {
type: 'final',
},
done: {},
},
},
degradedFieldFlyout: {
initial: 'pending',
states: {
@ -522,7 +509,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
}
: {};
}),
storeDataStreamIntegration: assign((context, event: DoneInvokeEvent<Integration>) => {
storeDataStreamIntegration: assign((context, event: DoneInvokeEvent<IntegrationType>) => {
return 'data' in event
? {
integration: event.data,
@ -602,6 +589,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
!event.data.isLatestBackingIndexUpdated
);
},
isDataStreamIsPartOfIntegration: (_, event) => {
return (
'data' in event &&
typeof event.data === 'object' &&
'isIntegration' in event.data &&
event.data.isIntegration
);
},
},
}
);
@ -634,9 +629,7 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
notifyFetchIntegrationDashboardsFailed: (_context, event: DoneInvokeEvent<Error>) =>
fetchIntegrationDashboardsFailedNotifier(toasts, event.data),
notifyFetchDatasetIntegrationsFailed: (context, event: DoneInvokeEvent<Error>) => {
const integrationName =
'dataStreamSettings' in context ? context.dataStreamSettings?.integration : undefined;
return fetchDataStreamIntegrationFailedNotifier(toasts, event.data, integrationName);
return fetchDataStreamIntegrationFailedNotifier(toasts, event.data);
},
notifySaveNewFieldLimitError: (_context, event: DoneInvokeEvent<Error>) =>
updateFieldLimitFailedNotifier(toasts, event.data),
@ -735,18 +728,15 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
dataStream: context.dataStream,
});
},
loadDataStreamIntegration: (context) => {
if ('dataStreamSettings' in context && context.dataStreamSettings?.integration) {
return dataStreamDetailsClient.getDataStreamIntegration({
integrationName: context.dataStreamSettings.integration,
});
}
return Promise.resolve();
checkAndLoadIntegration: (context) => {
return dataStreamDetailsClient.checkAndLoadIntegration({
dataStream: context.dataStream,
});
},
loadIntegrationDashboards: (context) => {
if ('dataStreamSettings' in context && context.dataStreamSettings?.integration) {
if ('integration' in context && context.integration && context.integration.integration) {
return dataStreamDetailsClient.getIntegrationDashboards({
integration: context.dataStreamSettings.integration,
integration: context.integration.integration.name,
});
}

50
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_details_controller/types.ts
View file

@ -20,7 +20,7 @@ import {
UpdateFieldLimitResponse,
} from '../../../common/api_types';
import { TableCriteria, TimeRangeConfig } from '../../../common/types';
import { Integration } from '../../../common/data_streams_stats/integration';
import { IntegrationType } from '../../../common/data_stream_details';
export interface DataStream {
name: string;
@ -53,7 +53,7 @@ export interface WithDefaultControllerState {
breakdownField?: string;
isBreakdownFieldEcs?: boolean;
isIndexNotFoundError?: boolean;
integration?: Integration;
integration?: IntegrationType;
expandedDegradedField?: string;
isNonAggregatable?: boolean;
fieldLimit?: FieldLimit;
@ -84,8 +84,11 @@ export interface WithDataStreamSettings {
}
export interface WithIntegration {
integration: Integration;
integrationDashboards?: Dashboard[];
integration: IntegrationType;
}
export interface WithIntegrationDashboards {
integrationDashboards: Dashboard[];
}
export interface WithDegradedFieldValues {
@ -116,15 +119,14 @@ export type DatasetQualityDetailsControllerTypeState =
value:
| 'initializing'
| 'initializing.nonAggregatableDataset.fetching'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'
| 'initializing.dataStreamDetails.fetching'
| 'initializing.dataStreamSettings.fetchingDataStreamSettings'
| 'initializing.dataStreamDetails.fetching';
| 'initializing.dataStreamSettings.errorFetchingDataStreamSettings'
| 'initializing.checkAndLoadIntegrationAndDashboards.checkingAndLoadingIntegration';
context: WithDefaultControllerState;
}
| {
value:
| 'initializing.nonAggregatableDataset.done'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching';
value: 'initializing.nonAggregatableDataset.done';
context: WithDefaultControllerState & WithNonAggregatableDatasetStatus;
}
| {
@ -139,25 +141,25 @@ export type DatasetQualityDetailsControllerTypeState =
value: 'initializing.checkBreakdownFieldIsEcs.done';
context: WithDefaultControllerState & WithBreakdownInEcsCheck;
}
| {
value: 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.done';
context: WithDefaultControllerState &
WithNonAggregatableDatasetStatus &
WithDegradedFieldsData;
}
| {
value:
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized';
| 'initializing.dataStreamSettings.fetchingDataStreamDegradedFields'
| 'initializing.dataStreamSettings.errorFetchingDegradedFields';
context: WithDefaultControllerState & WithDataStreamSettings;
}
| {
value: 'initializing.dataStreamSettings.doneFetchingDegradedFields';
context: WithDefaultControllerState & WithDataStreamSettings & WithDegradedFieldsData;
}
| {
value:
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done';
context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration;
| 'initializing.checkAndLoadIntegrationAndDashboards.loadingIntegrationDashboards'
| 'initializing.checkAndLoadIntegrationAndDashboards.unauthorizedToLoadDashboards';
context: WithDefaultControllerState & WithIntegration;
}
| {
value: 'initializing.checkAndLoadIntegrationAndDashboards.done';
context: WithDefaultControllerState & WithIntegration & WithIntegrationDashboards;
}
| {
value: 'initializing.degradedFieldFlyout.open';
@ -233,8 +235,8 @@ export type DatasetQualityDetailsControllerEvent =
| DoneInvokeEvent<DegradedFieldResponse>
| DoneInvokeEvent<DegradedFieldValues>
| DoneInvokeEvent<DataStreamSettings>
| DoneInvokeEvent<Integration>
| DoneInvokeEvent<Dashboard[]>
| DoneInvokeEvent<DegradedFieldAnalysis>
| DoneInvokeEvent<UpdateFieldLimitResponse>
| DoneInvokeEvent<DataStreamRolloverResponse>;
| DoneInvokeEvent<DataStreamRolloverResponse>
| DoneInvokeEvent<IntegrationType>;

105
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/index.ts Normal file
View file

@ -0,0 +1,105 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { PackageClient } from '@kbn/fleet-plugin/server';
import { Logger } from '@kbn/logging';
import { validateCustomComponentTemplate } from './validate_custom_component_template';
import { getIntegration, getIntegrations } from '../../integrations/get_integrations';
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name';
import { CheckAndLoadIntegrationResponse } from '../../../../common/api_types';
import { dataStreamService } from '../../../services';
// The function works on 2 conditions:
// 1. It checks if integration name is present in meta field response of the datastream.
// If yes, it considers it to be an integration. No further checks
// 2. If not, then it does the various checks
export async function checkAndLoadIntegration({
esClient,
packageClient,
logger,
dataStream,
}: {
esClient: ElasticsearchClient;
packageClient: PackageClient;
logger: Logger;
dataStream: string;
}): Promise<CheckAndLoadIntegrationResponse> {
const [dataStreamInfo] = await dataStreamService.getMatchingDataStreams(esClient, dataStream);
const indexTemplate = dataStreamInfo?.template;
const isManaged = dataStreamInfo?._meta?.managed;
const integrationNameFromDataStream = dataStreamInfo?._meta?.package?.name;
// Index template must be present and isManaged should be true or
// integration name should be present
// Else it's not an integration
if ((!indexTemplate || !isManaged) && !integrationNameFromDataStream) {
return { isIntegration: false, areAssetsAvailable: false };
}
// If integration name is present, then we find and return the integration
if (integrationNameFromDataStream) {
try {
const integrationDetailsMatchingDataStream = await getIntegration({
packageClient,
logger,
packageName: integrationNameFromDataStream,
});
if (integrationDetailsMatchingDataStream) {
return {
isIntegration: true,
integration: integrationDetailsMatchingDataStream,
areAssetsAvailable: true,
};
}
} catch (e) {
// This should ideally not happen. As integration name is present in Data stream
// meta response but the integration itself is not found
// Worst case i could think of is, may be the integration is deleted from the
// system at a later point of time
return { isIntegration: false, areAssetsAvailable: false };
}
}
// cleaning the index template name as some have @template suffix
const indexTemplateNameWithoutSuffix = getComponentTemplatePrefixFromIndexTemplate(indexTemplate);
// Check if index template name has both type and dataset part
const isDedicatedComponentTemplate = indexTemplateNameWithoutSuffix.split('-').length === 2;
// If only 1 part exists, then it's not a dedicated index template
// Data stream name must starts with this index template, then it's a dedicated index template else not
if (!isDedicatedComponentTemplate || !dataStream.startsWith(indexTemplateNameWithoutSuffix)) {
return { isIntegration: false, areAssetsAvailable: false };
}
const isValidCustomComponentTemplate = await validateCustomComponentTemplate({
esClient,
indexTemplateName: indexTemplate,
});
if (!isValidCustomComponentTemplate) {
return { isIntegration: false, areAssetsAvailable: false };
}
const datasetName = indexTemplateNameWithoutSuffix.split('-')[1];
const allIntegrations = await getIntegrations({ packageClient, logger });
const integrationFromDataset = allIntegrations.find(
(integration) => datasetName in (integration?.datasets ?? {})
);
if (integrationFromDataset) {
return { isIntegration: true, integration: integrationFromDataset, areAssetsAvailable: true };
}
// Since the logic reached the last statement, it means it passed all checks for assets being available
return { isIntegration: false, areAssetsAvailable: true };
}

34
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/check_and_load_integration/validate_custom_component_template.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { createDatasetQualityESClient } from '../../../utils';
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../common/utils/component_template_name';
export async function validateCustomComponentTemplate({
esClient,
indexTemplateName,
}: {
esClient: ElasticsearchClient;
indexTemplateName: string;
}): Promise<boolean> {
const datasetQualityESClient = createDatasetQualityESClient(esClient);
// cleaning the index template name as some have @template suffix
const componentTemplateName = getComponentTemplatePrefixFromIndexTemplate(indexTemplateName);
try {
const { index_templates: indexTemplates } = await datasetQualityESClient.indexTemplates({
name: indexTemplateName,
});
return indexTemplates.some((template) =>
template.index_template.composed_of.includes(componentTemplateName + '@custom')
);
} catch (error) {
return false;
}
}

40
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts
View file

@ -16,38 +16,12 @@ import { rangeQuery } from '@kbn/observability-plugin/server';
import { MAX_HOSTS_METRIC_VALUE } from '../../../../common/constants';
import { _IGNORED } from '../../../../common/es_fields';
import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_types';
import { DataStreamDetails } from '../../../../common/api_types';
import { createDatasetQualityESClient } from '../../../utils';
import { dataStreamService, datasetQualityPrivileges } from '../../../services';
import { datasetQualityPrivileges } from '../../../services';
import { getDataStreams } from '../get_data_streams';
import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats';
export async function getDataStreamSettings({
esClient,
dataStream,
}: {
esClient: ElasticsearchClient;
dataStream: string;
}): Promise<DataStreamSettings> {
const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([
getDataStreamCreatedOn(esClient, dataStream),
dataStreamService.getMatchingDataStreams(esClient, dataStream),
datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream),
]);
const integration = dataStreamInfo?._meta?.package?.name;
const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0];
const indexTemplate = dataStreamInfo?.template;
return {
createdOn,
integration,
datasetUserPrivileges,
lastBackingIndexName: lastBackingIndex?.index_name,
indexTemplate,
};
}
export async function getDataStreamDetails({
esClient,
dataStream,
@ -118,16 +92,6 @@ export async function getDataStreamDetails({
}
}
async function getDataStreamCreatedOn(esClient: ElasticsearchClient, dataStream: string) {
const indexSettings = await dataStreamService.getDataStreamIndexSettings(esClient, dataStream);
const indexesList = Object.values(indexSettings);
return indexesList
.map((index) => Number(index.settings?.index?.creation_date))
.sort((a, b) => a - b)[0];
}
type TermAggregation = Record<string, { terms: { field: string; size: number } }>;
const MAX_HOSTS = MAX_HOSTS_METRIC_VALUE + 1; // Adding 1 so that we can show e.g. '50+'

19
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/get_datastream_created_on.ts Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { dataStreamService } from '../../../services';
export async function getDataStreamCreatedOn(esClient: ElasticsearchClient, dataStream: string) {
const indexSettings = await dataStreamService.getDataStreamIndexSettings(esClient, dataStream);
const indexesList = Object.values(indexSettings);
return indexesList
.map((index) => Number(index.settings?.index?.creation_date))
.sort((a, b) => a - b)[0];
}

37
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_datastream_settings/index.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { datasetQualityPrivileges, dataStreamService } from '../../../services';
import { DataStreamSettings } from '../../../../common/api_types';
import { getDataStreamCreatedOn } from './get_datastream_created_on';
export async function getDataStreamSettings({
esClient,
dataStream,
}: {
esClient: ElasticsearchClient;
dataStream: string;
}): Promise<DataStreamSettings> {
const [createdOn, [dataStreamInfo], datasetUserPrivileges] = await Promise.all([
getDataStreamCreatedOn(esClient, dataStream),
dataStreamService.getMatchingDataStreams(esClient, dataStream),
datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream),
]);
const integration = dataStreamInfo?._meta?.package?.name;
const lastBackingIndex = dataStreamInfo?.indices?.at(-1);
const indexTemplate = dataStreamInfo?.template;
return {
createdOn,
integration,
datasetUserPrivileges,
lastBackingIndexName: lastBackingIndex?.index_name,
indexTemplate,
};
}

38
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts
View file

@ -18,11 +18,12 @@ import {
DataStreamDocsStat,
UpdateFieldLimitResponse,
DataStreamRolloverResponse,
CheckAndLoadIntegrationResponse,
} from '../../../common/api_types';
import { rangeRt, typeRt, typesRt } from '../../types/default_api_types';
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
import { datasetQualityPrivileges } from '../../services';
import { getDataStreamDetails, getDataStreamSettings } from './get_data_stream_details';
import { getDataStreamDetails } from './get_data_stream_details';
import { getDataStreams } from './get_data_streams';
import { getDataStreamsStats } from './get_data_streams_stats';
import { getDegradedDocsPaginated } from './get_degraded_docs';
@ -34,6 +35,8 @@ import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats';
import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results';
import { updateFieldLimit } from './update_field_limit';
import { createDatasetQualityESClient } from '../../utils';
import { getDataStreamSettings } from './get_datastream_settings';
import { checkAndLoadIntegration } from './check_and_load_integration';
const statsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/stats',
@ -293,6 +296,38 @@ const dataStreamSettingsRoute = createDatasetQualityServerRoute({
},
});
const checkAndLoadIntegrationRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/integration/check',
params: t.type({
path: t.type({
dataStream: t.string,
}),
}),
options: {
tags: [],
},
async handler(resources): Promise<CheckAndLoadIntegrationResponse> {
const { context, params, plugins, logger } = resources;
const { dataStream } = params.path;
const coreContext = await context.core;
// Query dataStreams as the current user as the Kibana internal user may not have all the required permissions
const esClient = coreContext.elasticsearch.client.asCurrentUser;
const fleetPluginStart = await plugins.fleet.start();
const packageClient = fleetPluginStart.packageService.asInternalUser;
const integration = await checkAndLoadIntegration({
esClient,
packageClient,
logger,
dataStream,
});
return integration;
},
});
const dataStreamDetailsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/details',
params: t.type({
@ -418,6 +453,7 @@ export const dataStreamsRouteRepository = {
...degradedFieldValuesRoute,
...dataStreamDetailsRoute,
...dataStreamSettingsRoute,
...checkAndLoadIntegrationRoute,
...analyzeDegradedFieldRoute,
...updateFieldLimitRoute,
...rolloverDataStream,

2
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/update_field_limit/index.ts
View file

@ -11,7 +11,7 @@ import { createDatasetQualityESClient } from '../../../utils';
import { updateComponentTemplate } from './update_component_template';
import { updateLastBackingIndexSettings } from './update_settings_last_backing_index';
import { UpdateFieldLimitResponse } from '../../../../common/api_types';
import { getDataStreamSettings } from '../get_data_stream_details';
import { getDataStreamSettings } from '../get_datastream_settings';
export async function updateFieldLimit({
esClient,

24
x-pack/plugins/observability_solution/dataset_quality/server/routes/integrations/get_integrations.ts
View file

@ -8,9 +8,29 @@
import { Logger } from '@kbn/core/server';
import { PackageClient } from '@kbn/fleet-plugin/server';
import { PackageNotFoundError } from '@kbn/fleet-plugin/server/errors';
import { PackageListItem, RegistryDataStream } from '@kbn/fleet-plugin/common';
import { PackageInfo, RegistryDataStream } from '@kbn/fleet-plugin/common';
import { IntegrationType } from '../../../common/api_types';
export async function getIntegration({
packageClient,
logger,
packageName,
}: {
packageClient: PackageClient;
logger: Logger;
packageName: string;
}): Promise<IntegrationType> {
const latestPackage = await packageClient.getLatestPackageInfo(packageName);
return {
name: latestPackage.name,
title: latestPackage.title,
version: latestPackage.version,
icons: latestPackage.icons,
datasets: await getDatasets({ packageClient, logger, pkg: latestPackage }),
};
}
export async function getIntegrations(options: {
packageClient: PackageClient;
logger: Logger;
@ -36,7 +56,7 @@ export async function getIntegrations(options: {
const getDatasets = async (options: {
packageClient: PackageClient;
logger: Logger;
pkg: PackageListItem;
pkg: Pick<PackageInfo, 'name' | 'version' | 'data_streams'>;
}) => {
const { packageClient, logger, pkg } = options;

7
x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts
View file

@ -13,6 +13,8 @@ import {
FieldCapsRequest,
FieldCapsResponse,
Indices,
IndicesGetIndexTemplateRequest,
IndicesGetIndexTemplateResponse,
IndicesGetMappingResponse,
IndicesGetSettingsResponse,
IndicesPutSettingsRequest,
@ -63,5 +65,10 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) {
rollover(params: { alias: string }): Promise<IndicesRolloverResponse> {
return esClient.indices.rollover(params);
},
indexTemplates(
params: IndicesGetIndexTemplateRequest
): Promise<IndicesGetIndexTemplateResponse> {
return esClient.indices.getIndexTemplate(params);
},
};
}

3
x-pack/plugins/observability_solution/dataset_quality/tsconfig.json
View file

@ -61,7 +61,8 @@
"@kbn/rison",
"@kbn/task-manager-plugin",
"@kbn/core-application-browser",
"@kbn/field-utils"
"@kbn/field-utils",
"@kbn/logging"
],
"exclude": [
"target/**/*"

1
x-pack/plugins/translations/translations/fr-FR.json
View file

@ -15310,7 +15310,6 @@
"xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "Nous n'avons pas pu obtenir les détails de votre flux de données.",
"xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "Les paramètres du flux de données n'ont pas pu être chargés.",
"xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "Nous n'avons pas pu obtenir vos tableaux de bord d'intégration.",
"xpack.datasetQuality.details.fetchIntegrationsFailed": "Nous n'avons pas pu obtenir les informations relatives à l'intégration de {integrationName}.",
"xpack.datasetQuality.details.indexTemplateActionText": "Modèle d'index",
"xpack.datasetQuality.details.integrationActionsText": "Actions d'intégration",
"xpack.datasetQuality.details.integrationnameText": "Intégration",

1
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -15289,7 +15289,6 @@
"xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "データストリーム詳細を取得できませんでした。",
"xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "データストリーム設定を読み込めませんでした。",
"xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "統合ダッシュボードを取得できませんでした。",
"xpack.datasetQuality.details.fetchIntegrationsFailed": "{integrationName}統合情報を取得できませんでした。",
"xpack.datasetQuality.details.indexTemplateActionText": "インデックステンプレート",
"xpack.datasetQuality.details.integrationActionsText": "統合アクション",
"xpack.datasetQuality.details.integrationnameText": "統合",

1
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -14983,7 +14983,6 @@
"xpack.datasetQuality.details.fetchDataStreamDetailsFailed": "无法获取数据流详情。",
"xpack.datasetQuality.details.fetchDataStreamSettingsFailed": "无法加载数据流设置。",
"xpack.datasetQuality.details.fetchIntegrationDashboardsFailed": "无法获取集成仪表板。",
"xpack.datasetQuality.details.fetchIntegrationsFailed": "无法获取 {integrationName} 集成信息。",
"xpack.datasetQuality.details.indexTemplateActionText": "索引模板",
"xpack.datasetQuality.details.integrationActionsText": "集成操作",
"xpack.datasetQuality.details.integrationnameText": "集成",

175
x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/check_and_load_integration.ts Normal file
View file

@ -0,0 +1,175 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services';
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
const samlAuth = getService('samlAuth');
const roleScopedSupertest = getService('roleScopedSupertest');
const synthtrace = getService('synthtrace');
const packageApi = getService('packageApi');
const start = '2024-11-04T11:00:00.000Z';
const end = '2024-11-04T11:01:00.000Z';
const type = 'logs';
const dataset = 'synth';
const nginxDataset = 'nginx.access';
const apmDataset = 'apm.app.test';
const namespace = 'default';
const serviceName = 'my-service';
const hostName = 'synth-host';
const dataStreamName = `${type}-${dataset}-${namespace}`;
const nginxDataStreamName = `${type}-${nginxDataset}-${namespace}`;
const apmAppDataStreamName = `${type}-${apmDataset}-${namespace}`;
const pkg = 'nginx';
async function callApiAs({
roleScopedSupertestWithCookieCredentials,
apiParams: { dataStream },
}: {
roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType;
apiParams: {
dataStream: string;
};
}) {
return roleScopedSupertestWithCookieCredentials.get(
`/internal/dataset_quality/data_streams/${dataStream}/integration/check`
);
}
describe('Check and load integrations', function () {
let adminRoleAuthc: RoleCredentials;
let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType;
let synthtraceLogsEsClient: LogsSynthtraceEsClient;
before(async () => {
synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient();
adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin');
supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
'admin',
{
useCookieHeader: true,
withInternalHeaders: true,
}
);
// Install nginx package
await packageApi.installPackage({
roleAuthc: adminRoleAuthc,
pkg,
});
await synthtraceLogsEsClient.index([
// Ingest degraded data in Nginx data stream
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset(nginxDataset)
.namespace(namespace)
.defaults({
'log.file.path': '/my-service.log',
'service.name': serviceName,
'host.name': hostName,
})
),
// ingest data in apm app data stream
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset(apmDataset)
.namespace(namespace)
.defaults({
'log.file.path': '/my-service.log',
'service.name': serviceName,
'host.name': hostName,
})
),
// Ingest data in regular datastream which is not an integration
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset(dataset)
.namespace(namespace)
.defaults({
'log.file.path': '/my-service.log',
'service.name': serviceName,
'host.name': hostName,
})
),
]);
});
after(async () => {
await synthtraceLogsEsClient.clean();
await packageApi.uninstallPackage({
roleAuthc: adminRoleAuthc,
pkg,
});
await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc);
});
describe('returns integration status', () => {
it('returns integration as false for regular data stream', async () => {
const resp = await callApiAs({
roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials,
apiParams: {
dataStream: dataStreamName,
},
});
expect(resp.body.isIntegration).to.be(false);
expect(resp.body.areAssetsAvailable).to.be(false);
});
it('returns integration as true for nginx data stream as we installed the integration', async () => {
const resp = await callApiAs({
roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials,
apiParams: {
dataStream: nginxDataStreamName,
},
});
expect(resp.body.isIntegration).to.be(true);
expect(resp.body.areAssetsAvailable).to.be(true);
expect(resp.body.integration.name).to.be(pkg);
expect(resp.body.integration.datasets[nginxDataset]).to.be.a('string');
});
it('returns integration as false but assets are available for apm.app data stream as its preinstalled', async () => {
const resp = await callApiAs({
roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials,
apiParams: {
dataStream: apmAppDataStreamName,
},
});
expect(resp.body.isIntegration).to.be(false);
expect(resp.body.areAssetsAvailable).to.be(true);
});
});
});
}

1
x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts
View file

@ -14,6 +14,7 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
loadTestFile(require.resolve('./data_stream_settings'));
loadTestFile(require.resolve('./data_stream_rollover'));
loadTestFile(require.resolve('./update_field_limit'));
loadTestFile(require.resolve('./check_and_load_integration'));
loadTestFile(require.resolve('./data_stream_total_docs'));
loadTestFile(require.resolve('./degraded_docs'));
loadTestFile(require.resolve('./degraded_fields'));

13
x-pack/test/functional/apps/dataset_quality/dataset_quality_details.ts
View file

@ -250,7 +250,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
);
});
it('Should navigate to integration overview page on clicking integration overview action', async () => {
it('should navigate to integration overview page on clicking integration overview action', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: bitbucketAuditDataStreamName,
});
@ -359,7 +359,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
);
});
it('should show the degraded fields table with data when present', async () => {
it('should show the degraded fields table with data and spark plots when present', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
@ -372,15 +372,6 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
expect(rows.length).to.eql(3);
});
it('should display Spark Plot for every row of degraded fields', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
const rows =
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
const sparkPlots = await testSubjects.findAll(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot

134
x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts
View file

@ -42,13 +42,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
const customComponentTemplateName = 'logs-synth@mappings';
const nginxAccessDatasetName = 'nginx.access';
const customComponentTemplateNameNginx = 'logs-nginx.access@custom';
const customComponentTemplateNameNginx = `logs-${nginxAccessDatasetName}@custom`;
const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`;
const nginxPkg = {
name: 'nginx',
version: '1.23.0',
};
const apmAppDatasetName = 'apm.app.tug';
const apmAppDataStreamName = `${type}-${apmAppDatasetName}-${defaultNamespace}`;
describe('Degraded fields flyout', () => {
describe('degraded field flyout open-close', () => {
before(async () => {
@ -131,7 +134,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
// Install Nginx Integration and ingest logs for it
await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg);
// Create custom component template to avoid issues with LogsDB
// Create custom component template for Nginx to avoid issues with LogsDB
await synthtrace.createComponentTemplate(
customComponentTemplateNameNginx,
logsNginxMappings(nginxAccessDatasetName)
@ -184,6 +187,29 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 26 fields in Apm DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
})
.timestamp(timestamp)
);
}),
]);
// Set Limit of 25
@ -199,6 +225,11 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
'mapping.total_fields.limit': 42,
});
// Set Limit of 26
await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, {
'mapping.total_fields.limit': 25,
});
await synthtrace.index([
// Ingest Degraded Logs with 26 field
timerange(moment(to).subtract(count, 'minute'), moment(to))
@ -248,11 +279,36 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
'cloud.project.id': generateShortId(),
})
.timestamp(timestamp)
);
}),
]);
// Rollover Datastream to reset the limit to default which is 1000
await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName);
await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName);
await PageObjects.datasetQuality.rolloverDataStream(apmAppDataStreamName);
// Set Limit of 26
await PageObjects.datasetQuality.setDataStreamSettings(
@ -274,6 +330,16 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
}
);
// Set Limit of 27
await PageObjects.datasetQuality.setDataStreamSettings(
PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({
dataset: apmAppDatasetName,
}) + '-000002',
{
'mapping.total_fields.limit': 27,
}
);
await synthtrace.index([
// Ingest Degraded Logs with 26 field
timerange(moment(to).subtract(count, 'minute'), moment(to))
@ -323,6 +389,30 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
'cloud.project.id': generateShortId(),
})
.timestamp(timestamp)
);
}),
]);
});
@ -568,6 +658,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
expandedDegradedField: 'test_field',
});
await PageObjects.datasetQuality.waitUntilPossibleMitigationsLoaded();
// Possible Mitigation Section should exist
await testSubjects.existOrFail(
'datasetQualityDetailsDegradedFieldFlyoutPossibleMitigationTitle'
@ -703,6 +795,36 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true);
});
it('should display increase field limit as a possible mitigation for special packages like apm app', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: apmAppDataStreamName,
expandedDegradedField: 'cloud.project',
});
// Field Limit Mitigation Section should exist
await testSubjects.existOrFail(
'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion'
);
// Should display the panel to increase field limit
await testSubjects.existOrFail(
'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel'
);
// Should display official online documentation link
await testSubjects.existOrFail(
'datasetQualityManualMitigationsPipelineOfficialDocumentationLink'
);
const linkButton = await testSubjects.find(
'datasetQualityManualMitigationsPipelineOfficialDocumentationLink'
);
const linkURL = await linkButton.getAttribute('href');
expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true);
});
it('should display increase field limit as a possible mitigation for non integration', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDatasetWithLimitDataStreamName,
@ -764,10 +886,10 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
expect(newFieldLimit).to.be(newLimit);
// Should display the apply button
await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton');
await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButton');
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
const applyButtonDisabledStatus = await applyButton.getAttribute('disabled');
@ -792,7 +914,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
);
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
const applyButtonDisabledStatus = await applyButton.getAttribute('disabled');
@ -814,7 +936,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
});
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
await applyButton.click();

5
x-pack/test/functional/page_objects/dataset_quality.ts
View file

@ -205,7 +205,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
},
async waitUntilPossibleMitigationsLoaded() {
await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiSkeletonRectangle', 20 * 1000);
await find.waitForDeletedByCssSelector(
'.euiFlyoutBody .datasetQualityDetailsFlyoutManualMitigationsLoading',
20 * 1000
);
},
async waitUntilDegradedFieldFlyoutLoaded() {

13
x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts
View file

@ -365,7 +365,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
);
});
it('should show the degraded fields table with data when present', async () => {
it('should show the degraded fields table with data and spark plots when present', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
@ -378,17 +378,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
expect(rows.length).to.eql(3);
});
it('should display Spark Plot for every row of degraded fields', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
await PageObjects.datasetQuality.waitUntilTableLoaded();
const rows =
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
const sparkPlots = await testSubjects.findAll(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualitySparkPlot

132
x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts
View file

@ -44,13 +44,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const customComponentTemplateName = 'logs-synth@mappings';
const nginxAccessDatasetName = 'nginx.access';
const customComponentTemplateNameNginx = 'logs-nginx.access@custom';
const customComponentTemplateNameNginx = `logs-${nginxAccessDatasetName}@custom`;
const nginxAccessDataStreamName = `${type}-${nginxAccessDatasetName}-${defaultNamespace}`;
const nginxPkg = {
name: 'nginx',
version: '1.23.0',
};
const apmAppDatasetName = 'apm.app.tug';
const apmAppDataStreamName = `${type}-${apmAppDatasetName}-${defaultNamespace}`;
describe('Degraded fields flyout', () => {
describe('degraded field flyout open-close', () => {
before(async () => {
@ -183,6 +186,29 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 26 fields in Apm DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
})
.timestamp(timestamp)
);
}),
]);
// Set Limit of 25
@ -198,6 +224,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
'mapping.total_fields.limit': 42,
});
// Set Limit of 26
await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, {
'mapping.total_fields.limit': 25,
});
await synthtrace.index([
// Ingest Degraded Logs with 26 field
timerange(moment(to).subtract(count, 'minute'), moment(to))
@ -247,11 +278,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
'cloud.project.id': generateShortId(),
})
.timestamp(timestamp)
);
}),
]);
// Rollover Datastream to reset the limit to default which is 1000
await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName);
await PageObjects.datasetQuality.rolloverDataStream(nginxAccessDataStreamName);
await PageObjects.datasetQuality.rolloverDataStream(apmAppDataStreamName);
// Set Limit of 26
await PageObjects.datasetQuality.setDataStreamSettings(
@ -273,6 +329,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
}
);
// Set Limit of 27
await PageObjects.datasetQuality.setDataStreamSettings(
PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({
dataset: apmAppDatasetName,
}) + '-000002',
{
'mapping.total_fields.limit': 27,
}
);
await synthtrace.index([
// Ingest Degraded Logs with 26 field
timerange(moment(to).subtract(count, 'minute'), moment(to))
@ -322,6 +388,30 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
.timestamp(timestamp)
);
}),
// Ingest Degraded Logs with 27 fields in Apm APP DataSet
timerange(moment(to).subtract(count, 'minute'), moment(to))
.interval('1m')
.rate(1)
.generator((timestamp) => {
return Array(1)
.fill(0)
.flatMap(() =>
log
.create()
.dataset(apmAppDatasetName)
.message('a log message')
.logLevel(MORE_THAN_1024_CHARS)
.service(serviceName)
.namespace(defaultNamespace)
.defaults({
'service.name': serviceName,
'trace.id': generateShortId(),
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
'cloud.project.id': generateShortId(),
})
.timestamp(timestamp)
);
}),
]);
await PageObjects.svlCommonPage.loginAsAdmin();
});
@ -722,6 +812,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true);
});
it('should display increase field limit as a possible mitigation for special packages like apm app', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: apmAppDataStreamName,
expandedDegradedField: 'cloud.project',
});
// Field Limit Mitigation Section should exist
await testSubjects.existOrFail(
'datasetQualityDetailsDegradedFieldFlyoutFieldLimitMitigationAccordion'
);
// Should display the panel to increase field limit
await testSubjects.existOrFail(
'datasetQualityDetailsDegradedFieldFlyoutIncreaseFieldLimitPanel'
);
// Should display official online documentation link
await testSubjects.existOrFail(
'datasetQualityManualMitigationsPipelineOfficialDocumentationLink'
);
const linkButton = await testSubjects.find(
'datasetQualityManualMitigationsPipelineOfficialDocumentationLink'
);
const linkURL = await linkButton.getAttribute('href');
expect(linkURL?.endsWith('mapping-settings-limit.html')).to.be(true);
});
it('should display increase field limit as a possible mitigation for non integration', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDatasetWithLimitDataStreamName,
@ -787,10 +907,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
expect(newFieldLimit).to.be(newLimit);
// Should display the apply button
await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButtonButton');
await testSubjects.existOrFail('datasetQualityIncreaseFieldMappingLimitButton');
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
const applyButtonDisabledStatus = await applyButton.getAttribute('disabled');
@ -817,7 +937,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
);
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
const applyButtonDisabledStatus = await applyButton.getAttribute('disabled');
@ -844,7 +964,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await retry.tryForTime(5000, async () => {
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
await applyButton.click();
@ -864,7 +984,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.header.waitUntilLoadingHasFinished();
const applyButton = await testSubjects.find(
'datasetQualityIncreaseFieldMappingLimitButtonButton'
'datasetQualityIncreaseFieldMappingLimitButton'
);
await applyButton.click();