mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[ML] Add functional tests for Data Drift view (#167911)
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Quynh Nguyen (Quinn)
GitHub
parent
33207e3e66
commit
f28349faf1
12 changed files with 522 additions and 91 deletions
|
|
@ -112,6 +112,9 @@ export const DataDriftOverviewTable = ({
|
|||
|
||||
return (
|
||||
<EuiButtonIcon
|
||||
data-test-subj={`dataDriftToggleDetails-${
|
||||
itemIdToExpandedRowMapValues[item.featureName] ? 'expanded' : 'collapsed'
|
||||
}`}
|
||||
onClick={() => toggleDetails(item)}
|
||||
aria-label={itemIdToExpandedRowMapValues[item.featureName] ? COLLAPSE_ROW : EXPAND_ROW}
|
||||
iconType={itemIdToExpandedRowMapValues[item.featureName] ? 'arrowDown' : 'arrowRight'}
|
||||
|
|
@ -149,7 +152,10 @@ export const DataDriftOverviewTable = ({
|
|||
'data-test-subj': 'mlDataDriftOverviewTableDriftDetected',
|
||||
sortable: true,
|
||||
textOnly: true,
|
||||
render: (driftDetected: boolean) => {
|
||||
render: (driftDetected: boolean, item) => {
|
||||
// @ts-expect-error currently ES two_sided does return string NaN, will be fixed
|
||||
// NaN happens when the distributions are non overlapping. This means there is a drift.
|
||||
if (item.similarityTestPValue === 'NaN') return dataComparisonYesLabel;
|
||||
return <span>{driftDetected ? dataComparisonYesLabel : dataComparisonNoLabel}</span>;
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -94,7 +94,11 @@ export const PageHeader: FC = () => {
|
|||
|
||||
return (
|
||||
<EuiPageHeader
|
||||
pageTitle={<div css={dataViewTitleHeader}>{dataView.getName()}</div>}
|
||||
pageTitle={
|
||||
<div data-test-subj={'mlDataDriftPageDataViewTitle'} css={dataViewTitleHeader}>
|
||||
{dataView.getName()}
|
||||
</div>
|
||||
}
|
||||
rightSideItems={[
|
||||
<EuiFlexGroup gutterSize="s" data-test-subj="dataComparisonTimeRangeSelectorSection">
|
||||
{hasValidTimeField ? (
|
||||
|
|
|
|||
|
|
@ -9,8 +9,11 @@
|
|||
* formatSignificanceLevel
|
||||
* @param significanceLevel
|
||||
*/
|
||||
export const formatSignificanceLevel = (significanceLevel: number) => {
|
||||
export const formatSignificanceLevel = (significanceLevel: number | 'NaN') => {
|
||||
// NaN happens when the distributions are non overlapping. This means there is a drift, and the p-value would be astronomically small.
|
||||
if (significanceLevel === 'NaN') return '< 0.000001';
|
||||
if (typeof significanceLevel !== 'number' || isNaN(significanceLevel)) return '';
|
||||
|
||||
if (significanceLevel < 1e-6) {
|
||||
return '< 0.000001';
|
||||
} else if (significanceLevel < 0.01) {
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ export const DocumentCountWithDualBrush: FC<DocumentCountContentProps> = ({
|
|||
timeRangeEarliest === undefined ||
|
||||
timeRangeLatest === undefined
|
||||
) {
|
||||
return totalCount !== undefined ? <TotalCountHeader totalCount={totalCount} /> : null;
|
||||
return totalCount !== undefined ? <TotalCountHeader totalCount={totalCount} id={id} /> : null;
|
||||
}
|
||||
|
||||
const chartPoints: LogRateHistogramItem[] = Object.entries(documentCountStats.buckets).map(
|
||||
|
|
@ -166,7 +166,7 @@ export const DocumentCountWithDualBrush: FC<DocumentCountContentProps> = ({
|
|||
data-test-subj={getDataTestSubject('dataDriftTotalDocCountHeader', id)}
|
||||
>
|
||||
<EuiFlexItem>
|
||||
<TotalCountHeader totalCount={totalCount} approximate={approximate} label={label} />
|
||||
<TotalCountHeader totalCount={totalCount} approximate={approximate} label={label} id={id} />
|
||||
</EuiFlexItem>
|
||||
|
||||
<EuiFlexGroup gutterSize="m" direction="row" alignItems="center">
|
||||
|
|
|
|||
|
|
@ -246,7 +246,7 @@ export function DataDriftIndexPatternsEditor({
|
|||
children: (
|
||||
<EuiFlexItem grow={false}>
|
||||
<DataViewEditor
|
||||
key={'reference'}
|
||||
id={'reference'}
|
||||
label={
|
||||
<FormattedMessage
|
||||
id="xpack.ml.dataDrift.indexPatternsEditor.referenceData"
|
||||
|
|
@ -274,7 +274,7 @@ export function DataDriftIndexPatternsEditor({
|
|||
children: (
|
||||
<EuiFlexItem grow={false}>
|
||||
<DataViewEditor
|
||||
key={'comparison'}
|
||||
id={'comparison'}
|
||||
label={
|
||||
<FormattedMessage
|
||||
id="xpack.ml.dataDrift.indexPatternsEditor.comparisonDataIndexPatternHelp"
|
||||
|
|
@ -329,7 +329,7 @@ export function DataDriftIndexPatternsEditor({
|
|||
}}
|
||||
isClearable={false}
|
||||
isDisabled={comparisonIndexPattern === '' && referenceIndexPattern === ''}
|
||||
data-test-subj="timestampField"
|
||||
data-test-subj="mlDataDriftTimestampField"
|
||||
aria-label={i18n.translate(
|
||||
'xpack.ml.dataDrift.indexPatternsEditor.timestampSelectAriaLabel',
|
||||
{
|
||||
|
|
@ -389,7 +389,7 @@ export function DataDriftIndexPatternsEditor({
|
|||
disabled={hasError}
|
||||
onClick={createDataViewAndRedirectToDataDriftPage.bind(null, true)}
|
||||
iconType="visTagCloud"
|
||||
data-test-subj="analyzeDataDriftButton"
|
||||
data-test-subj="analyzeDataDriftWithoutSavingButton"
|
||||
aria-label={i18n.translate(
|
||||
'xpack.ml.dataDrift.indexPatternsEditor.analyzeDataDriftWithoutSavingLabel',
|
||||
{
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import { useTableSettings } from '../../data_frame_analytics/pages/analytics_man
|
|||
import { canAppendWildcard, matchedIndicesDefault } from './data_drift_index_patterns_editor';
|
||||
|
||||
interface DataViewEditorProps {
|
||||
id: string;
|
||||
label: ReactNode;
|
||||
dataViewEditorService: DataViewEditorService;
|
||||
indexPattern: string;
|
||||
|
|
@ -42,6 +43,7 @@ const mustMatchError = i18n.translate(
|
|||
);
|
||||
|
||||
export function DataViewEditor({
|
||||
id,
|
||||
label,
|
||||
dataViewEditorService,
|
||||
indexPattern,
|
||||
|
|
@ -131,6 +133,7 @@ export function DataViewEditor({
|
|||
isInvalid={errorMessage !== undefined}
|
||||
fullWidth
|
||||
helpText={helpText}
|
||||
data-test-subj={`mlDataDriftIndexPatternFormRow-${id ?? ''}`}
|
||||
>
|
||||
<EuiFieldText
|
||||
value={indexPattern}
|
||||
|
|
@ -149,7 +152,7 @@ export function DataViewEditor({
|
|||
setIndexPattern(query);
|
||||
}}
|
||||
fullWidth
|
||||
data-test-subj="createIndexPatternTitleInput"
|
||||
data-test-subj={`mlDataDriftIndexPatternTitleInput-${id ?? ''}`}
|
||||
placeholder="example-pattern*"
|
||||
/>
|
||||
</EuiFormRow>
|
||||
|
|
@ -183,6 +186,12 @@ export function DataViewEditor({
|
|||
columns={columns}
|
||||
pagination={pagination}
|
||||
onChange={onTableChange}
|
||||
data-test-subject={`mlDataDriftIndexPatternTable-${id ?? ''}`}
|
||||
rowProps={(item) => {
|
||||
return {
|
||||
'data-test-subj': `mlDataDriftIndexPatternTableRow row-${id}`,
|
||||
};
|
||||
}}
|
||||
/>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGrid>
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ export const DataDriftIndexOrSearchRedirect: FC = () => {
|
|||
iconType="plusInCircleFilled"
|
||||
onClick={() => navigateToPath(createPath(ML_PAGES.DATA_DRIFT_CUSTOM))}
|
||||
disabled={!canEditDataView}
|
||||
data-test-subj={'dataDriftCreateDataViewButton'}
|
||||
>
|
||||
<FormattedMessage
|
||||
id="xpack.ml.dataDrift.createDataViewButton"
|
||||
|
|
|
|||
|
|
@ -429,7 +429,7 @@ export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
|
|||
: []),
|
||||
{
|
||||
id: 'models_map',
|
||||
'data-test-subj': 'mlTrainedModelsMap',
|
||||
'data-test-subj': 'mlTrainedModelMap',
|
||||
name: (
|
||||
<FormattedMessage
|
||||
id="xpack.ml.trainedModels.modelsList.expandedRow.modelsMapLabel"
|
||||
|
|
@ -437,7 +437,7 @@ export const ExpandedRow: FC<ExpandedRowProps> = ({ item }) => {
|
|||
/>
|
||||
),
|
||||
content: (
|
||||
<div data-test-subj={'mlTrainedModelDetailsContent'}>
|
||||
<div data-test-subj={'mlTrainedModelMapContent'}>
|
||||
<EuiSpacer size={'s'} />
|
||||
<EuiFlexItem css={{ height: 300 }}>
|
||||
<JobMap
|
||||
|
|
|
|||
|
|
@ -6,8 +6,38 @@
|
|||
*/
|
||||
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { farequoteDataViewTestDataWithQuery } from '../../aiops/log_rate_analysis_test_data';
|
||||
import { TestData } from '../../aiops/types';
|
||||
|
||||
export const farequoteKQLFiltersSearchTestData = {
|
||||
suiteTitle: 'KQL saved search and filters',
|
||||
isSavedSearch: true,
|
||||
dateTimeField: '@timestamp',
|
||||
sourceIndexOrSavedSearch: 'ft_farequote_filter_and_kuery',
|
||||
chartClickCoordinates: [0, 0] as [number, number],
|
||||
dataViewName: 'ft_farequote',
|
||||
totalDocCount: '5,674',
|
||||
};
|
||||
|
||||
const dataViewCreationTestData = {
|
||||
suiteTitle: 'from data view creation mode',
|
||||
isSavedSearch: true,
|
||||
dateTimeField: '@timestamp',
|
||||
chartClickCoordinates: [0, 0] as [number, number],
|
||||
totalDocCount: '86,274',
|
||||
};
|
||||
|
||||
const nonTimeSeriesTestData = {
|
||||
suiteTitle: 'from data view creation mode',
|
||||
isSavedSearch: false,
|
||||
dateTimeField: '@timestamp',
|
||||
sourceIndexOrSavedSearch: 'ft_ihp_outlier',
|
||||
chartClickCoordinates: [0, 0] as [number, number],
|
||||
dataViewName: 'ft_ihp_outlier',
|
||||
};
|
||||
|
||||
type TestData =
|
||||
| typeof farequoteKQLFiltersSearchTestData
|
||||
| typeof dataViewCreationTestData
|
||||
| typeof nonTimeSeriesTestData;
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const ml = getService('ml');
|
||||
|
|
@ -15,97 +45,193 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
const elasticChart = getService('elasticChart');
|
||||
const esArchiver = getService('esArchiver');
|
||||
|
||||
function runTests(testData: TestData) {
|
||||
it(`${testData.suiteTitle} loads the source data in data drift`, async () => {
|
||||
await elasticChart.setNewChartUiDebugFlag(true);
|
||||
async function assertDataDriftPageContent(testData: TestData) {
|
||||
await PageObjects.header.waitUntilLoadingHasFinished();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} loads the saved search selection page`
|
||||
);
|
||||
await ml.navigation.navigateToDataDrift();
|
||||
await ml.testExecution.logTestStep(`${testData.suiteTitle} displays the time range step`);
|
||||
await ml.dataDrift.assertTimeRangeSelectorSectionExists();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} loads the data drift index or saved search select page`
|
||||
);
|
||||
await ml.jobSourceSelection.selectSourceForDataDrift(testData.sourceIndexOrSavedSearch);
|
||||
});
|
||||
await ml.testExecution.logTestStep(`${testData.suiteTitle} loads data for full time range`);
|
||||
await ml.dataDrift.clickUseFullDataButton();
|
||||
|
||||
it(`${testData.suiteTitle} displays index details`, async () => {
|
||||
await ml.testExecution.logTestStep(`${testData.suiteTitle} displays the time range step`);
|
||||
await ml.dataDrift.assertTimeRangeSelectorSectionExists();
|
||||
await ml.dataDrift.setRandomSamplingOption('Reference', 'dvRandomSamplerOptionOff');
|
||||
await ml.dataDrift.setRandomSamplingOption('Comparison', 'dvRandomSamplerOptionOff');
|
||||
|
||||
await ml.testExecution.logTestStep(`${testData.suiteTitle} loads data for full time range`);
|
||||
await ml.dataDrift.clickUseFullDataButton();
|
||||
await PageObjects.header.waitUntilLoadingHasFinished();
|
||||
|
||||
await ml.dataDrift.setRandomSamplingOption('Reference', 'dvRandomSamplerOptionOff');
|
||||
await ml.dataDrift.setRandomSamplingOption('Comparison', 'dvRandomSamplerOptionOff');
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} displays elements in the doc count panel correctly`
|
||||
);
|
||||
await ml.dataDrift.assertPrimarySearchBarExists();
|
||||
await ml.dataDrift.assertReferenceDocCountContent();
|
||||
await ml.dataDrift.assertComparisonDocCountContent();
|
||||
|
||||
await PageObjects.header.waitUntilLoadingHasFinished();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} displays elements in the doc count panel correctly`
|
||||
);
|
||||
await ml.dataDrift.assertPrimarySearchBarExists();
|
||||
await ml.dataDrift.assertReferenceDocCountContent();
|
||||
await ml.dataDrift.assertComparisonDocCountContent();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} displays elements in the page correctly`
|
||||
);
|
||||
await ml.dataDrift.assertNoWindowParametersEmptyPromptExists();
|
||||
await ml.testExecution.logTestStep(
|
||||
`${testData.suiteTitle} displays elements on the page correctly`
|
||||
);
|
||||
await ml.dataDrift.assertNoWindowParametersEmptyPromptExists();
|
||||
|
||||
if (testData.chartClickCoordinates) {
|
||||
await ml.testExecution.logTestStep('clicks the document count chart to start analysis');
|
||||
await ml.dataDrift.clickDocumentCountChart(
|
||||
'dataDriftDocCountChart-Reference',
|
||||
testData.chartClickCoordinates
|
||||
);
|
||||
await ml.dataDrift.runAnalysis();
|
||||
});
|
||||
}
|
||||
await ml.dataDrift.runAnalysis();
|
||||
}
|
||||
|
||||
describe('data drift', async function () {
|
||||
for (const testData of [farequoteDataViewTestDataWithQuery]) {
|
||||
describe(`with '${testData.sourceIndexOrSavedSearch}'`, function () {
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
|
||||
before(async () => {
|
||||
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/ihp_outlier');
|
||||
await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier');
|
||||
|
||||
await ml.testResources.createIndexPatternIfNeeded(
|
||||
testData.sourceIndexOrSavedSearch,
|
||||
'@timestamp'
|
||||
);
|
||||
await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote');
|
||||
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');
|
||||
await ml.testResources.createSavedSearchFarequoteFilterAndKueryIfNeeded();
|
||||
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
await ml.testResources.setKibanaTimeZoneToUTC();
|
||||
await ml.securityUI.loginAsMlPowerUser();
|
||||
});
|
||||
after(async () => {
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/ml/ihp_outlier');
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote');
|
||||
await Promise.all([
|
||||
ml.testResources.deleteIndexPatternByTitle('ft_fare*'),
|
||||
ml.testResources.deleteIndexPatternByTitle('ft_fare*,ft_fareq*'),
|
||||
ml.testResources.deleteIndexPatternByTitle('ft_farequote'),
|
||||
ml.testResources.deleteIndexPatternByTitle('ft_ihp_outlier'),
|
||||
]);
|
||||
});
|
||||
|
||||
if (testData.dataGenerator === 'kibana_sample_data_logs') {
|
||||
await PageObjects.security.login('elastic', 'changeme', {
|
||||
expectSuccess: true,
|
||||
});
|
||||
|
||||
await PageObjects.common.navigateToUrl('home', '/tutorial_directory/sampleData', {
|
||||
useActualUrl: true,
|
||||
});
|
||||
await PageObjects.header.waitUntilLoadingHasFinished();
|
||||
await PageObjects.home.addSampleDataSet('logs');
|
||||
await PageObjects.header.waitUntilLoadingHasFinished();
|
||||
} else {
|
||||
await ml.securityUI.loginAsMlPowerUser();
|
||||
}
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await elasticChart.setNewChartUiDebugFlag(false);
|
||||
await ml.testResources.deleteIndexPatternByTitle(testData.sourceIndexOrSavedSearch);
|
||||
await esArchiver.unload('x-pack/test/functional/es_archives/ml/farequote');
|
||||
});
|
||||
|
||||
it(`${testData.suiteTitle} loads the ml page`, async () => {
|
||||
// Start navigation from the base of the ML app.
|
||||
await ml.navigation.navigateToMl();
|
||||
await elasticChart.setNewChartUiDebugFlag(true);
|
||||
});
|
||||
|
||||
runTests(testData);
|
||||
describe('with ft_farequote_filter_and_kuery from index selection page', async function () {
|
||||
after(async () => {
|
||||
await elasticChart.setNewChartUiDebugFlag(false);
|
||||
});
|
||||
}
|
||||
|
||||
it(`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the ml page`, async () => {
|
||||
// Start navigation from the base of the ML app.
|
||||
await ml.navigation.navigateToMl();
|
||||
await elasticChart.setNewChartUiDebugFlag(true);
|
||||
});
|
||||
|
||||
it(`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the source data in data drift`, async () => {
|
||||
await ml.testExecution.logTestStep(
|
||||
`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the data drift index or saved search select page`
|
||||
);
|
||||
await ml.navigation.navigateToDataDrift();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${farequoteKQLFiltersSearchTestData.suiteTitle} loads the data drift view`
|
||||
);
|
||||
await ml.jobSourceSelection.selectSourceForDataDrift(
|
||||
farequoteKQLFiltersSearchTestData.sourceIndexOrSavedSearch
|
||||
);
|
||||
await assertDataDriftPageContent(farequoteKQLFiltersSearchTestData);
|
||||
|
||||
if (farequoteKQLFiltersSearchTestData.dataViewName !== undefined) {
|
||||
await ml.dataDrift.assertDataViewTitle(farequoteKQLFiltersSearchTestData.dataViewName);
|
||||
}
|
||||
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Reference',
|
||||
farequoteKQLFiltersSearchTestData.totalDocCount
|
||||
);
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Comparison',
|
||||
farequoteKQLFiltersSearchTestData.totalDocCount
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe(dataViewCreationTestData.suiteTitle, function () {
|
||||
beforeEach(`${dataViewCreationTestData.suiteTitle} loads the ml page`, async () => {
|
||||
// Start navigation from the base of the ML app.
|
||||
await ml.navigation.navigateToMl();
|
||||
await elasticChart.setNewChartUiDebugFlag(true);
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} loads the saved search selection page`
|
||||
);
|
||||
await ml.navigation.navigateToDataDrift();
|
||||
});
|
||||
|
||||
it(`${dataViewCreationTestData.suiteTitle} allows analyzing data drift without saving`, async () => {
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} creates new data view`
|
||||
);
|
||||
await ml.dataDrift.navigateToCreateNewDataViewPage();
|
||||
await ml.dataDrift.assertIndexPatternNotEmptyFormErrorExists('reference');
|
||||
await ml.dataDrift.assertIndexPatternNotEmptyFormErrorExists('comparison');
|
||||
await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(true);
|
||||
await ml.dataDrift.assertAnalyzeDataDriftButtonState(true);
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} sets index patterns`
|
||||
);
|
||||
await ml.dataDrift.setIndexPatternInput('reference', 'ft_fare*');
|
||||
await ml.dataDrift.setIndexPatternInput('comparison', 'ft_fareq*');
|
||||
|
||||
await ml.dataDrift.selectTimeField(dataViewCreationTestData.dateTimeField);
|
||||
|
||||
await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(false);
|
||||
await ml.dataDrift.assertAnalyzeDataDriftButtonState(false);
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} redirects to data drift page`
|
||||
);
|
||||
await ml.dataDrift.clickAnalyzeWithoutSavingButton();
|
||||
await assertDataDriftPageContent(dataViewCreationTestData);
|
||||
await ml.dataDrift.assertDataViewTitle('ft_fare*,ft_fareq*');
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Reference',
|
||||
dataViewCreationTestData.totalDocCount
|
||||
);
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Comparison',
|
||||
dataViewCreationTestData.totalDocCount
|
||||
);
|
||||
});
|
||||
|
||||
it(`${dataViewCreationTestData.suiteTitle} hides analyze data drift without saving option if patterns are same`, async () => {
|
||||
await ml.dataDrift.navigateToCreateNewDataViewPage();
|
||||
await ml.dataDrift.assertAnalyzeWithoutSavingButtonState(true);
|
||||
await ml.dataDrift.assertAnalyzeDataDriftButtonState(true);
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} sets index patterns`
|
||||
);
|
||||
await ml.dataDrift.setIndexPatternInput('reference', 'ft_fare*');
|
||||
await ml.dataDrift.setIndexPatternInput('comparison', 'ft_fare*');
|
||||
|
||||
await ml.dataDrift.assertAnalyzeWithoutSavingButtonMissing();
|
||||
await ml.dataDrift.assertAnalyzeDataDriftButtonState(false);
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} redirects to data drift page`
|
||||
);
|
||||
await ml.dataDrift.clickAnalyzeDataDrift();
|
||||
await assertDataDriftPageContent(dataViewCreationTestData);
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
`${dataViewCreationTestData.suiteTitle} does not create new data view, and uses available one with matching index pattern`
|
||||
);
|
||||
await ml.dataDrift.assertDataViewTitle('ft_farequote');
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Reference',
|
||||
dataViewCreationTestData.totalDocCount
|
||||
);
|
||||
await ml.dataDrift.assertTotalDocumentCount(
|
||||
'Comparison',
|
||||
dataViewCreationTestData.totalDocCount
|
||||
);
|
||||
});
|
||||
|
||||
it(`${nonTimeSeriesTestData.suiteTitle} loads non-time series data`, async () => {
|
||||
await ml.jobSourceSelection.selectSourceForDataDrift(
|
||||
nonTimeSeriesTestData.sourceIndexOrSavedSearch
|
||||
);
|
||||
await ml.dataDrift.runAnalysis();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,6 +67,25 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
},
|
||||
};
|
||||
|
||||
const modelWithPipelineAndDestIndex = {
|
||||
modelId: 'dfa_regression_model_n_1',
|
||||
description: '',
|
||||
modelTypes: ['regression', 'tree_ensemble'],
|
||||
};
|
||||
const modelWithPipelineAndDestIndexExpectedValues = {
|
||||
dataViewTitle: `user-index_${modelWithPipelineAndDestIndex.modelId}`,
|
||||
index: `user-index_${modelWithPipelineAndDestIndex.modelId}`,
|
||||
name: `ml-inference-${modelWithPipelineAndDestIndex.modelId}`,
|
||||
description: '',
|
||||
inferenceConfig: {
|
||||
regression: {
|
||||
results_field: 'predicted_value',
|
||||
num_top_feature_importance_values: 0,
|
||||
},
|
||||
},
|
||||
fieldMap: {},
|
||||
};
|
||||
|
||||
before(async () => {
|
||||
for (const model of trainedModels) {
|
||||
await ml.api.importTrainedModel(model.id, model.name);
|
||||
|
|
@ -77,17 +96,32 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
|
||||
// Make sure the .ml-stats index is created in advance, see https://github.com/elastic/elasticsearch/issues/65846
|
||||
await ml.api.assureMlStatsIndexExists();
|
||||
|
||||
// Create ingest pipeline and destination index that's tied to model
|
||||
await ml.api.createIngestPipeline(modelWithPipelineAndDestIndex.modelId);
|
||||
await ml.api.createIndex(modelWithPipelineAndDestIndexExpectedValues.index, undefined, {
|
||||
index: { default_pipeline: `pipeline_${modelWithPipelineAndDestIndex.modelId}` },
|
||||
});
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await ml.api.stopAllTrainedModelDeploymentsES();
|
||||
await ml.api.deleteAllTrainedModelsES();
|
||||
|
||||
await ml.api.cleanMlIndices();
|
||||
await ml.api.deleteIndices(modelWithPipelineAndDestIndexExpectedValues.index);
|
||||
|
||||
await ml.api.deleteIngestPipeline(modelWithoutPipelineDataExpectedValues.name, false);
|
||||
await ml.api.deleteIngestPipeline(
|
||||
modelWithoutPipelineDataExpectedValues.duplicateName,
|
||||
false
|
||||
);
|
||||
await ml.api.cleanMlIndices();
|
||||
|
||||
// Need to delete index before ingest pipeline, else it will give error
|
||||
await ml.api.deleteIngestPipeline(modelWithPipelineAndDestIndex.modelId);
|
||||
await ml.testResources.deleteIndexPatternByTitle(
|
||||
modelWithPipelineAndDestIndexExpectedValues.dataViewTitle
|
||||
);
|
||||
});
|
||||
|
||||
describe('for ML user with read-only access', () => {
|
||||
|
|
@ -387,7 +421,43 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
);
|
||||
});
|
||||
|
||||
it('navigates to data drift', async () => {
|
||||
await ml.testExecution.logTestStep('should show the model map in the expanded row');
|
||||
await ml.trainedModelsTable.ensureRowIsExpanded(modelWithPipelineAndDestIndex.modelId);
|
||||
await ml.trainedModelsTable.assertModelsMapTabContent();
|
||||
|
||||
await ml.testExecution.logTestStep(
|
||||
'should navigate to data drift index pattern creation page'
|
||||
);
|
||||
|
||||
await ml.trainedModelsTable.assertAnalyzeDataDriftActionButtonEnabled(
|
||||
modelWithPipelineAndDestIndex.modelId,
|
||||
true
|
||||
);
|
||||
await ml.trainedModelsTable.clickAnalyzeDataDriftActionButton(
|
||||
modelWithPipelineAndDestIndex.modelId
|
||||
);
|
||||
|
||||
await ml.testExecution.logTestStep(`sets index pattern for reference data set`);
|
||||
await ml.dataDrift.setIndexPatternInput(
|
||||
'reference',
|
||||
`${modelWithPipelineAndDestIndexExpectedValues.index}*`
|
||||
);
|
||||
await ml.dataDrift.assertIndexPatternInput(
|
||||
'comparison',
|
||||
modelWithPipelineAndDestIndexExpectedValues.index
|
||||
);
|
||||
|
||||
await ml.testExecution.logTestStep(`redirects to data drift page`);
|
||||
await ml.trainedModelsTable.clickAnalyzeDataDriftWithoutSaving();
|
||||
await ml.navigation.navigateToTrainedModels();
|
||||
});
|
||||
|
||||
describe('with imported models', function () {
|
||||
before(async () => {
|
||||
await ml.navigation.navigateToTrainedModels();
|
||||
});
|
||||
|
||||
for (const model of trainedModels) {
|
||||
it(`renders expanded row content correctly for imported tiny model ${model.id} without pipelines`, async () => {
|
||||
await ml.trainedModelsTable.ensureRowIsExpanded(model.id);
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
import expect from '@kbn/expect';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
|
||||
type SubjectId = 'reference' | 'comparison';
|
||||
|
||||
export function MachineLearningDataDriftProvider({
|
||||
getService,
|
||||
getPageObjects,
|
||||
|
|
@ -17,6 +19,7 @@ export function MachineLearningDataDriftProvider({
|
|||
const PageObjects = getPageObjects(['discover', 'header']);
|
||||
const elasticChart = getService('elasticChart');
|
||||
const browser = getService('browser');
|
||||
const comboBox = getService('comboBox');
|
||||
|
||||
type RandomSamplerOption =
|
||||
| 'dvRandomSamplerOptionOnAutomatic'
|
||||
|
|
@ -29,11 +32,27 @@ export function MachineLearningDataDriftProvider({
|
|||
return `${testSubject}-${id}`;
|
||||
},
|
||||
|
||||
async assertDataViewTitle(expectedTitle: string) {
|
||||
const selector = 'mlDataDriftPageDataViewTitle';
|
||||
await testSubjects.existOrFail(selector);
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const title = await testSubjects.getVisibleText(selector);
|
||||
expect(title).to.eql(
|
||||
expectedTitle,
|
||||
`Expected data drift page's data view title to be '${expectedTitle}' (got '${title}')`
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
async assertTimeRangeSelectorSectionExists() {
|
||||
await testSubjects.existOrFail('dataComparisonTimeRangeSelectorSection');
|
||||
},
|
||||
|
||||
async assertTotalDocumentCount(selector: string, expectedFormattedTotalDocCount: string) {
|
||||
async assertTotalDocumentCount(
|
||||
id: 'Reference' | 'Comparison',
|
||||
expectedFormattedTotalDocCount: string
|
||||
) {
|
||||
const selector = `dataVisualizerTotalDocCount-${id}`;
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const docCount = await testSubjects.getVisibleText(selector);
|
||||
expect(docCount).to.eql(
|
||||
|
|
@ -206,9 +225,126 @@ export function MachineLearningDataDriftProvider({
|
|||
async runAnalysis() {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.click(`aiopsRerunAnalysisButton`);
|
||||
// As part of the interface for the histogram brushes, the button to clear the selection should be present
|
||||
await this.assertDataDriftTableExists();
|
||||
});
|
||||
},
|
||||
|
||||
async navigateToCreateNewDataViewPage() {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.click(`dataDriftCreateDataViewButton`);
|
||||
await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`);
|
||||
});
|
||||
},
|
||||
|
||||
async assertIndexPatternNotEmptyFormErrorExists(id: SubjectId) {
|
||||
const subj = `mlDataDriftIndexPatternFormRow-${id ?? ''}`;
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.existOrFail(subj);
|
||||
const row = await testSubjects.find(subj);
|
||||
const errorElements = await row.findAllByClassName('euiFormErrorText');
|
||||
expect(await errorElements[0].getVisibleText()).eql('Index pattern must not be empty.');
|
||||
});
|
||||
},
|
||||
|
||||
async assertIndexPatternInput(id: SubjectId, expectedText: string) {
|
||||
const inputSelector = `mlDataDriftIndexPatternTitleInput-${id}`;
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const input = await testSubjects.find(inputSelector);
|
||||
const text = await input.getAttribute('value');
|
||||
expect(text).eql(
|
||||
expectedText,
|
||||
`Expected ${inputSelector} to have text ${expectedText} (got ${text})`
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
async setIndexPatternInput(id: SubjectId, pattern: string) {
|
||||
const inputSelector = `mlDataDriftIndexPatternTitleInput-${id}`;
|
||||
|
||||
// The input for index pattern automatically appends "*" at the end of the string
|
||||
// So here we just omit that * at the end to avoid double characters
|
||||
|
||||
await retry.tryForTime(10 * 1000, async () => {
|
||||
const hasWildCard = pattern.endsWith('*');
|
||||
const trimmedPattern = hasWildCard ? pattern.substring(0, pattern.length - 1) : pattern;
|
||||
|
||||
const input = await testSubjects.find(inputSelector);
|
||||
await input.clearValue();
|
||||
|
||||
await testSubjects.setValue(inputSelector, trimmedPattern, {
|
||||
clearWithKeyboard: true,
|
||||
typeCharByChar: true,
|
||||
});
|
||||
|
||||
if (!hasWildCard) {
|
||||
// If original pattern does not have wildcard, make to delete the wildcard
|
||||
await input.focus();
|
||||
await browser.pressKeys(browser.keys.DELETE);
|
||||
}
|
||||
|
||||
await this.assertIndexPatternInput(id, pattern);
|
||||
});
|
||||
},
|
||||
|
||||
async assertAnalyzeWithoutSavingButtonMissing() {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.missingOrFail('analyzeDataDriftWithoutSavingButton');
|
||||
});
|
||||
},
|
||||
|
||||
async assertAnalyzeWithoutSavingButtonState(disabled = true) {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const isDisabled = !(await testSubjects.isEnabled('analyzeDataDriftWithoutSavingButton'));
|
||||
expect(isDisabled).to.equal(
|
||||
disabled,
|
||||
`Expect analyze without saving button disabled state to be ${disabled} (got ${isDisabled})`
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
async assertAnalyzeDataDriftButtonState(disabled = true) {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const isDisabled = !(await testSubjects.isEnabled('analyzeDataDriftButton'));
|
||||
expect(isDisabled).to.equal(
|
||||
disabled,
|
||||
`Expect analyze data drift button disabled state to be ${disabled} (got ${isDisabled})`
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
async clickAnalyzeWithoutSavingButton() {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.existOrFail('analyzeDataDriftWithoutSavingButton');
|
||||
await testSubjects.click('analyzeDataDriftWithoutSavingButton');
|
||||
await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`);
|
||||
});
|
||||
},
|
||||
|
||||
async clickAnalyzeDataDrift() {
|
||||
await retry.tryForTime(5000, async () => {
|
||||
await testSubjects.existOrFail('analyzeDataDriftButton');
|
||||
await testSubjects.click('analyzeDataDriftButton');
|
||||
await testSubjects.existOrFail(`mlPageDataDriftCustomIndexPatterns`);
|
||||
});
|
||||
},
|
||||
|
||||
async assertDataDriftTimestampField(expectedIdentifier: string) {
|
||||
await retry.tryForTime(2000, async () => {
|
||||
const comboBoxSelectedOptions = await comboBox.getComboBoxSelectedOptions(
|
||||
'mlDataDriftTimestampField > comboBoxInput'
|
||||
);
|
||||
expect(comboBoxSelectedOptions).to.eql(
|
||||
expectedIdentifier === '' ? [] : [expectedIdentifier],
|
||||
`Expected type field to be '${expectedIdentifier}' (got '${comboBoxSelectedOptions}')`
|
||||
);
|
||||
});
|
||||
},
|
||||
|
||||
async selectTimeField(timeFieldName: string) {
|
||||
await comboBox.set('mlDataDriftTimestampField', timeFieldName);
|
||||
|
||||
await this.assertDataDriftTimestampField(timeFieldName);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,6 +226,71 @@ export function TrainedModelsTableProvider(
|
|||
);
|
||||
}
|
||||
|
||||
public async assertModelAnalyzeDataDriftButtonExists(modelId: string, expectedValue: boolean) {
|
||||
const actionsExists = await testSubjects.exists(
|
||||
this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction')
|
||||
);
|
||||
|
||||
expect(actionsExists).to.eql(
|
||||
expectedValue,
|
||||
`Expected row analyze data drift action button for trained model '${modelId}' to be ${
|
||||
expectedValue ? 'visible' : 'hidden'
|
||||
} (got ${actionsExists ? 'visible' : 'hidden'})`
|
||||
);
|
||||
}
|
||||
|
||||
public async assertAnalyzeDataDriftActionButtonEnabled(
|
||||
modelId: string,
|
||||
expectedValue: boolean
|
||||
) {
|
||||
const actionsButtonExists = await this.doesModelCollapsedActionsButtonExist(modelId);
|
||||
|
||||
let isEnabled = null;
|
||||
await retry.tryForTime(5 * 1000, async () => {
|
||||
if (actionsButtonExists) {
|
||||
await this.toggleActionsContextMenu(modelId, true);
|
||||
const panelElement = await find.byCssSelector('.euiContextMenuPanel');
|
||||
const actionButton = await panelElement.findByTestSubject('mlModelsTableRowDeleteAction');
|
||||
isEnabled = await actionButton.isEnabled();
|
||||
// escape popover
|
||||
await browser.pressKeys(browser.keys.ESCAPE);
|
||||
} else {
|
||||
await this.assertModelDeleteActionButtonExists(modelId, true);
|
||||
isEnabled = await testSubjects.isEnabled(
|
||||
this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction')
|
||||
);
|
||||
}
|
||||
|
||||
expect(isEnabled).to.eql(
|
||||
expectedValue,
|
||||
`Expected row analyze data drift action button for trained model '${modelId}' to be '${
|
||||
expectedValue ? 'enabled' : 'disabled'
|
||||
}' (got '${isEnabled ? 'enabled' : 'disabled'}')`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
public async clickAnalyzeDataDriftActionButton(modelId: string) {
|
||||
await retry.tryForTime(30 * 1000, async () => {
|
||||
const actionsButtonExists = await this.doesModelCollapsedActionsButtonExist(modelId);
|
||||
if (actionsButtonExists) {
|
||||
await this.toggleActionsContextMenu(modelId, true);
|
||||
const panelElement = await find.byCssSelector('.euiContextMenuPanel');
|
||||
const actionButton = await panelElement.findByTestSubject(
|
||||
'mlModelsAnalyzeDataDriftAction'
|
||||
);
|
||||
await actionButton.click();
|
||||
// escape popover
|
||||
await browser.pressKeys(browser.keys.ESCAPE);
|
||||
} else {
|
||||
await this.assertModelDeleteActionButtonExists(modelId, true);
|
||||
await testSubjects.click(this.rowSelector(modelId, 'mlModelsAnalyzeDataDriftAction'));
|
||||
}
|
||||
|
||||
await testSubjects.existOrFail('mlPageDataDriftCustomIndexPatterns');
|
||||
});
|
||||
}
|
||||
|
||||
public async assertModelTestButtonExists(modelId: string, expectedValue: boolean) {
|
||||
const actionExists = await testSubjects.exists(
|
||||
this.rowSelector(modelId, 'mlModelsTableRowTestAction')
|
||||
|
|
@ -480,7 +545,7 @@ export function TrainedModelsTableProvider(
|
|||
}
|
||||
|
||||
public async assertTabContent(
|
||||
type: 'details' | 'stats' | 'inferenceConfig' | 'pipelines',
|
||||
type: 'details' | 'stats' | 'inferenceConfig' | 'pipelines' | 'map',
|
||||
expectVisible = true
|
||||
) {
|
||||
const tabTestSubj = `mlTrainedModel${upperFirst(type)}`;
|
||||
|
|
@ -500,6 +565,10 @@ export function TrainedModelsTableProvider(
|
|||
await this.assertTabContent('details', expectVisible);
|
||||
}
|
||||
|
||||
public async assertModelsMapTabContent(expectVisible = true) {
|
||||
await this.assertTabContent('map', expectVisible);
|
||||
}
|
||||
|
||||
public async assertInferenceConfigTabContent(expectVisible = true) {
|
||||
await this.assertTabContent('inferenceConfig', expectVisible);
|
||||
}
|
||||
|
|
@ -526,5 +595,12 @@ export function TrainedModelsTableProvider(
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async clickAnalyzeDataDriftWithoutSaving() {
|
||||
await retry.tryForTime(5 * 1000, async () => {
|
||||
await testSubjects.clickWhenNotDisabled('analyzeDataDriftWithoutSavingButton');
|
||||
await testSubjects.existOrFail('mlDataDriftTable');
|
||||
});
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue