Tag serverless UI tests for esGate (#188239)

## Summary

This PR tags the majority of serverless FTR UI tests with `esGate` so
they're included in the elasticsearch-serverless commit verification
process.

### Details

#### Why we do this

We had cases where elasticsearch-serverless changes broke Kibana
functionality and we only discovered this very late in the cycle, in
some cases even after release to production. An investigation showed
that these cases would have been caught by the Kibana serverless UI
tests. So this is added in the interest of closing gaps in ES-KBN
compatibility checks.

#### Stability and run time

Run time:
One requirement for adding more tests to the elasticsearch-serverless
verification is that the overall run time is not increased. When
comparing the ES verification run time details from the ES process with
the ES verification from the Kibana process (daily run where we already
include UI tests), we can see that the overall run time is dominated by
Jest integration tests. So enabling the UI tests will not extend the
overall run time for the ES process.

Test stability:
Initially, we didn't include UI tests in the elasticsearch-serverless
verification because of test flakiness concerns. This has greatly
improved since the early days of serverless tests and we only rarely see
UI test failures on `main` and if we do, they're addressed quickly (the
chart shows number of serverless UI test failures on `main` over the
past 30 days). We do not run the ES verification against latest `main`
but only against a successfully checked commit level, so the chance that
one of these UI failures makes it into the verification process is
overall low. Given the benefits of closing big test gaps, we consider it
worth the risk.

#### Test inclusion

* initially, the regular index files are included
* for this cycle, the feature flag specific UI tests are not included,
this might come as a follow-up
This commit is contained in:
Robert Oskamp 2024-07-16 08:44:13 +02:00 committed by GitHub
parent 642216820e
commit d5eac1ca9e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 60 additions and 4 deletions

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Console', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./console'));
});
}

View file

@ -13,6 +13,8 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
const kibanaServer = getService('kibanaServer');
describe('context app', function () {
this.tags(['esGate']);
before(async () => {
await browser.setWindowSize(1200, 800);
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext) => {
describe('Serverless Common UI - Dev Tools', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./search_profiler'));
});
};

View file

@ -14,7 +14,9 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid
const from = '2024-06-10T14:00:00.000Z';
const to = '2024-06-10T16:30:00.000Z';
describe('discover/context_awareness', () => {
describe('discover/context_awareness', function () {
this.tags(['esGate']);
before(async () => {
await esArchiver.load('test/functional/fixtures/es_archiver/discover/context_awareness');
await kibanaServer.importExport.load(

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('discover/embeddable', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./_saved_search_embeddable'));
});
}

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('discover/esql', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./_esql_view'));
});
}

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group1', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1300, 800);
});

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group2', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1600, 1200);
});

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group3', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1300, 800);
});

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group4', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1600, 1200);
});

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group5', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1300, 800);
});

View file

@ -12,6 +12,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
const browser = getService('browser');
describe('discover/group6', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1300, 800);
});

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('discover', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./reporting'));
loadTestFile(require.resolve('./visualize_field'));
});

View file

@ -10,6 +10,8 @@ export default ({ getService, loadTestFile }: FtrProviderContext) => {
const browser = getService('browser');
describe('Discover alerting', function () {
this.tags(['esGate']);
before(async function () {
await browser.setWindowSize(1600, 1200);
});

View file

@ -9,7 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Examples', function () {
this.tags('skipMKI');
this.tags(['skipMKI', 'esGate']);
loadTestFile(require.resolve('./data_view_field_editor_example'));
loadTestFile(require.resolve('./discover_customization_examples'));
loadTestFile(require.resolve('./field_formats'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Grok debugger', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./grok_debugger'));
});
}

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Home Page', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./home_page'));
loadTestFile(require.resolve('./sample_data'));
});

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext) => {
describe('Serverless Common UI - Management', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./index_management'));
loadTestFile(require.resolve('./transforms/search_bar_features'));
loadTestFile(require.resolve('./transforms/transform_list'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Painless lab', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./painless_lab'));
});
}

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless Common UI - Platform Security', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./api_keys'));
loadTestFile(require.resolve('./navigation/avatar_menu'));
loadTestFile(require.resolve('./user_profiles/user_profiles'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext) => {
describe('Reporting', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./management'));
});
};

View file

@ -9,7 +9,8 @@ import type { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Saved Objects Management', function () {
this.tags('skipMKI');
this.tags(['skipMKI', 'esGate']);
loadTestFile(require.resolve('./find'));
loadTestFile(require.resolve('./scroll_count'));
loadTestFile(require.resolve('./bulk_get'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext) => {
describe('Spaces', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./spaces_selection.ts'));
});
};

View file

@ -17,7 +17,9 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext
const config = getService('config');
let remoteEsArchiver;
describe('lens serverless - group 1', () => {
describe('lens serverless - group 1', function () {
this.tags(['esGate']);
const esArchive = 'x-pack/test/functional/es_archives/logstash_functional';
const localIndexPatternString = 'logstash-*';
const remoteIndexPatternString = 'ftr-remote:logstash-*';

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext) => {
describe('Visualizations - Group 2', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./open_in_lens/agg_based'));
});
};

View file

@ -11,6 +11,8 @@ export default ({ loadTestFile, getPageObject }: FtrProviderContext) => {
const svlCommonPage = getPageObject('svlCommonPage');
describe('Visualizations - Group 3', function () {
this.tags(['esGate']);
before(async () => {
await svlCommonPage.loginWithPrivilegedRole();
});

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless observability UI', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./landing_page'));
loadTestFile(require.resolve('./navigation'));
loadTestFile(require.resolve('./observability_logs_explorer'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless search UI', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./landing_page'));
loadTestFile(require.resolve('./connectors/connectors_overview'));
loadTestFile(require.resolve('./default_dataview'));

View file

@ -9,6 +9,8 @@ import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless security UI', function () {
this.tags(['esGate']);
loadTestFile(require.resolve('./ftr/landing_page'));
loadTestFile(require.resolve('./ftr/navigation'));
loadTestFile(require.resolve('./ftr/cases'));