github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 01:13:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[FTR] Move serverless tests depending on feature flag to its own config file (#163929)

Browse source 
## Summary

It seems beneficial to have feature flag tests in a separate test config
file
 - tests are still run on Kibana CI automatically
- tests are not run on MKI projects automatically, but you can deploy
custom project and run tests via feature flags config

All the feature flags within the same project should be places in the
same config to make sure there is no arguments conflict.
When the flag is moved to the yml configuration, we can rely on Kibana
CI and manually triggered deployment to make sure projects are
functioning correctly.

---------

Co-authored-by: Robert Oskamp <robert.oskamp@elastic.co>
This commit is contained in:
Dzmitry Lemechko 2023-08-22 14:09:41 +02:00 • committed by GitHub
parent 546a3cb1af
commit 7ece4e5df3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 469 additions and 304 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.buildkite/ftr_configs.yml
View file

@ -81,12 +81,18 @@ disabled:
# Serverless configs, currently only for manual tests runs, CI integration planned
- x-pack/test_serverless/api_integration/test_suites/common/config.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.ts
- x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.ts
- x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.ts
- x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/common/config.ts
- x-pack/test_serverless/functional/test_suites/observability/config.ts
- x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/search/config.ts
- x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
- x-pack/test_serverless/functional/test_suites/security/config.ts
- x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
defaultQueue: 'n2-4-spot'
enabled:

3
.buildkite/scripts/steps/functional/serverless_ftr.sh
View file

@ -9,17 +9,20 @@ export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT"
if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/search/config.ts"
"x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/search/config.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "observability" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/observability/config.ts"
"x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/observability/config.ts"
"x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts"
)
elif [[ "$SERVERLESS_ENVIRONMENT" == "security" ]]; then
SERVERLESS_CONFIGS=(
"x-pack/test_serverless/api_integration/test_suites/security/config.ts"
"x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts"
"x-pack/test_serverless/functional/test_suites/security/config.ts"
)
fi

27
x-pack/test_serverless/README.md
View file

@ -97,6 +97,33 @@ tests that should run in a serverless environment have to be added to the
Tests in this area should be clearly designed for the serverless environment,
particularly when it comes to timing for API requests and UI interaction.
### Testing with feature flags
**tl;dr:** Tests specific to functionality behind a feature flag need special
handling and are by default only tested locally / in CI but excluded from regular
test runs in MKI.
New features might be gated behind a feature flag and can only be enabled
through a yml configuration entry. By default, these features are not enabled
so they're not available in a regular serverless MKI project, which would make
end-to-end tests for such a feature fail. In order to still have tests for
features behind a feature flag, these tests need to be separated from the
regular tests.
For every project's `test_suites` directory, there are feature flags specific
config (`config.feature_flags.ts`) and index (`index.feature_flags.ts`) files
next to the regular `config.ts` and `index.ts`. These extra files are used to
cover all feature flag tests of the respective area.
If you want to add feature flag specific tests:
- Add your feature flag(s) to the `kbnServerArgs` in the `config.feature_flags.ts` file
- Load your test file(s) in the `index.feature_flags.ts` file
As mentioned above, these tests are not part of the regular test run against MKI
projects. If you still want to run feature flag tests against an MKI project,
this requires a Kibana docker build that has the feature flags enabled by default.
This docker image can then be used to create a project in serverless QA and the
feature flags tests can be pointed to the project.
## Run tests
Similar to how functional tests are run in `x-pack/test`, you can point the
functional tests server and test runner to config files in this `x-pack/test_serverless`

4
x-pack/test_serverless/api_integration/config.base.ts
View file

@ -25,9 +25,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
serverArgs: [
...svlSharedConfig.get('kbnTestServer.serverArgs'),
`--serverless=${options.serverlessProject}`,
`--xpack.alerting.enableFrameworkAlerts=true`,
'--xpack.observability.unsafe.thresholdRule.enabled=true',
'--server.publicBaseUrl=https://localhost:5601',
...(options.kbnServerArgs || []),
],
},
testFiles: options.testFiles,

54
x-pack/test_serverless/api_integration/services/alerting_api.ts
View file

@ -10,6 +10,9 @@ import type {
SearchResponse,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { MetricThresholdParams } from '@kbn/infra-plugin/common/alerting/metrics';
import { ThresholdParams } from '@kbn/observability-plugin/common/threshold_rule/types';
import { FtrProviderContext } from '../ftr_provider_context';
export function AlertingApiProvider({ getService }: FtrProviderContext) {
@ -86,5 +89,56 @@ export function AlertingApiProvider({ getService }: FtrProviderContext) {
return response;
});
},
async createIndexConnector({ name, indexName }: { name: string; indexName: string }) {
const { body } = await supertest
.post(`/api/actions/connector`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
name,
config: {
index: indexName,
refresh: true,
},
connector_type_id: '.index',
});
return body.id as string;
},
async createRule({
name,
ruleTypeId,
params,
actions = [],
tags = [],
schedule,
consumer,
}: {
ruleTypeId: string;
name: string;
params: MetricThresholdParams | ThresholdParams;
actions?: any[];
tags?: any[];
schedule?: { interval: string };
consumer: string;
}) {
const { body } = await supertest
.post(`/api/alerting/rule`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
params,
consumer,
schedule: schedule || {
interval: '5m',
},
tags,
name,
rule_type_id: ruleTypeId,
actions,
});
return body;
},
};
}

52
x-pack/test_serverless/api_integration/services/data_view_api.ts Normal file
View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../ftr_provider_context';
export function DataViewApiProvider({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
return {
async create({ id, name, title }: { id: string; name: string; title: string }) {
const { body } = await supertest
.post(`/api/content_management/rpc/create`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
contentTypeId: 'index-pattern',
data: {
fieldAttrs: '{}',
title,
timeFieldName: '@timestamp',
sourceFilters: '[]',
fields: '[]',
fieldFormatMap: '{}',
typeMeta: '{}',
runtimeFieldMap: '{}',
name,
},
options: { id },
version: 1,
});
return body;
},
async delete({ id }: { id: string }) {
const { body } = await supertest
.post(`/api/content_management/rpc/delete`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
contentTypeId: 'index-pattern',
id,
options: { force: true },
version: 1,
});
return body;
},
};
}

2
x-pack/test_serverless/api_integration/services/index.ts
View file

@ -12,6 +12,7 @@ import { services as svlSharedServices } from '../../shared/services';
import { SvlCommonApiServiceProvider } from './svl_common_api';
import { AlertingApiProvider } from './alerting_api';
import { DataViewApiProvider } from './data_view_api';
export const services = {
...xpackApiIntegrationServices,
@ -19,6 +20,7 @@ export const services = {
svlCommonApi: SvlCommonApiServiceProvider,
alertingApi: AlertingApiProvider,
dataViewApi: DataViewApiProvider,
};
export type InheritedFtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

16
x-pack/test_serverless/api_integration/test_suites/observability/cases/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Cases', function () {
loadTestFile(require.resolve('./get_case'));
loadTestFile(require.resolve('./find_cases'));
loadTestFile(require.resolve('./post_case'));
});
}

26
x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
import { services } from './apm_api_integration/common/services';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'oblt',
junit: {
reportName: 'Serverless Observability Feature Flags API Integration Tests',
},
suiteTags: { exclude: ['skipSvlOblt'] },
services,
// add feature flags
kbnServerArgs: ['--xpack.observability.unsafe.thresholdRule.enabled=true'],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

2
x-pack/test_serverless/api_integration/test_suites/observability/fleet.ts → x-pack/test_serverless/api_integration/test_suites/observability/fleet/fleet.ts
View file

@ -6,7 +6,7 @@
*/
import expect from 'expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');

71
x-pack/test_serverless/api_integration/test_suites/observability/helpers/alerting_api_helper.ts
View file

@ -1,71 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MetricThresholdParams } from '@kbn/infra-plugin/common/alerting/metrics';
import { ThresholdParams } from '@kbn/observability-plugin/common/threshold_rule/types';
import type { SuperTest, Test } from 'supertest';
export async function createIndexConnector({
supertest,
name,
indexName,
}: {
supertest: SuperTest<Test>;
name: string;
indexName: string;
}) {
const { body } = await supertest
.post(`/api/actions/connector`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
name,
config: {
index: indexName,
refresh: true,
},
connector_type_id: '.index',
});
return body.id as string;
}
export async function createRule({
supertest,
name,
ruleTypeId,
params,
actions = [],
tags = [],
schedule,
consumer,
}: {
supertest: SuperTest<Test>;
ruleTypeId: string;
name: string;
params: MetricThresholdParams | ThresholdParams;
actions?: any[];
tags?: any[];
schedule?: { interval: string };
consumer: string;
}) {
const { body } = await supertest
.post(`/api/alerting/rule`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
params,
consumer,
schedule: schedule || {
interval: '5m',
},
tags,
name,
rule_type_id: ruleTypeId,
actions,
});
return body;
}

90
x-pack/test_serverless/api_integration/test_suites/observability/helpers/alerting_wait_for_helpers.ts
View file

@ -1,90 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import pRetry from 'p-retry';
import type SuperTest from 'supertest';
import type { Client } from '@elastic/elasticsearch';
import type {
AggregationsAggregate,
SearchResponse,
} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
export async function waitForRuleStatus({
id,
expectedStatus,
supertest,
}: {
id: string;
expectedStatus: string;
supertest: SuperTest.SuperTest<SuperTest.Test>;
}): Promise<Record<string, any>> {
return pRetry(
async () => {
const response = await supertest
.get(`/api/alerting/rule/${id}`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo');
const { execution_status: executionStatus } = response.body || {};
const { status } = executionStatus || {};
if (status !== expectedStatus) {
throw new Error(`waitForStatus(${expectedStatus}): got ${status}`);
}
return executionStatus;
},
{ retries: 10 }
);
}
export async function waitForDocumentInIndex<T>({
esClient,
indexName,
}: {
esClient: Client;
indexName: string;
}): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
return pRetry(
async () => {
const response = await esClient.search<T>({ index: indexName });
if (response.hits.hits.length === 0) {
throw new Error('No hits found');
}
return response;
},
{ retries: 10 }
);
}
export async function waitForAlertInIndex<T>({
esClient,
indexName,
ruleId,
}: {
esClient: Client;
indexName: string;
ruleId: string;
}): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
return pRetry(
async () => {
const response = await esClient.search<T>({
index: indexName,
body: {
query: {
term: {
'kibana.alert.rule.uuid': ruleId,
},
},
},
});
if (response.hits.hits.length === 0) {
throw new Error('No hits found');
}
return response;
},
{ retries: 10 }
);
}

61
x-pack/test_serverless/api_integration/test_suites/observability/helpers/data_view.ts
View file

@ -1,61 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SuperTest, Test } from 'supertest';
export const createDataView = async ({
supertest,
id,
name,
title,
}: {
supertest: SuperTest<Test>;
id: string;
name: string;
title: string;
}) => {
const { body } = await supertest
.post(`/api/content_management/rpc/create`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
contentTypeId: 'index-pattern',
data: {
fieldAttrs: '{}',
title,
timeFieldName: '@timestamp',
sourceFilters: '[]',
fields: '[]',
fieldFormatMap: '{}',
typeMeta: '{}',
runtimeFieldMap: '{}',
name,
},
options: { id },
version: 1,
});
return body;
};
export const deleteDataView = async ({
supertest,
id,
}: {
supertest: SuperTest<Test>;
id: string;
}) => {
const { body } = await supertest
.post(`/api/content_management/rpc/delete`)
.set('kbn-xsrf', 'foo')
.set('x-elastic-internal-origin', 'foo')
.send({
contentTypeId: 'index-pattern',
id,
options: { force: true },
version: 1,
});
return body;
};

14
x-pack/test_serverless/api_integration/test_suites/observability/index.feature_flags.ts Normal file
View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Serverless observability API - feature flags', function () {
loadTestFile(require.resolve('./threshold_rule'));
});
}

15
x-pack/test_serverless/api_integration/test_suites/observability/index.ts
View file

@ -8,17 +8,10 @@
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless observability API', function () {
loadTestFile(require.resolve('./fleet'));
loadTestFile(require.resolve('./snapshot_telemetry'));
describe('Serverless observability API', function () {
loadTestFile(require.resolve('./fleet/fleet'));
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./apm_api_integration/feature_flags.ts'));
loadTestFile(require.resolve('./threshold_rule/avg_pct_fired'));
loadTestFile(require.resolve('./threshold_rule/avg_pct_no_data'));
loadTestFile(require.resolve('./threshold_rule/documents_count_fired'));
loadTestFile(require.resolve('./threshold_rule/custom_eq_avg_bytes_fired'));
loadTestFile(require.resolve('./threshold_rule/group_by_fired'));
loadTestFile(require.resolve('./cases/post_case'));
loadTestFile(require.resolve('./cases/find_cases'));
loadTestFile(require.resolve('./cases/get_case'));
loadTestFile(require.resolve('./cases'));
});
}

4
x-pack/test_serverless/api_integration/test_suites/observability/snapshot_telemetry.ts → x-pack/test_serverless/api_integration/test_suites/observability/telemetry/snapshot_telemetry.ts
View file

@ -12,8 +12,8 @@ import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/sch
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import { assertTelemetryPayload } from '@kbn/telemetry-tools';
import { FtrProviderContext } from '../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
import { FtrProviderContext } from '../../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../../test/api_integration/services/usage_api';
export default function ({ getService }: FtrProviderContext) {
const usageApi = getService('usageAPI');

15
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_fired.ts
View file

@ -11,14 +11,13 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/thr
import expect from '@kbn/expect';
import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/common/constants';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
import { createDataView, deleteDataView } from '../helpers/data_view';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
const supertest = getService('supertest');
const esDeleteAllIndices = getService('esDeleteAllIndices');
const alertingApi = getService('alertingApi');
const dataViewApi = getService('dataViewApi');
const logger = getService('log');
describe('Threshold rule - AVG - PCT - FIRED', () => {
@ -31,8 +30,7 @@ export default function ({ getService }: FtrProviderContext) {
before(async () => {
infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
await createDataView({
supertest,
await dataViewApi.create({
name: 'metrics-fake_hosts',
id: DATA_VIEW_ID,
title: 'metrics-fake_hosts',
@ -56,8 +54,7 @@ export default function ({ getService }: FtrProviderContext) {
index: '.kibana-event-log-*',
query: { term: { 'kibana.alert.rule.consumer': 'alerts' } },
});
await deleteDataView({
supertest,
await dataViewApi.delete({
id: DATA_VIEW_ID,
});
await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
@ -66,14 +63,12 @@ export default function ({ getService }: FtrProviderContext) {
describe('Rule creation', () => {
it('creates rule successfully', async () => {
actionId = await createIndexConnector({
supertest,
actionId = await alertingApi.createIndexConnector({
name: 'Index Connector: Threshold API test',
indexName: ALERT_ACTION_INDEX,
});
const createdRule = await createRule({
supertest,
const createdRule = await alertingApi.createRule({
tags: ['observability'],
consumer: 'alerts',
name: 'Threshold rule',

15
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/avg_pct_no_data.ts
View file

@ -10,13 +10,12 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/thr
import expect from '@kbn/expect';
import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/common/constants';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
import { createDataView, deleteDataView } from '../helpers/data_view';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
const supertest = getService('supertest');
const alertingApi = getService('alertingApi');
const dataViewApi = getService('dataViewApi');
describe('Threshold rule - AVG - PCT - NoData', () => {
const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@ -26,8 +25,7 @@ export default function ({ getService }: FtrProviderContext) {
let ruleId: string;
before(async () => {
await createDataView({
supertest,
await dataViewApi.create({
name: 'no-data-pattern',
id: DATA_VIEW_ID,
title: 'no-data-pattern',
@ -51,22 +49,19 @@ export default function ({ getService }: FtrProviderContext) {
index: '.kibana-event-log-*',
query: { term: { 'kibana.alert.rule.consumer': 'alerts' } },
});
await deleteDataView({
supertest,
await dataViewApi.delete({
id: DATA_VIEW_ID,
});
});
describe('Rule creation', () => {
it('creates rule successfully', async () => {
actionId = await createIndexConnector({
supertest,
actionId = await alertingApi.createIndexConnector({
name: 'Index Connector: Threshold API test',
indexName: ALERT_ACTION_INDEX,
});
const createdRule = await createRule({
supertest,
const createdRule = await alertingApi.createRule({
tags: ['observability'],
consumer: 'alerts',
name: 'Threshold rule',

15
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/custom_eq_avg_bytes_fired.ts
View file

@ -17,8 +17,6 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/thr
import expect from '@kbn/expect';
import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/common/constants';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
import { createDataView, deleteDataView } from '../helpers/data_view';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
@ -26,6 +24,7 @@ export default function ({ getService }: FtrProviderContext) {
const esDeleteAllIndices = getService('esDeleteAllIndices');
const logger = getService('log');
const alertingApi = getService('alertingApi');
const dataViewApi = getService('dataViewApi');
describe('Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => {
const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@ -37,8 +36,7 @@ export default function ({ getService }: FtrProviderContext) {
before(async () => {
infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
await createDataView({
supertest,
await dataViewApi.create({
name: 'metrics-fake_hosts',
id: DATA_VIEW_ID,
title: 'metrics-fake_hosts',
@ -62,8 +60,7 @@ export default function ({ getService }: FtrProviderContext) {
index: '.kibana-event-log-*',
query: { term: { 'kibana.alert.rule.consumer': 'alerts' } },
});
await deleteDataView({
supertest,
await dataViewApi.delete({
id: DATA_VIEW_ID,
});
await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
@ -72,14 +69,12 @@ export default function ({ getService }: FtrProviderContext) {
describe('Rule creation', () => {
it('creates rule successfully', async () => {
actionId = await createIndexConnector({
supertest,
actionId = await alertingApi.createIndexConnector({
name: 'Index Connector: Threshold API test',
indexName: ALERT_ACTION_INDEX,
});
const createdRule = await createRule({
supertest,
const createdRule = await alertingApi.createRule({
tags: ['observability'],
consumer: 'alerts',
name: 'Threshold rule',

15
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/documents_count_fired.ts
View file

@ -11,8 +11,6 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/thr
import expect from '@kbn/expect';
import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/common/constants';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
import { createDataView, deleteDataView } from '../helpers/data_view';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
@ -20,6 +18,7 @@ export default function ({ getService }: FtrProviderContext) {
const esDeleteAllIndices = getService('esDeleteAllIndices');
const logger = getService('log');
const alertingApi = getService('alertingApi');
const dataViewApi = getService('dataViewApi');
describe('Threshold rule - DOCUMENTS_COUNT - FIRED', () => {
const THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
@ -31,8 +30,7 @@ export default function ({ getService }: FtrProviderContext) {
before(async () => {
infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
await createDataView({
supertest,
await dataViewApi.create({
name: 'metrics-fake_hosts',
id: DATA_VIEW_ID,
title: 'metrics-fake_hosts',
@ -56,8 +54,7 @@ export default function ({ getService }: FtrProviderContext) {
index: '.kibana-event-log-*',
query: { term: { 'kibana.alert.rule.consumer': 'alerts' } },
});
await deleteDataView({
supertest,
await dataViewApi.delete({
id: DATA_VIEW_ID,
});
await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
@ -66,14 +63,12 @@ export default function ({ getService }: FtrProviderContext) {
describe('Rule creation', () => {
it('creates rule successfully', async () => {
actionId = await createIndexConnector({
supertest,
actionId = await alertingApi.createIndexConnector({
name: 'Index Connector: Threshold API test',
indexName: ALERT_ACTION_INDEX,
});
const createdRule = await createRule({
supertest,
const createdRule = await alertingApi.createRule({
tags: ['observability'],
consumer: 'alerts',
name: 'Threshold rule',

19
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/group_by_fired.ts
View file

@ -11,6 +11,7 @@
* 2.0.
*/
import { kbnTestConfig } from '@kbn/test';
import moment from 'moment';
import { cleanup, generate } from '@kbn/infra-forge';
import { Aggregators, Comparator } from '@kbn/observability-plugin/common/threshold_rule/types';
@ -18,8 +19,6 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/thr
import expect from '@kbn/expect';
import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/observability-plugin/common/constants';
import { FtrProviderContext } from '../../../ftr_provider_context';
import { createIndexConnector, createRule } from '../helpers/alerting_api_helper';
import { createDataView, deleteDataView } from '../helpers/data_view';
export default function ({ getService }: FtrProviderContext) {
const esClient = getService('es');
@ -27,6 +26,7 @@ export default function ({ getService }: FtrProviderContext) {
const esDeleteAllIndices = getService('esDeleteAllIndices');
const logger = getService('log');
const alertingApi = getService('alertingApi');
const dataViewApi = getService('dataViewApi');
let alertId: string;
let startedAt: string;
@ -40,8 +40,7 @@ export default function ({ getService }: FtrProviderContext) {
before(async () => {
infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
await createDataView({
supertest,
await dataViewApi.create({
name: 'metrics-fake_hosts',
id: DATA_VIEW_ID,
title: 'metrics-fake_hosts',
@ -65,8 +64,7 @@ export default function ({ getService }: FtrProviderContext) {
index: '.kibana-event-log-*',
query: { term: { 'kibana.alert.rule.consumer': 'alerts' } },
});
await deleteDataView({
supertest,
await dataViewApi.delete({
id: DATA_VIEW_ID,
});
await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
@ -75,14 +73,12 @@ export default function ({ getService }: FtrProviderContext) {
describe('Rule creation', () => {
it('creates rule successfully', async () => {
actionId = await createIndexConnector({
supertest,
actionId = await alertingApi.createIndexConnector({
name: 'Index Connector: Threshold API test',
indexName: ALERT_ACTION_INDEX,
});
const createdRule = await createRule({
supertest,
const createdRule = await alertingApi.createRule({
tags: ['observability'],
consumer: 'alerts',
name: 'Threshold rule',
@ -217,10 +213,11 @@ export default function ({ getService }: FtrProviderContext) {
}>({
indexName: ALERT_ACTION_INDEX,
});
const { protocol, hostname, port } = kbnTestConfig.getUrlParts();
expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.threshold');
expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
`https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
`${protocol}://${hostname}:${port}/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
);
expect(resp.hits.hits[0]._source?.reason).eql(
'Custom equation is 0.8 in the last 1 min for host-0. Alert when >= 0.2.'

18
x-pack/test_serverless/api_integration/test_suites/observability/threshold_rule/index.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Threshold Rule', function () {
loadTestFile(require.resolve('./avg_pct_fired'));
loadTestFile(require.resolve('./avg_pct_no_data'));
loadTestFile(require.resolve('./documents_count_fired'));
loadTestFile(require.resolve('./custom_eq_avg_bytes_fired'));
loadTestFile(require.resolve('./group_by_fired'));
});
}

24
x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'es',
junit: {
reportName: 'Serverless Search Feature Flags API Integration Tests',
},
suiteTags: { exclude: ['skipSvlSearch'] },
// add feature flags
kbnServerArgs: [],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

13
x-pack/test_serverless/api_integration/test_suites/search/index.feature_flags.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export default function () {
describe('Serverless search API - feature flags', function () {
// add tests that require feature flags, defined in config.feature_flags.ts
// loadTestFile(require.resolve(<path_to_test_file>));
});
}

6
x-pack/test_serverless/api_integration/test_suites/search/index.ts
View file

@ -8,9 +8,9 @@
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless search API', function () {
loadTestFile(require.resolve('./snapshot_telemetry'));
loadTestFile(require.resolve('./cases/post_case'));
describe('Serverless search API', function () {
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./cases/find_cases'));
loadTestFile(require.resolve('./cases/post_case'));
});
}

4
x-pack/test_serverless/api_integration/test_suites/search/snapshot_telemetry.ts → x-pack/test_serverless/api_integration/test_suites/search/telemetry/snapshot_telemetry.ts
View file

@ -12,8 +12,8 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import type { FtrProviderContext } from '../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
import type { FtrProviderContext } from '../../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../../test/api_integration/services/usage_api';
export default function ({ getService }: FtrProviderContext) {
const usageApi = getService('usageAPI');

16
x-pack/test_serverless/api_integration/test_suites/security/cases/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('Cases', function () {
loadTestFile(require.resolve('./get_case'));
loadTestFile(require.resolve('./find_cases'));
loadTestFile(require.resolve('./post_case'));
});
}

24
x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'security',
junit: {
reportName: 'Serverless Security Feature Flags API Integration Tests',
},
suiteTags: { exclude: ['skipSvlSec'] },
// add feature flags
kbnServerArgs: [],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

2
x-pack/test_serverless/api_integration/test_suites/security/fleet.ts → x-pack/test_serverless/api_integration/test_suites/security/fleet/fleet.ts
View file

@ -6,7 +6,7 @@
*/
import expect from 'expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
const svlCommonApi = getService('svlCommonApi');

13
x-pack/test_serverless/api_integration/test_suites/security/index.feature_flags.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export default function () {
describe('Serverless security API - feature flags', function () {
// add tests that require feature flags, defined in config.feature_flags.ts
// loadTestFile(require.resolve(<path_to_test_file>));
});
}

10
x-pack/test_serverless/api_integration/test_suites/security/index.ts
View file

@ -8,11 +8,9 @@
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('serverless security API', function () {
loadTestFile(require.resolve('./fleet'));
loadTestFile(require.resolve('./snapshot_telemetry'));
loadTestFile(require.resolve('./cases/post_case'));
loadTestFile(require.resolve('./cases/find_cases'));
loadTestFile(require.resolve('./cases/get_case'));
describe('Serverless security API', function () {
loadTestFile(require.resolve('./telemetry/snapshot_telemetry'));
loadTestFile(require.resolve('./fleet/fleet'));
loadTestFile(require.resolve('./cases'));
});
}

4
x-pack/test_serverless/api_integration/test_suites/security/snapshot_telemetry.ts → x-pack/test_serverless/api_integration/test_suites/security/telemetry/snapshot_telemetry.ts
View file

@ -12,8 +12,8 @@ import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/sch
import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
import { assertTelemetryPayload } from '@kbn/telemetry-tools';
import { FtrProviderContext } from '../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../test/api_integration/services/usage_api';
import { FtrProviderContext } from '../../../ftr_provider_context';
import type { UsageStatsPayloadTestFriendly } from '../../../../../test/api_integration/services/usage_api';
export default function ({ getService }: FtrProviderContext) {
const usageApi = getService('usageAPI');

24
x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'oblt',
junit: {
reportName: 'Serverless Observability Feature Flags Functional Tests',
},
suiteTags: { exclude: ['skipSvlOblt'] },
// add feature flags
kbnServerArgs: [],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

13
x-pack/test_serverless/functional/test_suites/observability/index.feature_flags.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export default function () {
describe('serverless observability UI - feature flags', function () {
// add tests that require feature flags, defined in config.feature_flags.ts
// loadTestFile(require.resolve(<path_to_test_file>));
});
}

24
x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'es',
junit: {
reportName: 'Serverless Search Feature Flags Functional Tests',
},
suiteTags: { exclude: ['skipSvlSearch'] },
// add feature flags
kbnServerArgs: [],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

13
x-pack/test_serverless/functional/test_suites/search/index.feature_flags.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export default function () {
describe('serverless search UI - feature flags', function () {
// add tests that require feature flags, defined in config.feature_flags.ts
// loadTestFile(require.resolve(<path_to_test_file>));
});
}

24
x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createTestConfig } from '../../config.base';
/**
* Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
* This tests most likely will fail on default MKI project
*/
export default createTestConfig({
serverlessProject: 'security',
junit: {
reportName: 'Serverless Security Feature Flags Functional Tests',
},
suiteTags: { exclude: ['skipSvlSec'] },
// add feature flags
kbnServerArgs: [],
// load tests in the index file
testFiles: [require.resolve('./index.feature_flags.ts')],
});

13
x-pack/test_serverless/functional/test_suites/security/index.feature_flags.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export default function () {
describe('serverless security UI - feature flags', function () {
// add tests that require feature flags, defined in config.feature_flags.ts
// loadTestFile(require.resolve(<path_to_test_file>));
});
}

1
x-pack/test_serverless/shared/config.base.ts
View file

@ -61,6 +61,7 @@ export default async () => {
},
])}`,
'--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"',
`--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`,
],
},

1
x-pack/test_serverless/shared/types/index.ts
View file

@ -9,6 +9,7 @@ import { InheritedServices } from '../../api_integration/services';
export interface CreateTestConfigOptions {
serverlessProject: 'es' | 'oblt' | 'security';
kbnServerArgs?: string[];
testFiles: string[];
junit: { reportName: string };
suiteTags?: { include?: string[]; exclude?: string[] };