diff --git a/.buildkite/ftr_base_serverless_configs.yml b/.buildkite/ftr_base_serverless_configs.yml index 2ff9ba667846..50393bb00f79 100644 --- a/.buildkite/ftr_base_serverless_configs.yml +++ b/.buildkite/ftr_base_serverless_configs.yml @@ -3,6 +3,7 @@ disabled: # Serverless deployment-agnostic default config for api-integration tests - x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts + - x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts # Serverless base config files - x-pack/test_serverless/api_integration/config.base.ts - x-pack/test_serverless/functional/config.base.ts diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml index e57c0c29bd56..9d0638b38811 100644 --- a/.buildkite/ftr_oblt_serverless_configs.yml +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -39,3 +39,4 @@ enabled: - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.synthetics.serverless.config.ts - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.streams.serverless.config.ts + - x-pack/test/api_integration/deployment_agnostic/feature_flag_configs/serverless/oblt.synthetics.serverless.config.ts diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml index 454f36b246f7..86c5b13ff22a 100644 --- a/.buildkite/ftr_oblt_stateful_configs.yml +++ b/.buildkite/ftr_oblt_stateful_configs.yml @@ -48,3 +48,4 @@ enabled: - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.apm.stateful.config.ts - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.ai_assistant.stateful.config.ts + - x-pack/test/api_integration/deployment_agnostic/feature_flag_configs/stateful/oblt.synthetics.stateful.config.ts diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index 46f302d91687..6b4de20f9664 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -1,6 +1,8 @@ disabled: # Stateful base config for deployment-agnostic tests - x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts + - x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts + # Base config files, only necessary to inform config finding script - src/platform/test/functional/config.base.js - src/platform/test/functional/firefox/config.base.ts diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts index 0195ff79d204..d8061694e7be 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts @@ -12,10 +12,5 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./burn_rate_rule')); loadTestFile(require.resolve('./es_query')); loadTestFile(require.resolve('./custom_threshold')); - - describe('Synthetics Alerting', () => { - loadTestFile(require.resolve('./synthetics/synthetics_default_rule')); - loadTestFile(require.resolve('./synthetics/custom_status_rule')); - }); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts b/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts new file mode 100644 index 000000000000..ba1ada4b14f7 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.serverless.config.base.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + fleetPackageRegistryDockerImage, + FtrConfigProviderContext, + Config, + defineDockerServersConfig, +} from '@kbn/test'; + +import { ScoutTestRunConfigCategory } from '@kbn/scout-info'; +import { ServerlessProjectType } from '@kbn/es'; +import path from 'path'; +import { DeploymentAgnosticCommonServices, services } from '../services'; +import { LOCAL_PRODUCT_DOC_PATH } from './common_paths'; +import { updateKbnServerArguments } from './helpers'; + +interface CreateTestConfigOptions { + serverlessProject: ServerlessProjectType; + esServerArgs?: string[]; + kbnServerArgs?: string[]; + services?: T; + testFiles: string[]; + junit: { reportName: string }; + suiteTags?: { include?: string[]; exclude?: string[] }; +} + +// include settings from elasticsearch controller +// https://github.com/elastic/elasticsearch-controller/blob/main/helm/values.yaml +const esServerArgsFromController = { + es: [], + oblt: [ + 'xpack.apm_data.enabled=true', + // for ML, data frame analytics are not part of this project type + 'xpack.ml.dfa.enabled=false', + ], + security: ['xpack.security.authc.api_key.cache.max_keys=70000'], + chat: [], +}; + +// include settings from kibana controller +// https://github.com/elastic/kibana-controller/blob/main/internal/controllers/kibana/config/config_settings.go +const kbnServerArgsFromController = { + es: [ + // useful for testing (also enabled in MKI QA) + '--coreApp.allowDynamicConfigOverrides=true', + ], + oblt: [ + '--coreApp.allowDynamicConfigOverrides=true', + // defined in MKI control plane + '--xpack.uptime.service.manifestUrl=mockDevUrl', + ], + security: [ + '--coreApp.allowDynamicConfigOverrides=true', + // disable fleet task that writes to metrics.fleet_server.* data streams, impacting functional tests + `--xpack.task_manager.unsafe.exclude_task_types=${JSON.stringify(['Fleet-Metrics-Task'])}`, + ], + chat: [], +}; + +export function createServerlessFeatureFlagTestConfig( + options: CreateTestConfigOptions +) { + return async ({ readConfigFile }: FtrConfigProviderContext): Promise => { + const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml'); + const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`]; + let kbnServerArgs: string[] = []; + + if (options.kbnServerArgs) { + kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs); + } + + /** + * This is used by CI to set the docker registry port + * you can also define this environment variable locally when running tests which + * will spin up a local docker package registry locally for you + * if this is defined it takes precedence over the `packageRegistryOverride` variable + */ + const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT; + + const svlSharedConfig = await readConfigFile( + require.resolve('@kbn/test-suites-serverless/shared/config.base') + ); + + return { + ...svlSharedConfig.getAll(), + + testConfigCategory: ScoutTestRunConfigCategory.API_TEST, + services: { + // services can be customized, but must extend DeploymentAgnosticCommonServices + ...(options.services || services), + }, + dockerServers: defineDockerServersConfig({ + registry: { + enabled: !!dockerRegistryPort, + image: fleetPackageRegistryDockerImage, + portInContainer: 8080, + port: dockerRegistryPort, + args: dockerArgs, + waitForLogLine: 'package manifests loaded', + waitForLogLineTimeoutMs: 60 * 2 * 1000, // 2 minutes + }, + }), + esTestCluster: { + ...svlSharedConfig.get('esTestCluster'), + serverArgs: [ + ...svlSharedConfig.get('esTestCluster.serverArgs'), + // custom native roles are enabled only for search and security projects + ...(options.serverlessProject !== 'oblt' + ? ['xpack.security.authc.native_roles.enabled=true'] + : []), + ...esServerArgsFromController[options.serverlessProject], + ...(options.esServerArgs || []), + ], + }, + kbnTestServer: { + ...svlSharedConfig.get('kbnTestServer'), + serverArgs: [ + ...svlSharedConfig.get('kbnTestServer.serverArgs'), + ...kbnServerArgsFromController[options.serverlessProject], + `--serverless=${options.serverlessProject}`, + ...(options.serverlessProject === 'oblt' + ? [ + // defined in MKI control plane. Necessary for Synthetics app testing + '--xpack.uptime.service.password=test', + '--xpack.uptime.service.username=localKibanaIntegrationTestsUser', + '--xpack.uptime.service.devUrl=mockDevUrl', + '--xpack.uptime.service.manifestUrl=mockDevUrl', + `--xpack.productDocBase.artifactRepositoryUrl=file:///${LOCAL_PRODUCT_DOC_PATH}`, + ] + : []), + ...(dockerRegistryPort + ? [`--xpack.fleet.registryUrl=http://localhost:${dockerRegistryPort}`] + : []), + ...kbnServerArgs, + ], + }, + testFiles: options.testFiles, + junit: options.junit, + suiteTags: { + include: options.suiteTags?.include, + exclude: [...(options.suiteTags?.exclude || []), 'skipServerless'], + }, + }; + }; +} diff --git a/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts b/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts new file mode 100644 index 000000000000..795ed4e6b3c3 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/default_configs/feature_flag.stateful.config.base.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + MOCK_IDP_REALM_NAME, + MOCK_IDP_ENTITY_ID, + MOCK_IDP_ATTRIBUTE_PRINCIPAL, + MOCK_IDP_ATTRIBUTE_ROLES, + MOCK_IDP_ATTRIBUTE_EMAIL, + MOCK_IDP_ATTRIBUTE_NAME, +} from '@kbn/mock-idp-utils'; +import { + fleetPackageRegistryDockerImage, + esTestConfig, + kbnTestConfig, + systemIndicesSuperuser, + FtrConfigProviderContext, + defineDockerServersConfig, +} from '@kbn/test'; +import { ScoutTestRunConfigCategory } from '@kbn/scout-info'; +import path from 'path'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { STATEFUL_ROLES_ROOT_PATH } from '@kbn/es'; +import { DeploymentAgnosticCommonServices, services } from '../services'; +import { AI_ASSISTANT_SNAPSHOT_REPO_PATH, LOCAL_PRODUCT_DOC_PATH } from './common_paths'; +import { updateKbnServerArguments } from './helpers'; + +interface CreateTestConfigOptions { + esServerArgs?: string[]; + kbnServerArgs?: string[]; + services?: T; + testFiles: string[]; + junit: { reportName: string }; + suiteTags?: { include?: string[]; exclude?: string[] }; +} + +export function createStatefulFeatureFlagTestConfig( + options: CreateTestConfigOptions +) { + return async ({ readConfigFile }: FtrConfigProviderContext) => { + // if config is executed on CI or locally + const isRunOnCI = process.env.CI; + + const packageRegistryConfig = path.join(__dirname, './fixtures/package_registry_config.yml'); + const dockerArgs: string[] = ['-v', `${packageRegistryConfig}:/package-registry/config.yml`]; + let kbnServerArgs: string[] = []; + + if (options.kbnServerArgs) { + kbnServerArgs = await updateKbnServerArguments(options.kbnServerArgs); + } + + /** + * This is used by CI to set the docker registry port + * you can also define this environment variable locally when running tests which + * will spin up a local docker package registry locally for you + * if this is defined it takes precedence over the `packageRegistryOverride` variable + */ + const dockerRegistryPort: string | undefined = process.env.FLEET_PACKAGE_REGISTRY_PORT; + + const xPackAPITestsConfig = await readConfigFile(require.resolve('../../config.ts')); + + // TODO: move to kbn-es because currently metadata file has hardcoded entityID and Location + const idpPath = require.resolve( + '@kbn/security-api-integration-helpers/saml/idp_metadata_mock_idp.xml' + ); + const samlIdPPlugin = path.resolve( + __dirname, + '../../../security_api_integration/plugins/saml_provider' + ); + + const servers = { + kibana: { + ...kbnTestConfig.getUrlParts(systemIndicesSuperuser), + protocol: process.env.TEST_CLOUD ? 'https' : 'http', + }, + elasticsearch: { + ...esTestConfig.getUrlParts(), + protocol: process.env.TEST_CLOUD ? 'https' : 'http', + }, + }; + + const kbnUrl = `${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`; + + return { + servers, + testConfigCategory: ScoutTestRunConfigCategory.API_TEST, + dockerServers: defineDockerServersConfig({ + registry: { + enabled: !!dockerRegistryPort, + image: fleetPackageRegistryDockerImage, + portInContainer: 8080, + port: dockerRegistryPort, + args: dockerArgs, + waitForLogLine: 'package manifests loaded', + waitForLogLineTimeoutMs: 60 * 2 * 1000, // 2 minutes + }, + }), + testFiles: options.testFiles, + security: { disableTestUser: true }, + // services can be customized, but must extend DeploymentAgnosticCommonServices + services: options.services || services, + junit: options.junit, + suiteTags: { + include: options.suiteTags?.include, + exclude: [...(options.suiteTags?.exclude || []), 'skipStateful'], + }, + + esTestCluster: { + ...xPackAPITestsConfig.get('esTestCluster'), + serverArgs: [ + ...xPackAPITestsConfig.get('esTestCluster.serverArgs'), + 'xpack.security.authc.token.enabled=true', + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.order=0`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.metadata.path=${idpPath}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.idp.entity_id=${MOCK_IDP_ENTITY_ID}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.entity_id=${kbnUrl}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.acs=${kbnUrl}/api/security/saml/callback`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.sp.logout=${kbnUrl}/logout`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.principal=${MOCK_IDP_ATTRIBUTE_PRINCIPAL}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.groups=${MOCK_IDP_ATTRIBUTE_ROLES}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.name=${MOCK_IDP_ATTRIBUTE_NAME}`, + `xpack.security.authc.realms.saml.${MOCK_IDP_REALM_NAME}.attributes.mail=${MOCK_IDP_ATTRIBUTE_EMAIL}`, + `path.repo=${AI_ASSISTANT_SNAPSHOT_REPO_PATH}`, + ...(options.esServerArgs || []), + ], + files: [ + // Passing the roles that are equivalent to the ones we have in serverless + path.resolve(REPO_ROOT, STATEFUL_ROLES_ROOT_PATH, 'roles.yml'), + ], + }, + kbnTestServer: { + ...xPackAPITestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'), + // if the config is run locally, explicitly enable mock-idp-plugin for UI role selector + ...(isRunOnCI ? [] : ['--mock_idp_plugin.enabled=true']), + // This ensures that we register the Security SAML API endpoints. + // In the real world the SAML config is injected by control plane. + `--plugin-path=${samlIdPPlugin}`, + '--xpack.cloud.id=ftr_fake_cloud_id', + // Ensure that SAML is used as the default authentication method whenever a user navigates to Kibana. In other + // words, Kibana should attempt to authenticate the user using the provider with the lowest order if the Login + // Selector is disabled (replicating Serverless configuration). By declaring `cloud-basic` with a higher + // order, we indicate that basic authentication can still be used, but only if explicitly requested when the + // user navigates to `/login` page directly and enters username and password in the login form. + '--xpack.security.authc.selector.enabled=false', + `--xpack.security.authc.providers=${JSON.stringify({ + saml: { 'cloud-saml-kibana': { order: 0, realm: MOCK_IDP_REALM_NAME } }, + basic: { 'cloud-basic': { order: 1 } }, + })}`, + `--server.publicBaseUrl=${servers.kibana.protocol}://${servers.kibana.hostname}:${servers.kibana.port}`, + '--xpack.uptime.service.password=test', + '--xpack.uptime.service.username=localKibanaIntegrationTestsUser', + '--xpack.uptime.service.devUrl=mockDevUrl', + '--xpack.uptime.service.manifestUrl=mockDevUrl', + '--xpack.observabilityAIAssistant.disableKbSemanticTextMigration=true', + `--xpack.productDocBase.artifactRepositoryUrl=file:///${LOCAL_PRODUCT_DOC_PATH}`, + ...(dockerRegistryPort + ? [`--xpack.fleet.registryUrl=http://localhost:${dockerRegistryPort}`] + : []), + ...kbnServerArgs, + ], + }, + }; + }; +} diff --git a/x-pack/test/api_integration/deployment_agnostic/default_configs/helpers.ts b/x-pack/test/api_integration/deployment_agnostic/default_configs/helpers.ts new file mode 100644 index 000000000000..c16644bf18ba --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/default_configs/helpers.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getPreConfiguredActions } from '../../../alerting_api_integration/common/config'; +import { getTlsWebhookServerUrls } from '../../../alerting_api_integration/common/lib/get_tls_webhook_servers'; + +export const updateKbnServerArguments = async (kbnServerArgs: string[]) => { + const tlsWebhookServers = await getTlsWebhookServerUrls(6300, 6399); + const updatedArgs = kbnServerArgs.map((arg) => + arg.startsWith('--xpack.actions.preconfigured') + ? `--xpack.actions.preconfigured=${getPreConfiguredActions(tlsWebhookServers)}` + : arg + ); + + return updatedArgs; +}; diff --git a/x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts b/x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts index 1b45292060e5..3d817eb52252 100644 --- a/x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts +++ b/x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts @@ -14,8 +14,6 @@ import { import { ScoutTestRunConfigCategory } from '@kbn/scout-info'; import { ServerlessProjectType } from '@kbn/es'; import path from 'path'; -import { getPreConfiguredActions } from '../../../alerting_api_integration/common/config'; -import { getTlsWebhookServerUrls } from '../../../alerting_api_integration/common/lib/get_tls_webhook_servers'; import { DeploymentAgnosticCommonServices, services } from '../services'; import { LOCAL_PRODUCT_DOC_PATH } from './common_paths'; @@ -75,7 +73,6 @@ export function createServerlessTestConfig