[8.0] [Uptime] Update functional test directory to use a pinned version of package registry via docker (#117736) (#123823)

* [Uptime] Update functional test directory to use a pinned version of package registry via docker (#117736)

* update functional test directory to use a pinned version of package registry via docker

* remove console log

* adjust config

* skip synthetics tests if no docker image

* remove extra configs

* move synthetics tests to a different directory

* update tests

* update tests

* remove duplicate tests

* update helpers

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
# Conflicts:
#	x-pack/test/functional_synthetics/apps/uptime/synthetics_integration.ts

* Update x-pack/test/functional_synthetics/apps/uptime/synthetics_integration.ts

x-pack/plugins/uptime/public/components/fleet_package/browser/throttling_fields.tsx

@@ -120,7 +120,6 @@ export const ThrottlingFields = memo<Props>(({ validate }) => {
         }
         labelAppend={<OptionalLabel />}
         isInvalid={!!validate[ConfigKey.LATENCY]?.(fields)}
-        data-test-subj="syntheticsBrowserLatency"
         error={
           <FormattedMessage
             id="xpack.uptime.createPackagePolicy.stepConfigure.browserAdvancedSettings.throttling.latency.error"
@@ -137,6 +136,7 @@ export const ThrottlingFields = memo<Props>(({ validate }) => {
               configKey: ConfigKey.LATENCY,
             })
           }
+          data-test-subj="syntheticsBrowserLatency"
           append={
             <EuiText size="xs">
               <strong>ms</strong>

x-pack/scripts/functional_tests.js

@@ -24,6 +24,7 @@ const alwaysImportedTests = [
   require.resolve('../test/saved_object_tagging/functional/config.ts'),
   require.resolve('../test/usage_collection/config.ts'),
   require.resolve('../test/fleet_functional/config.ts'),
+  require.resolve('../test/functional_synthetics/config.js'),
 ];
 const onlyNotInCoverageTests = [
   require.resolve('../test/api_integration/config_security_basic.ts'),

x-pack/test/functional/apps/uptime/index.ts

@@ -59,7 +59,6 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => {
       loadTestFile(require.resolve('./locations'));
       loadTestFile(require.resolve('./settings'));
       loadTestFile(require.resolve('./certificates'));
-      loadTestFile(require.resolve('./synthetics_integration'));
     });
 
     describe('with generated data but no data reset', () => {

x-pack/test/functional/config.js

@@ -10,6 +10,13 @@ import { resolve } from 'path';
 import { services } from './services';
 import { pageObjects } from './page_objects';
 
+// Docker image to use for Fleet API integration tests.
+// This hash comes from the latest successful build of the Snapshot Distribution of the Package Registry, for
+// example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1.
+// It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry.
+export const dockerImage =
+  'docker.elastic.co/package-registry/distribution:ffcbe0ba25b9bae09a671249cbb1b25af0aa1994';
+
 // the default export of config files must be a config provider
 // that returns an object with the projects config values
 export default async function ({ readConfigFile }) {
@@ -84,7 +91,7 @@ export default async function ({ readConfigFile }) {
         '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions
         '--xpack.encryptedSavedObjects.encryptionKey="DkdXazszSCYexXqz4YktBGHCRkV6hyNK"',
         '--xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled=true',
-        '--savedObjects.maxImportPayloadBytes=10485760', // for OSS test management/_import_objects
+        '--savedObjects.maxImportPayloadBytes=10485760', // for OSS test management/_import_objects,
       ],
     },
     uiSettings: {
@@ -484,7 +491,7 @@ export default async function ({ readConfigFile }) {
           },
         },
 
-        //Kibana feature privilege isn't specific to advancedSetting. It can be anything. https://github.com/elastic/kibana/issues/35965
+        // Kibana feature privilege isn't specific to advancedSetting. It can be anything. https://github.com/elastic/kibana/issues/35965
         test_api_keys: {
           elasticsearch: {
             cluster: ['manage_security', 'manage_api_key'],

x-pack/test/functional/fixtures/package_registry_config.yml

@@ -0,0 +1,4 @@
+package_paths:
+  - /packages/production
+  - /packages/staging
+  - /packages/snapshot

x-pack/test/functional/page_objects/index.ts

@@ -24,7 +24,6 @@ import { StatusPageObject } from './status_page';
 import { UpgradeAssistantPageObject } from './upgrade_assistant_page';
 import { RollupPageObject } from './rollup_page';
 import { UptimePageObject } from './uptime_page';
-import { SyntheticsIntegrationPageProvider } from './synthetics_integration_page';
 import { ApiKeysPageProvider } from './api_keys_page';
 import { LicenseManagementPageProvider } from './license_management_page';
 import { IndexManagementPageProvider } from './index_management_page';
@@ -67,7 +66,6 @@ export const pageObjects = {
   statusPage: StatusPageObject,
   upgradeAssistant: UpgradeAssistantPageObject,
   uptime: UptimePageObject,
-  syntheticsIntegration: SyntheticsIntegrationPageProvider,
   rollup: RollupPageObject,
   apiKeys: ApiKeysPageProvider,
   licenseManagement: LicenseManagementPageProvider,

x-pack/test/functional/services/uptime/uptime.ts

@@ -15,7 +15,6 @@ import { UptimeAlertsProvider } from './alerts';
 import { UptimeMLAnomalyProvider } from './ml_anomaly';
 import { UptimeCertProvider } from './certificates';
 import { UptimeOverviewProvider } from './overview';
-import { SyntheticsPackageProvider } from './synthetics_package';
 
 export function UptimeProvider(context: FtrProviderContext) {
   const common = UptimeCommonProvider(context);
@@ -26,7 +25,6 @@ export function UptimeProvider(context: FtrProviderContext) {
   const ml = UptimeMLAnomalyProvider(context);
   const cert = UptimeCertProvider(context);
   const overview = UptimeOverviewProvider(context);
-  const syntheticsPackage = SyntheticsPackageProvider(context);
 
   return {
     common,
@@ -37,6 +35,5 @@ export function UptimeProvider(context: FtrProviderContext) {
     ml,
     cert,
     overview,
-    syntheticsPackage,
   };
 }

x-pack/test/functional_synthetics/README.md

@@ -0,0 +1,3 @@
+# Kibana Functional Testing
+
+See our [Functional Testing Guide](https://www.elastic.co/guide/en/kibana/current/development-tests.html#development-functional-tests)

x-pack/test/functional_synthetics/apps/uptime/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default ({ loadTestFile, getService }: FtrProviderContext) => {
+  describe('Uptime app', function () {
+    describe('with generated data', () => {
+      loadTestFile(require.resolve('./synthetics_integration'));
+    });
+  });
+};

x-pack/test/functional_synthetics/apps/uptime/synthetics_integration.ts

@@ -8,8 +8,10 @@
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { FullAgentPolicy } from '../../../../plugins/fleet/common';
+import { skipIfNoDockerRegistry } from '../../helpers';
 
-export default function ({ getPageObjects, getService }: FtrProviderContext) {
+export default function (providerContext: FtrProviderContext) {
+  const { getPageObjects, getService } = providerContext;
   const monitorName = 'Sample Synthetics integration';
 
   const uptimePage = getPageObjects(['syntheticsIntegration']);
@@ -129,9 +131,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
     type: `synthetics/${monitorType}`,
     use_output: 'default',
   });
-
   describe('When on the Synthetics Integration Policy Create Page', function () {
-    this.tags(['ciGroup10']);
+    skipIfNoDockerRegistry(providerContext);
     const basicConfig = {
       name: monitorName,
       apmServiceName: 'Sample APM Service',
@@ -172,8 +173,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       });
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/103390
-    describe.skip('create new policy', () => {
+    describe('create new policy', () => {
       let version: string;
 
       beforeEach(async () => {
@@ -558,6 +558,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
               schedule: '@every 3m',
               timeout: '16s',
               tags: [config.tags],
+              throttling: '5d/3u/20l',
               'service.name': config.apmServiceName,
               'source.zip_url.url': config.zipUrl,
               'source.zip_url.folder': config.folder,
@@ -607,6 +608,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
               schedule: '@every 3m',
               timeout: '16s',
               tags: [config.tags],
+              throttling: '5d/3u/20l',
               'service.name': config.apmServiceName,
               'source.inline.script': config.inlineScript,
               __ui: {
@@ -665,6 +667,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
               schedule: '@every 3m',
               timeout: '16s',
               tags: [config.tags],
+              throttling: '1337d/1338u/1339l',
               'service.name': config.apmServiceName,
               'source.zip_url.url': config.zipUrl,
               'source.zip_url.folder': config.folder,
@@ -672,11 +675,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
               'source.zip_url.password': config.password,
               params: JSON.parse(config.params),
               synthetics_args: [advancedConfig.syntheticsArgs],
-              'throttling.is_enabled': advancedConfig.isThrottlingEnabled,
-              'throttling.download_speed': advancedConfig.downloadSpeed,
-              'throttling.upload_speed': advancedConfig.uploadSpeed,
-              'throttling.latency': advancedConfig.latency,
-              'throttling.config': `${advancedConfig.downloadSpeed}d/${advancedConfig.uploadSpeed}u/${advancedConfig.latency}l`,
               __ui: {
                 is_tls_enabled: false,
                 is_zip_url_tls_enabled: false,
@@ -740,11 +738,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
               'source.zip_url.password': config.password,
               params: JSON.parse(config.params),
               synthetics_args: [advancedConfig.syntheticsArgs],
-              'throttling.is_enabled': advancedConfig.isThrottlingEnabled,
-              'throttling.download_speed': advancedConfig.downloadSpeed,
-              'throttling.upload_speed': advancedConfig.uploadSpeed,
-              'throttling.latency': advancedConfig.latency,
-              'throttling.config': 'false',
+              throttling: false,
               __ui: {
                 is_tls_enabled: false,
                 is_zip_url_tls_enabled: false,

x-pack/test/functional_synthetics/config.js

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import path, { resolve } from 'path';
+
+import { defineDockerServersConfig } from '@kbn/test';
+
+import { services } from './services';
+import { pageObjects } from './page_objects';
+
+// Docker image to use for Fleet API integration tests.
+// This hash comes from the latest successful build of the Snapshot Distribution of the Package Registry, for
+// example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1.
+// It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry that updates Synthetics.
+export const dockerImage =
+  'docker.elastic.co/package-registry/distribution:48202133e7506873aff3cc7c3b1d284158727779';
+
+// the default export of config files must be a config provider
+// that returns an object with the projects config values
+export default async function ({ readConfigFile }) {
+  const registryPort = process.env.FLEET_PACKAGE_REGISTRY_PORT;
+
+  const kibanaCommonConfig = await readConfigFile(
+    require.resolve('../../../test/common/config.js')
+  );
+  const kibanaFunctionalConfig = await readConfigFile(
+    require.resolve('../../../test/functional/config.js')
+  );
+
+  // mount the config file for the package registry as well as
+  // the directory containing additional packages into the container
+  const dockerArgs = [
+    '-v',
+    `${path.join(
+      path.dirname(__filename),
+      './fixtures/package_registry_config.yml'
+    )}:/package-registry/config.yml`,
+  ];
+
+  return {
+    // list paths to the files that contain your plugins tests
+    testFiles: [resolve(__dirname, './apps/uptime')],
+
+    services,
+    pageObjects,
+
+    servers: kibanaFunctionalConfig.get('servers'),
+
+    esTestCluster: {
+      license: 'trial',
+      from: 'snapshot',
+      serverArgs: ['path.repo=/tmp/', 'xpack.security.authc.api_key.enabled=true'],
+    },
+
+    kbnTestServer: {
+      ...kibanaCommonConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...kibanaCommonConfig.get('kbnTestServer.serverArgs'),
+        '--status.allowAnonymous=true',
+        '--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d',
+        '--xpack.maps.showMapsInspectorAdapter=true',
+        '--xpack.maps.preserveDrawingBuffer=true',
+        '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions
+        '--xpack.encryptedSavedObjects.encryptionKey="DkdXazszSCYexXqz4YktBGHCRkV6hyNK"',
+        '--xpack.discoverEnhanced.actions.exploreDataInContextMenu.enabled=true',
+        '--savedObjects.maxImportPayloadBytes=10485760', // for OSS test management/_import_objects,
+        ...(registryPort ? [`--xpack.fleet.registryUrl=http://localhost:${registryPort}`] : []),
+      ],
+    },
+    uiSettings: {
+      defaults: {
+        'accessibility:disableAnimations': true,
+        'dateFormat:tz': 'UTC',
+        'visualization:visualize:legacyPieChartsLibrary': true,
+      },
+    },
+    // the apps section defines the urls that
+    // `PageObjects.common.navigateTo(appKey)` will use.
+    // Merge urls for your plugin with the urls defined in
+    // Kibana's config in order to use this helper
+    apps: {
+      ...kibanaFunctionalConfig.get('apps'),
+      fleet: {
+        pathname: '/app/fleet',
+      },
+    },
+
+    // choose where screenshots should be saved
+    screenshots: {
+      directory: resolve(__dirname, 'screenshots'),
+    },
+
+    junit: {
+      reportName: 'Chrome Elastic Synthetics Integration UI Functional Tests',
+    },
+    dockerServers: defineDockerServersConfig({
+      registry: {
+        enabled: !!registryPort,
+        image: dockerImage,
+        portInContainer: 8080,
+        port: registryPort,
+        args: dockerArgs,
+        waitForLogLine: 'package manifests loaded',
+      },
+    }),
+  };
+}

x-pack/test/functional_synthetics/fixtures/package_registry_config.yml

@@ -0,0 +1,4 @@
+package_paths:
+  - /packages/production
+  - /packages/staging
+  - /packages/snapshot

x-pack/test/functional_synthetics/ftr_provider_context.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
+export class FtrService extends GenericFtrService<FtrProviderContext> {}

x-pack/test/functional_synthetics/helpers.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Context } from 'mocha';
+import { ToolingLog } from '@kbn/dev-utils';
+import { FtrProviderContext } from './ftr_provider_context';
+
+export function warnAndSkipTest(mochaContext: Context, log: ToolingLog) {
+  log.warning(
+    'disabling tests because DockerServers service is not enabled, set FLEET_PACKAGE_REGISTRY_PORT to run them'
+  );
+  mochaContext.skip();
+}
+
+export function skipIfNoDockerRegistry(providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const dockerServers = getService('dockerServers');
+
+  const server = dockerServers.get('registry');
+  const log = getService('log');
+
+  beforeEach(function beforeSetupWithDockerRegistry() {
+    if (!server.enabled) {
+      warnAndSkipTest(this, log);
+    }
+  });
+}

x-pack/test/functional_synthetics/page_objects/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { pageObjects as kibanaFunctionalPageObjects } from '../../../../test/functional/page_objects';
+
+import { SyntheticsIntegrationPageProvider } from './synthetics_integration_page';
+
+// just like services, PageObjects are defined as a map of
+// names to Providers. Merge in Kibana's or pick specific ones
+export const pageObjects = {
+  ...kibanaFunctionalPageObjects,
+  syntheticsIntegration: SyntheticsIntegrationPageProvider,
+};

x-pack/test/functional_synthetics/page_objects/synthetics_integration_page.ts

@@ -451,7 +451,7 @@ export function SyntheticsIntegrationPageProvider({
       await testSubjects.click('syntheticsBrowserAdvancedFieldsAccordion');
 
       const throttleSwitch = await this.findThrottleSwitch();
-      if ((await throttleSwitch.isSelected()) !== isThrottlingEnabled) {
+      if (!isThrottlingEnabled) {
         await throttleSwitch.click();
       }
 

x-pack/test/functional_synthetics/services/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { services as kibanaFunctionalServices } from '../../../../test/functional/services';
+import { services as commonServices } from '../../common/services';
+import { UptimeProvider } from './uptime';
+
+// define the name and providers for services that should be
+// available to your tests. If you don't specify anything here
+// only the built-in services will be available
+export const services = {
+  ...kibanaFunctionalServices,
+  ...commonServices,
+  uptime: UptimeProvider,
+};

x-pack/test/functional_synthetics/services/uptime/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { UptimeProvider } from './uptime';

x-pack/test/functional_synthetics/services/uptime/uptime.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+import { SyntheticsPackageProvider } from './synthetics_package';
+
+export function UptimeProvider(context: FtrProviderContext) {
+  const syntheticsPackage = SyntheticsPackageProvider(context);
+
+  return {
+    syntheticsPackage,
+  };
+}