[Fleet] Run install all packages in CI (#136035)

.buildkite/ftr_configs.yml

@@ -55,6 +55,7 @@ disabled:
   - x-pack/test/load/config.ts
   - x-pack/test/plugin_api_perf/config.js
   - x-pack/test/screenshot_creation/config.ts
+  - x-pack/test/fleet_packages/config.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

.buildkite/scripts/steps/fleet/install_all_packages.sh

@@ -2,10 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
+source .buildkite/scripts/steps/functional/common.sh
 
 echo '--- Installing all packages'
-cd x-pack/plugins/fleet
-node scripts/install_all_packages
+
+checks-reporter-with-killswitch "Fleet packages Tests" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --config x-pack/test/fleet_packages/config.ts

x-pack/test/fleet_packages/config.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+
+export const BUNDLED_PACKAGE_DIR = '/tmp/fleet_bundled_packages';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
+
+  return {
+    testFiles: [require.resolve('./tests')],
+    servers: xPackAPITestsConfig.get('servers'),
+    services: xPackAPITestsConfig.get('services'),
+    junit: {
+      reportName: 'X-Pack Fleet packages tests',
+    },
+    esTestCluster: xPackAPITestsConfig.get('esTestCluster'),
+    kbnTestServer: {
+      ...xPackAPITestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+        '--xpack.cloudSecurityPosture.enabled=true',
+        // Enable debug fleet logs by default
+        `--logging.loggers[0].name=plugins.fleet`,
+        `--logging.loggers[0].level=debug`,
+        `--logging.loggers[0].appenders=${JSON.stringify(['default'])}`,
+      ],
+    },
+  };
+}

x-pack/test/fleet_packages/ftr_provider_context.d.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test';
+
+// import { services } from './services';
+
+// export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;
+export type FtrProviderContext = GenericFtrProviderContext<{}, {}>;

x-pack/test/fleet_packages/tests/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Fleet packages test', function () {
+    loadTestFile(require.resolve('./install_all'));
+  });
+}

x-pack/test/fleet_packages/tests/install_all.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../api_integration/ftr_provider_context';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const logger = getService('log');
+
+  function installPackage(
+    name: string,
+    version: string
+  ): Promise<{ name: string; success: boolean; error?: any }> {
+    return supertest
+      .post(`/api/fleet/epm/packages/${name}/${version}`)
+      .set('kbn-xsrf', 'xxx')
+      .send({ force: true })
+      .expect(200)
+      .then(() => {
+        return { name, success: true };
+      })
+      .catch((error) => {
+        return { name, success: false, error };
+      });
+  }
+
+  describe('install all fleet packages', function () {
+    this.timeout(1000 * 60 * 60); // 1 hour
+    it('should work and install all packages', async () => {
+      const {
+        body: { items: packages },
+      } = await supertest.get('/api/fleet/epm/packages').expect(200);
+      const allResults = [];
+      for (const pkg of packages) {
+        const pkgName = `${pkg.name}@${pkg.version}`;
+        const res = await installPackage(pkg.name, pkg.version);
+        allResults.push(res);
+        if (!res.success) {
+          logger.info(`❌ ${pkgName} failed: ${res?.error?.message}`);
+        } else {
+          logger.info(`✅ ${pkgName}`);
+        }
+      }
+      const failedInstall = allResults.filter((res) => res.success === false);
+      if (failedInstall.length) {
+        throw new Error(
+          `Some package installe failed: ${failedInstall.map((res) => res.name).join(', ')}`
+        );
+      }
+    });
+  });
+}