[kbn/es] capture es debug files (#132355) (#132382)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit 63fd2dd552)

# Conflicts:
#	packages/kbn-test/BUILD.bazel
#	packages/kbn-test/src/es/test_es_cluster.ts

.buildkite/scripts/lifecycle/post_command.sh

@@ -25,6 +25,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/apps/reporting/reports/session/*.pdf'
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
+  buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
 
   echo "--- Run Failed Test Reporter"
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'

packages/kbn-test/BUILD.bazel

@@ -86,12 +86,14 @@ TYPES_DEPS = [
   "@npm//elastic-apm-node",
   "@npm//del",
   "@npm//form-data",
+  "@npm//globby",
   "@npm//jest",
   "@npm//jest-cli",
   "@npm//jest-snapshot",
   "@npm//redux",
   "@npm//rxjs",
   "@npm//xmlbuilder",
+  "@npm//@types/archiver",
   "@npm//@types/chance",
   "@npm//@types/enzyme",
   "@npm//@types/he",
@@ -107,6 +109,7 @@ TYPES_DEPS = [
   "@npm//@types/react-redux",
   "@npm//@types/react-router-dom",
   "@npm//@types/semver",
+  "@npm//@types/uuid",
   "@npm//@types/xml2js",
 ]
 

packages/kbn-test/src/es/test_es_cluster.ts

@@ -9,6 +9,11 @@
 import Path from 'path';
 import { format } from 'url';
 import del from 'del';
+import Uuid from 'uuid';
+import globby from 'globby';
+import createArchiver from 'archiver';
+import Fs from 'fs';
+import { pipeline } from 'stream/promises';
 // @ts-expect-error in js
 import { Cluster } from '@kbn/es';
 import { Client } from '@elastic/elasticsearch';
@@ -53,6 +58,7 @@ export interface ICluster {
   cleanup: () => Promise<void>;
   getClient: () => KibanaClient;
   getHostUrls: () => string[];
+  captureDebugFiles: () => Promise<void>;
 }
 
 export type EsTestCluster<Options extends CreateTestEsClusterOptions = CreateTestEsClusterOptions> =
@@ -282,6 +288,51 @@ export function createTestEsCluster<
       await Promise.all(nodeStopPromises.map(async (stop) => await stop()));
 
       log.info('[es] stopped');
+
+      await this.captureDebugFiles();
+    }
+
+    async captureDebugFiles() {
+      const debugFiles = await globby([`**/hs_err_pid*.log`, `**/replay_pid*.log`, `**/*.hprof`], {
+        cwd: config.installPath,
+        absolute: true,
+      });
+
+      if (!debugFiles.length) {
+        log.info('[es] no debug files found, assuming es did not write any');
+        return;
+      }
+
+      const uuid = Uuid.v4();
+      const debugPath = Path.resolve(KIBANA_ROOT, `data/es_debug_${uuid}.tar.gz`);
+      log.error(`[es] debug files found, archiving install to ${debugPath}`);
+      const archiver = createArchiver('tar', { gzip: true });
+      const promise = pipeline(archiver, Fs.createWriteStream(debugPath));
+
+      const archiveDirname = `es_debug_${uuid}`;
+      for (const name of Fs.readdirSync(config.installPath)) {
+        if (name === 'modules' || name === 'jdk') {
+          // drop these large and unnecessary directories
+          continue;
+        }
+
+        const src = Path.resolve(config.installPath, name);
+        const dest = Path.join(archiveDirname, name);
+        const stat = Fs.statSync(src);
+        if (stat.isDirectory()) {
+          archiver.directory(src, dest);
+        } else {
+          archiver.file(src, { name: dest });
+        }
+      }
+
+      archiver.finalize();
+      await promise;
+
+      // cleanup the captured debug files
+      for (const path of debugFiles) {
+        Fs.rmSync(path, { force: true });
+      }
     }
 
     async cleanup() {