[Ops] Snapshot saved object migrations (#167983)

## Goal
We'd like to increase visibility to when Saved Object migrations and
schema changes are added to serverless releases.

## Plan
- add post-build step to export Saved Object schema snapshot, upload it
to google storage, by commit hash.
- build comparison logic, that when given 2 hashes (e.g.: #current ->
#to-be-released) it can highlight schema changes
- display the output from the comparison where it would make most sense,
ideas:
- prior to release, as a script that can be ran (or a browser-based
inspector)
- prior to release, as a buildkite step, that displays the output, and
waits for a confirmation on it
- during the quality-gating, as an output during the quality-gate run,
to be confirmed by the RM

## Summary
The PR intends to satisfy the first step of the plan, to add a
post-merge step to snapshot SO schema/migration states for later use.
 

Example run:
https://buildkite.com/elastic/kibana-pull-request/builds/165992#018b0583-c575-47f4-bade-7b45f6cf3f4d
The files are not public, they require google cloud access, but could be
accessed by scripts we create.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/on_merge.yml

@@ -270,6 +270,13 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json
+    label: 'Extract Saved Object migration plugin types'
+    agents:
+      queue: n2-4-spot
+    artifact_paths:
+      "target/plugin_so_types_snapshot.json"
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/scripts/steps/archive_so_migration_snapshot.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+SO_MIGRATIONS_SNAPSHOT_FOLDER=kibana-so-types-snapshots
+SNAPSHOT_FILE_PATH="${1:-target/plugin_so_types_snapshot.json}"
+
+echo "--- Creating snapshot of Saved Object migration info"
+node scripts/snapshot_plugin_types --outputPath "$SNAPSHOT_FILE_PATH"
+
+echo "--- Uploading as ${BUILDKITE_COMMIT}.json"
+SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_FOLDER}/${BUILDKITE_COMMIT}.json"
+gsutil cp "$SNAPSHOT_FILE_PATH" "gs://$SNAPSHOT_PATH"
+
+buildkite-agent annotate --context so_migration_snapshot --style success \
+  'Saved Object type snapshot is available at <a href="https://storage.cloud.google.com/'"$SNAPSHOT_PATH"'">'"$SNAPSHOT_PATH"'</a>'
+
+echo "Success!"

packages/kbn-dev-cli-runner/src/run.ts

@@ -24,7 +24,7 @@ export interface RunContext {
   addCleanupTask: (task: CleanupTask) => void;
   flagsReader: FlagsReader;
 }
-export type RunFn = (context: RunContext) => Promise<void> | void;
+export type RunFn<T = void> = (context: RunContext) => Promise<T> | void;
 
 export interface RunOptions {
   usage?: string;
@@ -35,7 +35,7 @@ export interface RunOptions {
   flags?: FlagOptions;
 }
 
-export async function run(fn: RunFn, options: RunOptions = {}) {
+export async function run<T>(fn: RunFn<T>, options: RunOptions = {}): Promise<T | undefined> {
   const flags = getFlags(process.argv.slice(2), options.flags, options.log?.defaultLevel);
   const log = new ToolingLog({
     level: pickLevelFromFlags(flags, {
@@ -66,21 +66,23 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
     return;
   }
 
+  let result: T | undefined;
   try {
     await withProcRunner(log, async (procRunner) => {
-      await fn({
-        log,
-        flags,
-        procRunner,
-        statsMeta: metrics.meta,
-        addCleanupTask: cleanup.add.bind(cleanup),
-        flagsReader: new FlagsReader(flags, {
-          aliases: {
-            ...options.flags?.alias,
-            ...DEFAULT_FLAG_ALIASES,
-          },
-        }),
-      });
+      result =
+        (await fn({
+          log,
+          flags,
+          procRunner,
+          statsMeta: metrics.meta,
+          addCleanupTask: cleanup.add.bind(cleanup),
+          flagsReader: new FlagsReader(flags, {
+            aliases: {
+              ...options.flags?.alias,
+              ...DEFAULT_FLAG_ALIASES,
+            },
+          }),
+        })) || undefined;
     });
   } catch (error) {
     cleanup.execute(error);
@@ -92,4 +94,6 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
   }
 
   await metrics.reportSuccess();
+
+  return result;
 }

scripts/snapshot_plugin_types.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/so_migration/so_migration_cli');

src/dev/so_migration/snapshot_plugin_types.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as cp from 'child_process';
+
+import {
+  extractMigrationInfo,
+  getMigrationHash,
+  SavedObjectTypeMigrationInfo,
+  // TODO: how to resolve this? Where to place this script?
+  // eslint-disable-next-line @kbn/imports/no_boundary_crossing
+} from '@kbn/core-test-helpers-so-type-serializer';
+import {
+  createTestServers,
+  createRootWithCorePlugins,
+  // TODO: how to resolve this? Where to place this script?
+  // eslint-disable-next-line @kbn/imports/no_boundary_crossing
+} from '@kbn/core-test-helpers-kbn-server';
+import { REPO_ROOT } from '@kbn/repo-info';
+import { ToolingLog } from '@kbn/tooling-log';
+
+import { mkdirp } from '../build/lib';
+
+type MigrationInfoRecord = Pick<
+  SavedObjectTypeMigrationInfo,
+  'name' | 'migrationVersions' | 'schemaVersions' | 'modelVersions' | 'mappings'
+> & {
+  hash: string;
+};
+
+type ServerHandles = Awaited<ReturnType<typeof startServers>> | undefined;
+
+/**
+ * Starts up ES & Kibana to extract plugin migration information, and save it to @param outputPath.
+ * @param log Tooling log handed over from CLI runner
+ * @param outputPath Path (absolute or relative to) where the extracted migration info should be saved to in a JSON format.
+ */
+async function takeSnapshot({ log, outputPath }: { log: ToolingLog; outputPath: string }) {
+  let serverHandles: ServerHandles;
+
+  const snapshotOutputPath = path.isAbsolute(outputPath)
+    ? outputPath
+    : path.resolve(REPO_ROOT, outputPath);
+
+  try {
+    serverHandles = await startServers();
+
+    const typeRegistry = serverHandles.coreStart.savedObjects.getTypeRegistry();
+    const allTypes = typeRegistry.getAllTypes();
+
+    const migrationInfoMap = allTypes.reduce((map, type) => {
+      const migrationInfo = extractMigrationInfo(type);
+      map[type.name] = {
+        name: migrationInfo.name,
+        migrationVersions: migrationInfo.migrationVersions,
+        hash: getMigrationHash(type),
+        modelVersions: migrationInfo.modelVersions,
+        schemaVersions: migrationInfo.schemaVersions,
+        mappings: migrationInfo.mappings,
+      };
+      return map;
+    }, {} as Record<string, MigrationInfoRecord>);
+
+    await writeSnapshotFile(snapshotOutputPath, migrationInfoMap);
+    log.info('Snapshot taken!');
+
+    return migrationInfoMap;
+  } finally {
+    log.debug('Shutting down servers');
+    await shutdown(log, serverHandles);
+  }
+}
+
+async function startServers() {
+  const { startES } = createTestServers({
+    adjustTimeout: () => {},
+  });
+
+  const esServer = await startES();
+  const kibanaRoot = createRootWithCorePlugins({}, { oss: false });
+  await kibanaRoot.preboot();
+  await kibanaRoot.setup();
+  const coreStart = await kibanaRoot.start();
+  return { esServer, kibanaRoot, coreStart };
+}
+
+async function writeSnapshotFile(
+  snapshotOutputPath: string,
+  typeDefinitions: Record<string, MigrationInfoRecord>
+) {
+  const timestamp = Date.now();
+  const date = new Date().toISOString();
+  const buildUrl = process.env.BUILDKITE_BUILD_URL;
+  const prId = process.env.BUILDKITE_MESSAGE?.match(/\(#(\d+)\)/)?.[1];
+  const pullRequestUrl = prId ? `https://github.com/elastic/kibana/pulls/${prId}` : null;
+  const kibanaCommitHash = process.env.BUILDKITE_COMMIT || getLocalHash();
+
+  const payload = {
+    meta: {
+      timestamp,
+      date,
+      kibanaCommitHash,
+      buildUrl,
+      pullRequestUrl,
+    },
+    typeDefinitions,
+  };
+
+  await mkdirp(path.dirname(snapshotOutputPath));
+  fs.writeFileSync(snapshotOutputPath, JSON.stringify(payload, null, 2));
+}
+
+async function shutdown(log: ToolingLog, serverHandles: ServerHandles) {
+  if (!serverHandles) {
+    log.debug('No server to terminate.');
+    return;
+  }
+
+  try {
+    await serverHandles.kibanaRoot.shutdown();
+
+    log.info("Kibana's shutdown done!");
+  } catch (ex) {
+    log.error('Error while stopping kibana.');
+    log.error(ex);
+  }
+
+  try {
+    await serverHandles.esServer.stop();
+    log.info('ES Stopped!');
+  } catch (ex) {
+    log.error('Error while stopping ES.');
+    log.error(ex);
+  }
+}
+
+function getLocalHash() {
+  try {
+    const stdout = cp.execSync('git rev-parse HEAD');
+    return stdout.toString().trim();
+  } catch (e) {
+    return null;
+  }
+}
+
+export { takeSnapshot };

src/dev/so_migration/so_migration_cli.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { run } from '@kbn/dev-cli-runner';
+
+import { takeSnapshot } from './snapshot_plugin_types';
+
+const scriptName = process.argv[1].replace(/^.*scripts\//, 'scripts/');
+const DEFAULT_OUTPUT_PATH = 'target/plugin_so_types_snapshot.json';
+
+run(
+  async ({ log, flagsReader, procRunner }) => {
+    const outputPath = flagsReader.getPositionals()[0] || DEFAULT_OUTPUT_PATH;
+
+    const result = await takeSnapshot({ outputPath, log });
+
+    return {
+      outputPath,
+      result,
+      log,
+    };
+  },
+  {
+    usage: [process.argv0, scriptName, '[outputPath]'].join(' '),
+    description: `Takes a snapshot of all Kibana plugin Saved Object migrations' information, in a JSON format.`,
+    flags: {
+      string: ['outputPath'],
+      help: `
+        --outputPath\tA path (absolute or relative to the repo root) where to output the snapshot of the SO migration info (default: ${DEFAULT_OUTPUT_PATH})
+      `,
+    },
+  }
+)
+  .then((success) => {
+    if (success) {
+      success.log.info('Snapshot successfully taken to: ' + success.outputPath);
+    }
+    // Kibana won't stop because some async processes are stuck polling, we need to shut down the process.
+    process.exit(0);
+  })
+  .catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error(err);
+    process.exit(1);
+  });

src/dev/tsconfig.json

@@ -40,5 +40,7 @@
     "@kbn/import-locator",
     "@kbn/config-schema",
     "@kbn/journeys",
+    "@kbn/core-test-helpers-so-type-serializer",
+    "@kbn/core-test-helpers-kbn-server",
   ]
 }