[Ops] Create SO migration snapshot comparion script (#168623)

## Summary
Continuation on: #167980 

Now that we have the snapshots created for merges, we can compare the
existing snapshots.
This PR creates a CLI for grabbing and comparing these snapshots.

The CLI looks like this: 
```
  node scripts/snapshot_plugin_types compare --from <rev|filename|url> --to <rev|filename|url> [--outputPath <outputPath>]

  Compares two Saved Object snapshot files based on hashes, filenames or urls.

  Options:
    --from            The source snapshot to compare from. Can be a revision, filename or url.
    --to              The target snapshot to compare to. Can be a revision, filename or url.
    --outputPath      The path to write the comparison report to. If omitted, raw JSON will be output to stdout.
    --verbose, -v      Log verbosely
```

.buildkite/scripts/steps/archive_so_migration_snapshot.sh

@@ -7,7 +7,7 @@ SO_MIGRATIONS_SNAPSHOT_FOLDER=kibana-so-types-snapshots
 SNAPSHOT_FILE_PATH="${1:-target/plugin_so_types_snapshot.json}"
 
 echo "--- Creating snapshot of Saved Object migration info"
-node scripts/snapshot_plugin_types --outputPath "$SNAPSHOT_FILE_PATH"
+node scripts/snapshot_plugin_types snapshot --outputPath "$SNAPSHOT_FILE_PATH"
 
 echo "--- Uploading as ${BUILDKITE_COMMIT}.json"
 SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_FOLDER}/${BUILDKITE_COMMIT}.json"

scripts/snapshot_plugin_types.js

@@ -7,4 +7,31 @@
  */
 
 require('../src/setup_node_env');
-require('../src/dev/so_migration/so_migration_cli');
+
+var command = process.argv[2];
+
+switch (command) {
+  case 'snapshot':
+    require('../src/dev/so_migration/so_migration_snapshot_cli');
+    break;
+  case 'compare':
+    require('../src/dev/so_migration/so_migration_compare_cli');
+    break;
+  default:
+    printHelp();
+    break;
+}
+
+function printHelp() {
+  var scriptName = process.argv[1].replace(/^.*scripts\//, 'scripts/');
+
+  console.log(`
+  Usage: node ${scriptName} <command>
+
+  Commands:
+    snapshot  - Create a snapshot of the current Saved Object types
+    compare   - Compare two snapshots to reveal changes in Saved Object types
+  `);
+
+  process.exit(0);
+}

src/dev/so_migration/compare_snapshots.ts

@@ -0,0 +1,180 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ToolingLog } from '@kbn/tooling-log';
+import { readFile } from 'fs/promises';
+import { existsSync, mkdirSync, writeFileSync } from 'fs';
+import * as os from 'os';
+import { execSync } from 'child_process';
+import { basename, dirname, resolve } from 'path';
+import { MigrationInfoRecord, MigrationSnapshot } from './types';
+import { downloadFile } from './util/download_file';
+
+const SO_MIGRATIONS_BUCKET_PREFIX = 'https://storage.googleapis.com/kibana-so-types-snapshots';
+
+interface CompareSnapshotsParameters {
+  from: string;
+  to: string;
+  log: ToolingLog;
+  outputPath?: string;
+}
+
+async function compareSnapshots({
+  outputPath,
+  log,
+  from,
+  to,
+}: CompareSnapshotsParameters): Promise<any> {
+  validateInput({
+    from,
+    to,
+  });
+
+  const fromSnapshotPath = isFile(from) ? from : await downloadSnapshot(from, log);
+  const toSnapshotPath = isFile(to) ? to : await downloadSnapshot(to, log);
+
+  const fromSnapshot = await loadJson(fromSnapshotPath);
+  const toSnapshot = await loadJson(toSnapshotPath);
+
+  const result = compareSnapshotFiles(fromSnapshot, toSnapshot);
+
+  log.info(
+    `Snapshots compared: ${from} <=> ${to}. ` +
+      `${result.hasChanges ? 'No changes' : 'Changed: ' + result.changed.join(', ')}`
+  );
+
+  if (outputPath) {
+    writeSnapshot(outputPath, result);
+    log.info(`Output written to: ${outputPath}`);
+  } else {
+    log.info(
+      `Emitting result to STDOUT... (Enable '--silent' or '--quiet' to disable non-parseable output)`
+    );
+    // eslint-disable-next-line no-console
+    console.log(JSON.stringify(result, null, 2));
+  }
+
+  return result;
+}
+
+function validateInput({ from, to }: { from: string; to: string }) {
+  if (!from || !to) {
+    throw new Error('"--from" and "--to" must be specified');
+  }
+
+  if (from === to) {
+    throw new Error('"from" and "to" must be different');
+  }
+}
+
+function writeSnapshot(outputPath: string, result: any) {
+  const json = JSON.stringify(result, null, 2);
+  mkdirSync(dirname(outputPath), { recursive: true });
+  writeFileSync(outputPath, json);
+}
+
+function isFile(str: string) {
+  try {
+    return existsSync(str);
+  } catch (err) {
+    return false;
+  }
+}
+
+async function downloadToTemp(googleCloudUrl: string, log: ToolingLog): Promise<string> {
+  const fileName = basename(googleCloudUrl);
+  const filePath = resolve(os.tmpdir(), fileName);
+
+  if (existsSync(filePath)) {
+    log.info('Snapshot already exists at: ' + filePath);
+    return filePath;
+  } else {
+    try {
+      log.info('Downloading snapshot from: ' + googleCloudUrl);
+      await downloadFile(googleCloudUrl, filePath);
+      log.info('File downloaded: ' + filePath);
+      return filePath;
+    } catch (err) {
+      log.error("Couldn't download snapshot from: " + googleCloudUrl);
+      throw err;
+    }
+  }
+}
+
+function downloadSnapshot(gitRev: string, log: ToolingLog): Promise<string> {
+  const fullCommitHash = expandGitRev(gitRev);
+  const googleCloudUrl = `${SO_MIGRATIONS_BUCKET_PREFIX}/${fullCommitHash}.json`;
+
+  return downloadToTemp(googleCloudUrl, log);
+}
+
+function expandGitRev(gitRev: string) {
+  if (gitRev.match(/^[0-9a-f]{40}$/)) {
+    return gitRev;
+  } else {
+    try {
+      return execSync(`git rev-parse ${gitRev}`, { stdio: ['pipe', 'pipe', null] })
+        .toString()
+        .trim();
+    } catch (err) {
+      throw new Error(`Couldn't expand git rev: ${gitRev} - ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Collects all plugin names that have different hashes in the two snapshots.
+ * @param fromSnapshot
+ * @param toSnapshot
+ */
+function compareSnapshotFiles(fromSnapshot: MigrationSnapshot, toSnapshot: MigrationSnapshot) {
+  const pluginNames = Object.keys(fromSnapshot.typeDefinitions);
+  const pluginNamesWithChangedHash = pluginNames.filter((pluginName) => {
+    const fromHash = fromSnapshot.typeDefinitions[pluginName].hash;
+    const toHash = toSnapshot.typeDefinitions[pluginName].hash;
+    return fromHash !== toHash;
+  });
+
+  const restOfPluginNames = pluginNames.filter((e) => !pluginNamesWithChangedHash.includes(e));
+
+  const changes = pluginNamesWithChangedHash.reduce((changesObj, pluginName) => {
+    const fromMigrationInfo = fromSnapshot.typeDefinitions[pluginName];
+    const toMigrationInfo = toSnapshot.typeDefinitions[pluginName];
+    changesObj[pluginName] = {
+      from: fromMigrationInfo,
+      to: toMigrationInfo,
+    };
+    return changesObj;
+  }, {} as Record<string, { from: MigrationInfoRecord; to: MigrationInfoRecord }>);
+
+  return {
+    hasChanges: pluginNamesWithChangedHash.length > 0,
+    from: fromSnapshot.meta.kibanaCommitHash,
+    to: toSnapshot.meta.kibanaCommitHash,
+    changed: pluginNamesWithChangedHash,
+    unchanged: restOfPluginNames,
+    changes,
+  };
+}
+
+async function loadJson(filePath: string) {
+  try {
+    const fileContent = await readFile(filePath, { encoding: 'utf-8' });
+    return JSON.parse(fileContent);
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`Snapshot file not found: ${filePath}`);
+    } else if (err.message.includes('Unexpected token')) {
+      throw new Error(`Snapshot file is not a valid JSON: ${filePath}`);
+    } else {
+      throw err;
+    }
+  }
+}
+
+export { compareSnapshots };

src/dev/so_migration/snapshot_plugin_types.ts

@@ -10,30 +10,15 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as cp from 'child_process';
 
-import {
-  extractMigrationInfo,
-  getMigrationHash,
-  SavedObjectTypeMigrationInfo,
-  // TODO: how to resolve this? Where to place this script?
-  // eslint-disable-next-line @kbn/imports/no_boundary_crossing
-} from '@kbn/core-test-helpers-so-type-serializer';
-import {
-  createTestServers,
-  createRootWithCorePlugins,
-  // TODO: how to resolve this? Where to place this script?
-  // eslint-disable-next-line @kbn/imports/no_boundary_crossing
-} from '@kbn/core-test-helpers-kbn-server';
+// eslint-disable-next-line @kbn/imports/no_boundary_crossing
+import { extractMigrationInfo, getMigrationHash } from '@kbn/core-test-helpers-so-type-serializer';
+// eslint-disable-next-line @kbn/imports/no_boundary_crossing
+import { createRootWithCorePlugins, createTestServers } from '@kbn/core-test-helpers-kbn-server';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { ToolingLog } from '@kbn/tooling-log';
 
 import { mkdirp } from '../build/lib';
-
-type MigrationInfoRecord = Pick<
-  SavedObjectTypeMigrationInfo,
-  'name' | 'migrationVersions' | 'schemaVersions' | 'modelVersions' | 'mappings'
-> & {
-  hash: string;
-};
+import type { MigrationSnapshot, MigrationInfoRecord, MigrationSnapshotMeta } from './types';
 
 type ServerHandles = Awaited<ReturnType<typeof startServers>> | undefined;
 
@@ -68,7 +53,12 @@ async function takeSnapshot({ log, outputPath }: { log: ToolingLog; outputPath:
       return map;
     }, {} as Record<string, MigrationInfoRecord>);
 
-    await writeSnapshotFile(snapshotOutputPath, migrationInfoMap);
+    const payload: MigrationSnapshot = {
+      meta: collectSOSnapshotMeta(),
+      typeDefinitions: migrationInfoMap,
+    };
+
+    await writeSnapshotFile(snapshotOutputPath, payload);
     log.info('Snapshot taken!');
 
     return migrationInfoMap;
@@ -91,30 +81,26 @@ async function startServers() {
   return { esServer, kibanaRoot, coreStart };
 }
 
-async function writeSnapshotFile(
-  snapshotOutputPath: string,
-  typeDefinitions: Record<string, MigrationInfoRecord>
-) {
+async function writeSnapshotFile(snapshotOutputPath: string, payload: MigrationSnapshot) {
+  await mkdirp(path.dirname(snapshotOutputPath));
+  fs.writeFileSync(snapshotOutputPath, JSON.stringify(payload, null, 2));
+}
+
+function collectSOSnapshotMeta(): MigrationSnapshotMeta {
   const timestamp = Date.now();
   const date = new Date().toISOString();
-  const buildUrl = process.env.BUILDKITE_BUILD_URL;
+  const buildUrl = process.env.BUILDKITE_BUILD_URL || null;
   const prId = process.env.BUILDKITE_MESSAGE?.match(/\(#(\d+)\)/)?.[1];
   const pullRequestUrl = prId ? `https://github.com/elastic/kibana/pulls/${prId}` : null;
   const kibanaCommitHash = process.env.BUILDKITE_COMMIT || getLocalHash();
 
-  const payload = {
-    meta: {
-      timestamp,
-      date,
-      kibanaCommitHash,
-      buildUrl,
-      pullRequestUrl,
-    },
-    typeDefinitions,
+  return {
+    timestamp,
+    date,
+    kibanaCommitHash,
+    buildUrl,
+    pullRequestUrl,
   };
-
-  await mkdirp(path.dirname(snapshotOutputPath));
-  fs.writeFileSync(snapshotOutputPath, JSON.stringify(payload, null, 2));
 }
 
 async function shutdown(log: ToolingLog, serverHandles: ServerHandles) {

src/dev/so_migration/so_migration_compare_cli.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { run } from '@kbn/dev-cli-runner';
+
+import { compareSnapshots } from './compare_snapshots';
+
+const scriptName = process.argv[1].replace(/^.*scripts\//, 'scripts/');
+
+run(
+  async ({ log, flagsReader, procRunner }) => {
+    const outputPath = flagsReader.string('outputPath');
+
+    const from = flagsReader.requiredString('from');
+    const to = flagsReader.requiredString('to');
+
+    const result = await compareSnapshots({ from, to, outputPath, log });
+
+    return {
+      outputPath,
+      result,
+      log,
+    };
+  },
+  {
+    usage: [
+      process.argv0,
+      scriptName,
+      'compare',
+      '--from <rev|filename|url>',
+      '--to <rev|filename|url>',
+      '[--outputPath <outputPath>]',
+    ].join(' '),
+    description: `Compares two Saved Object snapshot files based on hashes, filenames or urls.`,
+    flags: {
+      string: ['outputPath', 'from', 'to'],
+      help: `
+        --from            The source snapshot to compare from. Can be a revision, filename or url.
+        --to              The target snapshot to compare to. Can be a revision, filename or url.
+        --outputPath      The path to write the comparison report to. If omitted, raw JSON will be output to stdout.
+      `,
+    },
+  }
+)
+  .then((success) => {
+    // Kibana won't stop because some async processes are stuck polling, we need to shut down the process.
+    process.exit(0);
+  })
+  .catch((err) => {
+    // eslint-disable-next-line no-console
+    console.error(err);
+    process.exit(1);
+  });

src/dev/so_migration/so_migration_snapshot_cli.ts

@@ -15,7 +15,7 @@ const DEFAULT_OUTPUT_PATH = 'target/plugin_so_types_snapshot.json';
 
 run(
   async ({ log, flagsReader, procRunner }) => {
-    const outputPath = flagsReader.getPositionals()[0] || DEFAULT_OUTPUT_PATH;
+    const outputPath = flagsReader.string('outputPath') || DEFAULT_OUTPUT_PATH;
 
     const result = await takeSnapshot({ outputPath, log });
 
@@ -26,7 +26,7 @@ run(
     };
   },
   {
-    usage: [process.argv0, scriptName, '[outputPath]'].join(' '),
+    usage: [process.argv0, scriptName, 'snapshot', '[--outputPath <outputPath>]'].join(' '),
     description: `Takes a snapshot of all Kibana plugin Saved Object migrations' information, in a JSON format.`,
     flags: {
       string: ['outputPath'],

src/dev/so_migration/types.d.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { SavedObjectTypeMigrationInfo } from '@kbn/core-test-helpers-so-type-serializer';
+
+export type MigrationInfoRecord = Pick<
+  SavedObjectTypeMigrationInfo,
+  'name' | 'migrationVersions' | 'schemaVersions' | 'modelVersions' | 'mappings'
+> & {
+  hash: string;
+};
+
+export interface MigrationSnapshotMeta {
+  date: string;
+  kibanaCommitHash: string | null;
+  buildUrl: string | null;
+  pullRequestUrl: string | null;
+  timestamp: number;
+}
+
+export interface MigrationSnapshot {
+  meta: MigrationSnapshotMeta;
+  typeDefinitions: Record<string, MigrationInfoRecord>;
+}

src/dev/so_migration/util/download_file.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createWriteStream, unlinkSync } from 'fs';
+import https from 'https';
+
+export function downloadFile(url: string, outputPath: string) {
+  const file = createWriteStream(outputPath);
+
+  return new Promise((res, rej) => {
+    https
+      .get(url, (response) => {
+        if (response.statusCode !== 200) {
+          rej(response.statusMessage);
+        } else {
+          response.pipe(file);
+
+          file.on('finish', () => {
+            file.close();
+            res(undefined);
+          });
+        }
+      })
+      .on('error', (err) => {
+        unlinkSync(outputPath);
+        rej(err);
+      });
+  });
+}