[ci] build next docs in PRs when relevant files change (#149991)

After chatting with @KOTungseth, @scottybollinger, and @glitteringkatie
we've decided to add a CI step to the Kibana repo that will run when
changes to next-doc related code is made. This step will checkout the
repository containing configuration for the docs.elastic.dev website
(which is currently private, sorry) and then ensure that the build can
be completed with a local copy of all the repositories. It does this by
reading the `config/content.js` files and cloning all of the
repositories listed, then rewriting the content.js file with a map
telling the build system to read files from the local repos (which are
pre-cached by the packer cache job) and the local Kibana repo (which
represents the changes in the PR).

This script also runs locally by running `node
scripts/validate_next_docs`.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/pull_request/check_next_docs.yml

@@ -0,0 +1,10 @@
+steps:
+  - command: .buildkite/scripts/steps/next_docs/build_and_validate_docs.sh
+    label: 'Build and Validate Next Docs'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/scripts/common/env.sh

@@ -12,6 +12,8 @@ PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
 export PARENT_DIR
 export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
 
+export DOCS_REPO_CACHE_DIR="$HOME/.docs-repos"
+
 # A few things, such as Chrome, respect this variable
 # For many agent types, the workspace is mounted on a local ssd, so will be faster than the default tmp dir location
 if [[ -d /opt/local-ssd/buildkite ]]; then

.buildkite/scripts/packer_cache.sh

@@ -13,3 +13,15 @@ yarn kbn bootstrap
 for version in $(cat versions.json | jq -r '.versions[].version'); do
   node scripts/es snapshot --download-only --base-path "$ES_CACHE_DIR" --version "$version"
 done
+
+echo "--- Logging into vault to access private repos"
+VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+retry 5 30 vault login -no-print "$VAULT_TOKEN"
+
+GITHUB_TOKEN="$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)"
+export GITHUB_TOKEN
+
+echo "--- Cloning repos for docs build"
+node scripts/validate_next_docs --clone-only

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -171,6 +171,19 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/webpack_bundle_analyzer.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /\.docnav\.json$/,
+        /\.apidocs\.json$/,
+        /\.devdocs\.json$/,
+        /\.mdx$/,
+        /^dev_docs\/.*(png|gif|jpg|jpeg|webp)$/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:build-next-docs')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/check_next_docs.yml'));
+    }
+
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
     uploadPipeline(pipeline.join('\n'));

.buildkite/scripts/steps/next_docs/build_and_validate_docs.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/bootstrap.sh
+
+echo "--- Build docs"
+
+echo "⚠️ run 'node scripts/validate_next_docs --debug' locally to debug ⚠️"
+node scripts/validate_next_docs
+
+
+

.eslintrc.js

@@ -1736,6 +1736,7 @@ module.exports = {
     {
       files: [
         'packages/kbn-{package-*,repo-*,dep-*}/**/*',
+        'packages/kbn-validate-next-docs-cli/**/*',
         'packages/kbn-find-used-node-modules/**/*',
       ],
       rules: {

.github/CODEOWNERS

@@ -669,6 +669,7 @@ packages/kbn-utility-types @elastic/kibana-core
 packages/kbn-utility-types-jest @elastic/kibana-operations
 packages/kbn-utils @elastic/kibana-operations
 x-pack/plugins/ux @elastic/uptime
+packages/kbn-validate-next-docs-cli @elastic/kibana-operations
 src/plugins/vis_default_editor @elastic/kibana-visualizations
 src/plugins/vis_types/gauge @elastic/kibana-visualizations
 src/plugins/vis_types/heatmap @elastic/kibana-visualizations

package.json

@@ -1104,6 +1104,7 @@
     "@kbn/tooling-log": "link:packages/kbn-tooling-log",
     "@kbn/ts-projects": "link:packages/kbn-ts-projects",
     "@kbn/ts-type-check-cli": "link:packages/kbn-ts-type-check-cli",
+    "@kbn/validate-next-docs-cli": "link:packages/kbn-validate-next-docs-cli",
     "@kbn/web-worker-stub": "link:packages/kbn-web-worker-stub",
     "@kbn/whereis-pkg-cli": "link:packages/kbn-whereis-pkg-cli",
     "@kbn/yarn-lock-validator": "link:packages/kbn-yarn-lock-validator",

packages/kbn-validate-next-docs-cli/README.md

@@ -0,0 +1,5 @@
+# @kbn/validate-next-docs-cli
+
+Clone a bunch of repos needed to build the next-docs with the latest version of the changes in the repo. This is only used for validating changes in CI with full validation of all IDs. This isn't intended for local development (except for debugging).
+
+To build the docs locally run: `./scripts/dev_docs.sh`

packages/kbn-validate-next-docs-cli/config.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fs from 'fs';
+
+import { Repo } from './repos';
+
+export interface Source {
+  type: string;
+  location: string;
+  subdirs?: string[];
+}
+
+export class Config {
+  path: string;
+  backupPath: string;
+
+  constructor(private readonly repo: Repo) {
+    this.path = this.repo.resolve('config/content.js');
+    this.backupPath = `${this.path}.backup`;
+    this.restore();
+  }
+
+  private read(): { content: { sources: Source[]; nav: unknown } } {
+    delete require.cache[this.path];
+    return require(this.path);
+  }
+
+  private restore() {
+    if (Fs.existsSync(this.backupPath)) {
+      Fs.renameSync(this.backupPath, this.path);
+    }
+  }
+
+  getSources() {
+    return this.read().content.sources;
+  }
+
+  setSources(sources: Source[]) {
+    this.restore();
+    Fs.copyFileSync(this.path, this.backupPath);
+
+    const current = this.read();
+
+    Fs.writeFileSync(
+      this.path,
+      `module.exports = ${JSON.stringify(
+        {
+          content: {
+            ...current.content,
+            sources,
+          },
+        },
+        null,
+        2
+      )}`
+    );
+  }
+}

packages/kbn-validate-next-docs-cli/error.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { createFailError } from '@kbn/dev-cli-errors';
+
+export function quietFail(msg: string): never {
+  throw createFailError(
+    `${msg}${
+      process.env.CI
+        ? ` (in order to avoid potential info leaks, we've hidden the logging output. Please run this command locally with --debug to get logging output)`
+        : ''
+    }`
+  );
+}

packages/kbn-validate-next-docs-cli/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-validate-next-docs-cli'],
+};

packages/kbn-validate-next-docs-cli/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/validate-next-docs-cli",
+  "owner": "@elastic/kibana-operations",
+  "devOnly": true
+}

packages/kbn-validate-next-docs-cli/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/validate-next-docs-cli",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "main": "./validate_next_docs_cli"
+}

packages/kbn-validate-next-docs-cli/repos.ts

@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Fs from 'fs';
+import Os from 'os';
+import Path from 'path';
+import Rl from 'readline';
+
+import Chalk from 'chalk';
+import execa from 'execa';
+import { ToolingLog } from '@kbn/tooling-log';
+
+import { quietFail } from './error';
+
+function lines(read: NodeJS.ReadableStream) {
+  return Rl.createInterface({
+    input: read,
+    crlfDelay: Infinity,
+  });
+}
+
+function getGithubBase() {
+  try {
+    const originUrl = execa.sync('git', ['remote', 'get-url', 'origin'], {
+      encoding: 'utf8',
+    }).stdout;
+
+    if (originUrl.startsWith('git@')) {
+      return `git@github.com:`;
+    }
+  } catch {
+    // noop
+  }
+
+  if (process.env.GITHUB_TOKEN) {
+    return `https://${process.env.GITHUB_TOKEN}@github.com/`;
+  }
+
+  return `https://github.com/`;
+}
+
+export class Repo {
+  constructor(
+    private readonly log: ToolingLog,
+    private readonly name: string,
+    private readonly dir: string,
+    private readonly githubBase: string
+  ) {}
+
+  resolve(...segs: string[]) {
+    return Path.resolve(this.dir, ...segs);
+  }
+
+  exists() {
+    if (!Fs.existsSync(this.dir)) {
+      return false;
+    }
+
+    if (!Fs.statSync(this.dir).isDirectory()) {
+      throw new Error(`[${this.name}] directory exists but it's not a directory like it should be`);
+    }
+
+    return true;
+  }
+
+  async run(cmd: string, args: string[], opts: { desc: string; showOutput?: boolean }) {
+    try {
+      if (!opts.showOutput) {
+        await execa(cmd, args, {
+          cwd: this.dir,
+          maxBuffer: Infinity,
+        });
+        return;
+      }
+
+      await this.log.indent(4, async () => {
+        const proc = execa(cmd, args, {
+          cwd: this.dir,
+          stdio: ['ignore', 'pipe', 'pipe'],
+        });
+
+        await Promise.all([
+          (async () => {
+            for await (const line of lines(proc.stdout!)) {
+              this.log.write(Chalk.dim('out '), line);
+            }
+          })(),
+          (async () => {
+            for await (const line of lines(proc.stderr!)) {
+              this.log.write(Chalk.red('err '), line);
+            }
+          })(),
+          new Promise((resolve, reject) => {
+            proc.once('exit', resolve).once('exit', reject);
+          }),
+        ]);
+      });
+    } catch (error) {
+      this.log.debug(
+        `Failed to run [${opts.desc}] in [${this.name}]`,
+        opts.showOutput ? error : undefined
+      );
+
+      throw quietFail(`${this.name}: ${opts.desc} failed`);
+    }
+  }
+
+  async update() {
+    try {
+      this.log.info('updating', this.name);
+      await this.run('git', ['reset', '--hard'], { desc: 'git reset --hard' });
+      await this.run(
+        'git',
+        [
+          'clean',
+          '-fdx',
+          ...(this.name === 'elastic/docs.elastic.dev'
+            ? ['-e', '/.docsmobile', '-e', '/node_modules']
+            : []),
+        ],
+        { desc: 'git clean -fdx' }
+      );
+      await this.run('git', ['pull'], { desc: 'git pull' });
+    } catch {
+      quietFail(`failed to update ${this.name}`);
+    }
+  }
+
+  async clone() {
+    try {
+      this.log.info('cloning', this.name);
+      Fs.mkdirSync(this.dir, { recursive: true });
+      const depth = process.env.CI ? ['--depth', '1'] : [];
+      await this.run('git', ['clone', ...depth, `${this.githubBase}${this.name}.git`, '.'], {
+        desc: 'git clone ...',
+      });
+    } catch {
+      quietFail(`Failed to clone ${this.name}`);
+    }
+  }
+}
+
+export class Repos {
+  githubBase = getGithubBase();
+  repoDir = Path.resolve(Os.userInfo().homedir, '.cache/next-docs/repos');
+
+  constructor(private readonly log: ToolingLog) {}
+
+  async init(repoName: string) {
+    const repo = new Repo(
+      this.log,
+      repoName,
+      Path.resolve(this.repoDir, repoName),
+      this.githubBase
+    );
+
+    if (repo.exists()) {
+      await repo.update();
+    } else {
+      await repo.clone();
+    }
+
+    return repo;
+  }
+}

packages/kbn-validate-next-docs-cli/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/dev-cli-errors",
+    "@kbn/tooling-log",
+    "@kbn/dev-cli-runner",
+    "@kbn/repo-info",
+  ]
+}

packages/kbn-validate-next-docs-cli/validate_next_docs_cli.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { run } from '@kbn/dev-cli-runner';
+import { createFailError } from '@kbn/dev-cli-errors';
+import { REPO_ROOT } from '@kbn/repo-info';
+
+import { Repos } from './repos';
+import { Config, Source } from './config';
+import { quietFail } from './error';
+
+run(
+  async ({ log, flagsReader }) => {
+    const cloneOnly = flagsReader.boolean('clone-only');
+
+    const repos = new Repos(log);
+    const docsRepo = await repos.init('elastic/docs.elastic.dev');
+    const contentConfig = new Config(docsRepo);
+    const sources = contentConfig.getSources().filter((s) => s.location !== 'elastic/kibana');
+
+    if (sources.some((s) => s.type !== 'github')) {
+      throw createFailError(
+        'expected all content.js sources from docs.elastic.dev to have "type: github"'
+      );
+    }
+
+    const localCloneSources = await Promise.all(
+      sources.map(async (source): Promise<Source> => {
+        const repo = await repos.init(source.location);
+
+        return {
+          type: 'file',
+          location: repo.resolve(),
+        };
+      })
+    );
+
+    if (cloneOnly) {
+      log.success('cloned repos');
+      return;
+    }
+
+    log.info('[docs.elastic.dev] updated sources to point to local repos');
+    contentConfig.setSources([
+      ...localCloneSources,
+      {
+        type: 'file',
+        location: REPO_ROOT,
+      },
+    ]);
+
+    const showOutput = flagsReader.boolean('debug') || flagsReader.boolean('verbose');
+    try {
+      log.info('[docs.elastic.dev] installing deps with yarn');
+      await docsRepo.run('yarn', [], { desc: 'yarn install', showOutput });
+
+      log.info('[docs.elastic.dev] initializing docsmobile');
+      await docsRepo.run('yarn', ['docsmobile', 'init'], {
+        desc: 'yarn docsmobile init',
+        showOutput,
+      });
+
+      log.info('[docs.elastic.dev] building');
+      await docsRepo.run('yarn', ['build'], { desc: 'yarn build', showOutput });
+    } catch {
+      quietFail(`failed to build docs`);
+    }
+
+    log.success('docs built successfully');
+  },
+  {
+    flags: {
+      boolean: ['clone-only'],
+      help: `
+        --clone-only       Simply clone the repos, used to populate the worker images
+      `,
+    },
+  }
+);

scripts/validate_next_docs.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('@kbn/validate-next-docs-cli');

tsconfig.base.json

@@ -1332,6 +1332,8 @@
       "@kbn/utils/*": ["packages/kbn-utils/*"],
       "@kbn/ux-plugin": ["x-pack/plugins/ux"],
       "@kbn/ux-plugin/*": ["x-pack/plugins/ux/*"],
+      "@kbn/validate-next-docs-cli": ["packages/kbn-validate-next-docs-cli"],
+      "@kbn/validate-next-docs-cli/*": ["packages/kbn-validate-next-docs-cli/*"],
       "@kbn/vis-default-editor-plugin": ["src/plugins/vis_default_editor"],
       "@kbn/vis-default-editor-plugin/*": ["src/plugins/vis_default_editor/*"],
       "@kbn/vis-type-gauge-plugin": ["src/plugins/vis_types/gauge"],

yarn.lock

@@ -5389,6 +5389,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/validate-next-docs-cli@link:packages/kbn-validate-next-docs-cli":
+  version "0.0.0"
+  uid ""
+
 "@kbn/vis-default-editor-plugin@link:src/plugins/vis_default_editor":
   version "0.0.0"
   uid ""