mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[OAS] Capture and commit serverless bundle (#184915)
## Summary Close https://github.com/elastic/kibana/issues/184719 Adds the ability to capture OAS for `serverless` build flavor in addition to `traditional`. By default the CLI will run for both, but this can be controlled by passing in one of two new flags: `--no-serverless` or `--no-traditional`. See `oas_docs/bundle.serverless.json` for output. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jean-Louis Leysens
GitHub
parent
8f3359c43d
commit
cf7196fa1f
8 changed files with 757 additions and 97 deletions
|
|
@ -0,0 +1,125 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import { encode } from 'node:querystring';
|
||||
import type { ChildProcess } from 'node:child_process';
|
||||
import fetch from 'node-fetch';
|
||||
import * as Rx from 'rxjs';
|
||||
import { startTSWorker } from '@kbn/dev-utils';
|
||||
import { createTestEsCluster } from '@kbn/test';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import { createTestServerlessInstances } from '@kbn/core-test-helpers-kbn-server';
|
||||
import type { Result } from './kibana_worker';
|
||||
import { sortAndPrettyPrint } from './run_capture_oas_snapshot_cli';
|
||||
import { buildFlavourEnvArgName } from './common';
|
||||
|
||||
interface CaptureOasSnapshotArgs {
|
||||
log: ToolingLog;
|
||||
buildFlavour: 'serverless' | 'traditional';
|
||||
outputFile: string;
|
||||
update: boolean;
|
||||
filters?: {
|
||||
pathStartsWith?: string[];
|
||||
excludePathsMatching?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
const MB = 1024 * 1024;
|
||||
const twoDeci = (num: number) => Math.round(num * 100) / 100;
|
||||
|
||||
export async function captureOasSnapshot({
|
||||
log,
|
||||
filters = {},
|
||||
buildFlavour,
|
||||
update,
|
||||
outputFile,
|
||||
}: CaptureOasSnapshotArgs): Promise<void> {
|
||||
const { excludePathsMatching = [], pathStartsWith } = filters;
|
||||
// internal consts
|
||||
const port = 5622;
|
||||
// We are only including /api/status for now
|
||||
excludePathsMatching.push(
|
||||
'/{path*}',
|
||||
// Our internal asset paths
|
||||
'/XXXXXXXXXXXX/'
|
||||
);
|
||||
|
||||
let esCluster: undefined | { stop(): Promise<void> };
|
||||
let kbWorker: undefined | ChildProcess;
|
||||
|
||||
try {
|
||||
log.info('Starting es...');
|
||||
esCluster = await log.indent(4, async () => {
|
||||
if (buildFlavour === 'serverless') {
|
||||
const { startES } = createTestServerlessInstances();
|
||||
return await startES();
|
||||
}
|
||||
const cluster = createTestEsCluster({ log });
|
||||
await cluster.start();
|
||||
return { stop: () => cluster.cleanup() };
|
||||
});
|
||||
|
||||
log.info('Starting Kibana...');
|
||||
kbWorker = await log.indent(4, async () => {
|
||||
log.info('Loading core with all plugins enabled so that we can capture OAS for all...');
|
||||
const { msg$, proc } = startTSWorker<Result>({
|
||||
log,
|
||||
src: require.resolve('./kibana_worker'),
|
||||
env: { ...process.env, [buildFlavourEnvArgName]: buildFlavour },
|
||||
});
|
||||
await Rx.firstValueFrom(
|
||||
msg$.pipe(
|
||||
Rx.map((msg) => {
|
||||
if (msg !== 'ready')
|
||||
throw new Error(`received unexpected message from worker (expected "ready"): ${msg}`);
|
||||
})
|
||||
)
|
||||
);
|
||||
return proc;
|
||||
});
|
||||
|
||||
const qs = encode({
|
||||
access: 'public',
|
||||
version: '2023-10-31', // hard coded for now, we can make this configurable later
|
||||
pathStartsWith,
|
||||
excludePathsMatching,
|
||||
});
|
||||
const url = `http://localhost:${port}/api/oas?${qs}`;
|
||||
log.info(`Fetching OAS at ${url}...`);
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
'kbn-xsrf': 'kbn-oas-snapshot',
|
||||
authorization: `Basic ${Buffer.from('elastic:changeme').toString('base64')}`,
|
||||
},
|
||||
});
|
||||
if (result.status !== 200) {
|
||||
log.error(`Failed to fetch OAS: ${JSON.stringify(result, null, 2)}`);
|
||||
throw new Error(`Failed to fetch OAS: ${result.status}`);
|
||||
}
|
||||
const currentOas = await result.json();
|
||||
log.info(`Recieved OAS, writing to ${outputFile}...`);
|
||||
if (update) {
|
||||
await fs.writeFile(outputFile, sortAndPrettyPrint(currentOas));
|
||||
const { size: sizeBytes } = await fs.stat(outputFile);
|
||||
log.success(`OAS written to ${outputFile}. File size ~${twoDeci(sizeBytes / MB)} MB.`);
|
||||
} else {
|
||||
log.success(
|
||||
`OAS recieved, not writing to file. Got OAS for ${
|
||||
Object.keys(currentOas.paths).length
|
||||
} paths.`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
log.error(`Failed to capture OAS: ${JSON.stringify(err, null, 2)}`);
|
||||
throw err;
|
||||
} finally {
|
||||
kbWorker?.kill('SIGILL');
|
||||
await esCluster?.stop();
|
||||
}
|
||||
}
|
||||
9
packages/kbn-capture-oas-snapshot-cli/src/common.ts
Normal file
9
packages/kbn-capture-oas-snapshot-cli/src/common.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export const buildFlavourEnvArgName = 'CAPTURE_OAS_SNAPSHOT_WORKER_BUILD_FLAVOR';
|
||||
|
|
@ -6,9 +6,13 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
|
||||
import {
|
||||
createRootWithCorePlugins,
|
||||
createTestServerlessInstances,
|
||||
} from '@kbn/core-test-helpers-kbn-server';
|
||||
import { set } from '@kbn/safer-lodash-set';
|
||||
import { PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH } from '@kbn/core-plugins-server-internal/src/constants';
|
||||
import { buildFlavourEnvArgName } from './common';
|
||||
|
||||
export type Result = 'ready';
|
||||
|
||||
|
|
@ -16,6 +20,10 @@ export type Result = 'ready';
|
|||
if (!process.send) {
|
||||
throw new Error('worker must be run in a node.js fork');
|
||||
}
|
||||
const buildFlavour = process.env[buildFlavourEnvArgName];
|
||||
if (!buildFlavour) throw new Error(`env arg ${buildFlavourEnvArgName} must be provided`);
|
||||
|
||||
const serverless = buildFlavour === 'serverless';
|
||||
|
||||
const settings = {
|
||||
logging: {
|
||||
|
|
@ -30,7 +38,8 @@ export type Result = 'ready';
|
|||
};
|
||||
set(settings, PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH, true);
|
||||
|
||||
const root = createRootWithCorePlugins(settings, {
|
||||
const cliArgs = {
|
||||
serverless,
|
||||
basePath: false,
|
||||
cache: false,
|
||||
dev: true,
|
||||
|
|
@ -40,11 +49,21 @@ export type Result = 'ready';
|
|||
oss: false,
|
||||
runExamples: false,
|
||||
watch: false,
|
||||
});
|
||||
};
|
||||
|
||||
await root.preboot();
|
||||
await root.setup();
|
||||
await root.start();
|
||||
if (serverless) {
|
||||
// Satisfy spaces config for serverless:
|
||||
set(settings, 'xpack.spaces.allowFeatureVisibility', false);
|
||||
const { startKibana } = createTestServerlessInstances({
|
||||
kibana: { settings, cliArgs },
|
||||
});
|
||||
await startKibana();
|
||||
} else {
|
||||
const root = createRootWithCorePlugins(settings, cliArgs);
|
||||
await root.preboot();
|
||||
await root.setup();
|
||||
await root.start();
|
||||
}
|
||||
|
||||
const result: Result = 'ready';
|
||||
|
||||
|
|
|
|||
|
|
@ -7,18 +7,10 @@
|
|||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs/promises';
|
||||
import { encode } from 'node:querystring';
|
||||
import fetch from 'node-fetch';
|
||||
import { run } from '@kbn/dev-cli-runner';
|
||||
import { startTSWorker } from '@kbn/dev-utils';
|
||||
import { createTestEsCluster } from '@kbn/test';
|
||||
import * as Rx from 'rxjs';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import chalk from 'chalk';
|
||||
import type { Result } from './kibana_worker';
|
||||
|
||||
const OAS_FILE_PATH = path.resolve(REPO_ROOT, './oas_docs/bundle.json');
|
||||
import { captureOasSnapshot } from './capture_oas_snapshot';
|
||||
|
||||
export const sortAndPrettyPrint = (object: object) => {
|
||||
const keys = new Set<string>();
|
||||
|
|
@ -29,84 +21,45 @@ export const sortAndPrettyPrint = (object: object) => {
|
|||
return JSON.stringify(object, Array.from(keys).sort(), 2);
|
||||
};
|
||||
|
||||
const MB = 1024 * 1024;
|
||||
const twoDeci = (num: number) => Math.round(num * 100) / 100;
|
||||
const OAS_OUTPUT_DIR = path.resolve(REPO_ROOT, './oas_docs');
|
||||
|
||||
run(
|
||||
async ({ log, flagsReader, addCleanupTask }) => {
|
||||
async ({ log, flagsReader }) => {
|
||||
const serverless = flagsReader.boolean('serverless');
|
||||
const traditional = flagsReader.boolean('traditional');
|
||||
if (!serverless && !traditional) {
|
||||
log.error(
|
||||
'Not capturing any OAS, remove one or both of `--no-serverless` or `--no-traditional` flags to run this CLI'
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const update = flagsReader.boolean('update');
|
||||
const pathStartsWith = flagsReader.arrayOfStrings('include-path');
|
||||
const excludePathsMatching = flagsReader.arrayOfStrings('exclude-path') ?? [];
|
||||
|
||||
// internal consts
|
||||
const port = 5622;
|
||||
// We are only including /api/status for now
|
||||
excludePathsMatching.push(
|
||||
'/{path*}',
|
||||
// Our internal asset paths
|
||||
'/XXXXXXXXXXXX/'
|
||||
);
|
||||
|
||||
log.info('Starting es...');
|
||||
await log.indent(4, async () => {
|
||||
const cluster = createTestEsCluster({ log });
|
||||
await cluster.start();
|
||||
addCleanupTask(() => cluster.cleanup());
|
||||
});
|
||||
|
||||
log.info('Starting Kibana...');
|
||||
await log.indent(4, async () => {
|
||||
log.info('Loading core with all plugins enabled so that we can capture OAS for all...');
|
||||
const { msg$, proc } = startTSWorker<Result>({
|
||||
if (traditional) {
|
||||
log.info('Capturing OAS for traditional Kibana...');
|
||||
await captureOasSnapshot({
|
||||
log,
|
||||
src: require.resolve('./kibana_worker'),
|
||||
buildFlavour: 'traditional',
|
||||
outputFile: path.resolve(OAS_OUTPUT_DIR, 'bundle.json'),
|
||||
filters: { pathStartsWith, excludePathsMatching },
|
||||
update,
|
||||
});
|
||||
await Rx.firstValueFrom(
|
||||
msg$.pipe(
|
||||
Rx.map((msg) => {
|
||||
if (msg !== 'ready')
|
||||
throw new Error(`received unexpected message from worker (expected "ready"): ${msg}`);
|
||||
})
|
||||
)
|
||||
);
|
||||
addCleanupTask(() => proc.kill('SIGILL'));
|
||||
});
|
||||
log.success('Captured OAS for traditional Kibana.');
|
||||
}
|
||||
|
||||
try {
|
||||
const qs = encode({
|
||||
access: 'public',
|
||||
version: '2023-10-31', // hard coded for now, we can make this configurable later
|
||||
pathStartsWith,
|
||||
excludePathsMatching,
|
||||
if (serverless) {
|
||||
log.info('Capturing OAS for serverless Kibana...');
|
||||
await captureOasSnapshot({
|
||||
log,
|
||||
buildFlavour: 'serverless',
|
||||
outputFile: path.resolve(OAS_OUTPUT_DIR, 'bundle.serverless.json'),
|
||||
filters: { pathStartsWith, excludePathsMatching },
|
||||
update,
|
||||
});
|
||||
const url = `http://localhost:${port}/api/oas?${qs}`;
|
||||
log.info(`Fetching OAS at ${url}...`);
|
||||
const result = await fetch(url, {
|
||||
headers: {
|
||||
'kbn-xsrf': 'kbn-oas-snapshot',
|
||||
authorization: `Basic ${Buffer.from('elastic:changeme').toString('base64')}`,
|
||||
},
|
||||
});
|
||||
if (result.status !== 200) {
|
||||
log.error(`Failed to fetch OAS: ${JSON.stringify(result, null, 2)}`);
|
||||
throw new Error(`Failed to fetch OAS: ${result.status}`);
|
||||
}
|
||||
const currentOas = await result.json();
|
||||
log.info(`Recieved OAS, writing to ${OAS_FILE_PATH}...`);
|
||||
if (update) {
|
||||
await fs.writeFile(OAS_FILE_PATH, sortAndPrettyPrint(currentOas));
|
||||
const { size: sizeBytes } = await fs.stat(OAS_FILE_PATH);
|
||||
log.success(`OAS written to ${OAS_FILE_PATH}. File size ~${twoDeci(sizeBytes / MB)} MB.`);
|
||||
} else {
|
||||
log.success(
|
||||
`OAS recieved, not writing to file. Got OAS for ${
|
||||
Object.keys(currentOas.paths).length
|
||||
} paths.`
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
log.error(`Failed to capture OAS: ${JSON.stringify(err, null, 2)}`);
|
||||
throw err;
|
||||
log.success('Captured OAS for serverless Kibana.');
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -114,15 +67,19 @@ run(
|
|||
Get the current OAS from Kibana's /api/oas API
|
||||
`,
|
||||
flags: {
|
||||
boolean: ['update'],
|
||||
boolean: ['update', 'serverless', 'traditional'],
|
||||
string: ['include-path', 'exclude-path'],
|
||||
default: {
|
||||
fix: false,
|
||||
serverless: true,
|
||||
traditional: true,
|
||||
},
|
||||
help: `
|
||||
--include-path Path to include. Path must start with provided value. Can be passed multiple times.
|
||||
--exclude-path Path to exclude. Path must NOT start with provided value. Can be passed multiple times.
|
||||
--update Write the current OAS to ${chalk.cyan(OAS_FILE_PATH)}.
|
||||
--update Write the current OAS bundles to ${chalk.cyan(OAS_OUTPUT_DIR)}.
|
||||
--no-serverless Whether to skip OAS for serverless Kibana. Defaults to false.
|
||||
--no-traditional Whether to skip OAS for traditional Kibana. Defaults to false.
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,5 +21,6 @@
|
|||
"@kbn/dev-cli-runner",
|
||||
"@kbn/test",
|
||||
"@kbn/dev-utils",
|
||||
"@kbn/tooling-log",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue