mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
[kbn-es] add required flag projectType for serverless script (#175549)
## Summary in https://github.com/elastic/kibana/pull/174284 we split serverless roles into individual files per project. If you run `yarn es serverless --ssl` ES will be provisioned only with roles specified for elastisearch project type. To use roles for oblt/security projects, you have to override roles file with `--resources` flag: ``` yarn es serverless --ssl --resources /Users/dmle/github/kibana/packages/kbn-es/src/serverless_resources/project_roles/security/roles.yml ``` Since it is confusing and not dev-friendly approach, this PR adds new required flag to `serverless` script: `--projectType` Usage: `yarn es serverless --projectType=es --ssl` `yarn es --serverless=oblt --ssl` roles.yml file will be picked up based on `projectType` value, you still have an option to override it using `--resources` flag How to test: `yarn es serverless --project-type=oblt --ssl` `yarn start serverless=oblt --ssl` You should be able to login with all roles defined or Observability (and other) project. Cli docs were updated: ``` [main][~/github/kibana]$ node scripts/es --help usage: es <command> [<args>] Assists with running Elasticsearch for Kibana development Available commands: snapshot - Downloads and run from a nightly snapshot source - Build and run from source archive - Install and run from an Elasticsearch tar build_snapshots - Build and collect ES snapshots docker - Run an Elasticsearch Docker image serverless - Run Serverless Elasticsearch through Docker To start a serverless instance use the 'serverless' command with '--projectType' flag or use the '--serverless=<ProjectType>' shortcut, for example: es --serverless=es Global options: --help ``` --------- Co-authored-by: Robert Oskamp <traeluki@gmail.com>
This commit is contained in:
Dzmitry Lemechko
GitHub
parent
f9125ba079
commit
e2fc23f57e
8 changed files with 104 additions and 38 deletions
|
|
@ -9,7 +9,7 @@
|
|||
import Path from 'path';
|
||||
import { defaultsDeep } from 'lodash';
|
||||
import { Client, HttpConnection } from '@elastic/elasticsearch';
|
||||
import { Cluster } from '@kbn/es';
|
||||
import { Cluster, ServerlessProjectType } from '@kbn/es';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { esTestConfig } from '@kbn/test';
|
||||
|
|
@ -28,6 +28,8 @@ export interface TestServerlessUtils {
|
|||
|
||||
const ES_BASE_PATH_DIR = Path.join(REPO_ROOT, '.es/es_test_serverless');
|
||||
|
||||
const projectType: ServerlessProjectType = 'es';
|
||||
|
||||
/**
|
||||
* See docs in {@link TestUtils}. This function provides the same utilities but
|
||||
* configured for serverless.
|
||||
|
|
@ -79,6 +81,7 @@ function createServerlessES() {
|
|||
es,
|
||||
start: async () => {
|
||||
await es.runServerless({
|
||||
projectType,
|
||||
basePath: ES_BASE_PATH_DIR,
|
||||
port: esPort,
|
||||
background: true,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,12 @@ function help() {
|
|||
|
||||
${availableCommands.join('\n ')}
|
||||
|
||||
To start a serverless instance use the 'serverless' command with
|
||||
'--projectType' flag or use the '--serverless=<ProjectType>'
|
||||
shortcut, for example:
|
||||
|
||||
es --serverless=es
|
||||
|
||||
Global options:
|
||||
|
||||
--help
|
||||
|
|
@ -46,7 +52,16 @@ export async function run(defaults = {}) {
|
|||
default: defaults,
|
||||
});
|
||||
const args = options._;
|
||||
const commandName = args[0];
|
||||
let commandName = args[0];
|
||||
|
||||
// Converting --serverless flag to command
|
||||
// `es --serverless=<projectType>` is just a shortcut for
|
||||
// `es serverless --projectType=<projectType>`
|
||||
if (options.serverless) {
|
||||
const projectType: string = options.serverless;
|
||||
commandName = 'serverless';
|
||||
args.push('--projectType', projectType);
|
||||
}
|
||||
|
||||
if (args.length === 0 || (!commandName && options.help)) {
|
||||
help();
|
||||
|
|
|
|||
|
|
@ -19,8 +19,13 @@ import {
|
|||
ES_SERVERLESS_DEFAULT_IMAGE,
|
||||
DEFAULT_PORT,
|
||||
ServerlessOptions,
|
||||
isServerlessProjectType,
|
||||
serverlessProjectTypes,
|
||||
} from '../utils';
|
||||
import { Command } from './types';
|
||||
import { createCliError } from '../errors';
|
||||
|
||||
const supportedProjectTypesStr = Array.from(serverlessProjectTypes).join(' | ').trim();
|
||||
|
||||
export const serverless: Command = {
|
||||
description: 'Run Serverless Elasticsearch through Docker',
|
||||
|
|
@ -29,6 +34,7 @@ export const serverless: Command = {
|
|||
return dedent`
|
||||
Options:
|
||||
|
||||
--projectType Serverless project type: ${supportedProjectTypesStr}
|
||||
--tag Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH}
|
||||
--image Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}]
|
||||
--background Start ES serverless without attaching to the first node's logs
|
||||
|
|
@ -54,8 +60,8 @@ export const serverless: Command = {
|
|||
|
||||
Examples:
|
||||
|
||||
es serverless --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64
|
||||
es serverless --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
|
||||
es serverless --projectType es --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64
|
||||
es serverless --projectType oblt --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
|
||||
`;
|
||||
},
|
||||
run: async (defaults = {}) => {
|
||||
|
|
@ -66,20 +72,41 @@ export const serverless: Command = {
|
|||
});
|
||||
const reportTime = getTimeReporter(log, 'scripts/es serverless');
|
||||
|
||||
// replacing --serverless with --projectType when flag is passed from 'scripts/es'
|
||||
// `es --serverless=<projectType>` is just a shortcut for
|
||||
// `es serverless --projectType=<projectType>`
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv[0].startsWith('--serverless')) {
|
||||
const projectTypeArg = argv[0].replace('--serverless', '--projectType');
|
||||
argv[0] = projectTypeArg;
|
||||
}
|
||||
|
||||
const options = getopts(argv, {
|
||||
alias: {
|
||||
basePath: 'base-path',
|
||||
esArgs: 'E',
|
||||
files: 'F',
|
||||
projectType: 'project-type',
|
||||
},
|
||||
|
||||
string: ['tag', 'image', 'basePath', 'resources', 'host', 'kibanaUrl'],
|
||||
string: ['projectType', 'tag', 'image', 'basePath', 'resources', 'host', 'kibanaUrl'],
|
||||
boolean: ['clean', 'ssl', 'kill', 'background', 'skipTeardown', 'waitForReady'],
|
||||
|
||||
default: defaults,
|
||||
}) as unknown as ServerlessOptions;
|
||||
|
||||
if (!options.projectType) {
|
||||
throw createCliError(
|
||||
`--projectType flag is required and must be a string: ${supportedProjectTypesStr}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!isServerlessProjectType(options.projectType)) {
|
||||
throw createCliError(
|
||||
`Invalid projectPype '${options.projectType}', supported values: ${supportedProjectTypesStr}`
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
* The nodes will be killed immediately if background = true and skipTeardown = false
|
||||
* because the CLI process exits after starting the nodes. We handle this here instead of
|
||||
|
|
|
|||
|
|
@ -723,12 +723,16 @@ describe('#kill()', () => {
|
|||
});
|
||||
|
||||
describe('#runServerless()', () => {
|
||||
const defaultOptions = {
|
||||
projectType: 'es' as dockerUtils.ServerlessProjectType,
|
||||
basePath: installPath,
|
||||
};
|
||||
test(`rejects if #start() was called before`, async () => {
|
||||
mockEsBin({ start: true });
|
||||
|
||||
const cluster = new Cluster({ log });
|
||||
await cluster.start(installPath, esClusterExecOptions);
|
||||
await expect(cluster.runServerless({ basePath: installPath })).rejects.toThrowError(
|
||||
await expect(cluster.runServerless(defaultOptions)).rejects.toThrowError(
|
||||
'ES stateful cluster has already been started'
|
||||
);
|
||||
});
|
||||
|
|
@ -738,7 +742,7 @@ describe('#runServerless()', () => {
|
|||
|
||||
const cluster = new Cluster({ log });
|
||||
await cluster.run(installPath, esClusterExecOptions);
|
||||
await expect(cluster.runServerless({ basePath: installPath })).rejects.toThrowError(
|
||||
await expect(cluster.runServerless(defaultOptions)).rejects.toThrowError(
|
||||
'ES stateful cluster has already been started'
|
||||
);
|
||||
});
|
||||
|
|
@ -756,7 +760,7 @@ describe('#runServerless()', () => {
|
|||
);
|
||||
|
||||
const cluster = new Cluster({ log });
|
||||
const promise = cluster.runServerless({ basePath: installPath });
|
||||
const promise = cluster.runServerless(defaultOptions);
|
||||
await ensureNoResolve(promise);
|
||||
resolveRunServerlessCluster!();
|
||||
await expect(ensureResolve(promise, 'runServerless()')).resolves.toEqual(nodeNames);
|
||||
|
|
@ -767,8 +771,8 @@ describe('#runServerless()', () => {
|
|||
runServerlessClusterMock.mockResolvedValueOnce(nodeNames);
|
||||
|
||||
const cluster = new Cluster({ log });
|
||||
await cluster.runServerless({ basePath: installPath });
|
||||
await expect(cluster.runServerless({ basePath: installPath })).rejects.toThrowError(
|
||||
await cluster.runServerless(defaultOptions);
|
||||
await expect(cluster.runServerless(defaultOptions)).rejects.toThrowError(
|
||||
'ES serverless docker cluster has already been started'
|
||||
);
|
||||
});
|
||||
|
|
@ -776,7 +780,7 @@ describe('#runServerless()', () => {
|
|||
test('rejects if #runServerlessCluster() rejects', async () => {
|
||||
runServerlessClusterMock.mockRejectedValueOnce(new Error('foo'));
|
||||
const cluster = new Cluster({ log });
|
||||
await expect(cluster.runServerless({ basePath: installPath })).rejects.toThrowError('foo');
|
||||
await expect(cluster.runServerless(defaultOptions)).rejects.toThrowError('foo');
|
||||
});
|
||||
|
||||
test('passes through all options+log to #runServerlessCluster()', async () => {
|
||||
|
|
@ -785,6 +789,7 @@ describe('#runServerless()', () => {
|
|||
|
||||
const cluster = new Cluster({ log });
|
||||
const serverlessOptions = {
|
||||
projectType: 'es' as dockerUtils.ServerlessProjectType,
|
||||
clean: true,
|
||||
basePath: installPath,
|
||||
teardown: true,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
verifyDockerInstalled,
|
||||
getESp12Volume,
|
||||
ServerlessOptions,
|
||||
ServerlessProjectType,
|
||||
} from './docker';
|
||||
import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
|
||||
import { CA_CERT_PATH, ES_P12_PATH } from '@kbn/dev-utils';
|
||||
|
|
@ -66,6 +67,7 @@ const logWriter = new ToolingLogCollectingWriter();
|
|||
log.setWriters([logWriter]);
|
||||
|
||||
const KIBANA_ROOT = process.cwd();
|
||||
const projectType: ServerlessProjectType = 'es';
|
||||
const baseEsPath = `${KIBANA_ROOT}/.es`;
|
||||
const serverlessDir = 'stateless';
|
||||
const serverlessObjectStorePath = `${baseEsPath}/${serverlessDir}`;
|
||||
|
|
@ -504,7 +506,10 @@ describe('setupServerlessVolumes()', () => {
|
|||
[baseEsPath]: {},
|
||||
});
|
||||
|
||||
const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath });
|
||||
const volumeCmd = await setupServerlessVolumes(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
});
|
||||
|
||||
volumeCmdTest(volumeCmd);
|
||||
await expect(Fsp.access(serverlessObjectStorePath)).resolves.not.toThrow();
|
||||
|
|
@ -513,7 +518,7 @@ describe('setupServerlessVolumes()', () => {
|
|||
test('should use an existing object store', async () => {
|
||||
mockFs(existingObjectStore);
|
||||
|
||||
const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath });
|
||||
const volumeCmd = await setupServerlessVolumes(log, { projectType, basePath: baseEsPath });
|
||||
|
||||
volumeCmdTest(volumeCmd);
|
||||
await expect(
|
||||
|
|
@ -524,7 +529,11 @@ describe('setupServerlessVolumes()', () => {
|
|||
test('should remove an existing object store when clean is passed', async () => {
|
||||
mockFs(existingObjectStore);
|
||||
|
||||
const volumeCmd = await setupServerlessVolumes(log, { basePath: baseEsPath, clean: true });
|
||||
const volumeCmd = await setupServerlessVolumes(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
clean: true,
|
||||
});
|
||||
|
||||
volumeCmdTest(volumeCmd);
|
||||
await expect(
|
||||
|
|
@ -537,6 +546,7 @@ describe('setupServerlessVolumes()', () => {
|
|||
createMockIdpMetadataMock.mockResolvedValue('<xml/>');
|
||||
|
||||
const volumeCmd = await setupServerlessVolumes(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
ssl: true,
|
||||
kibanaUrl: 'https://localhost:5603/',
|
||||
|
|
@ -561,6 +571,7 @@ describe('setupServerlessVolumes()', () => {
|
|||
test('should use resource overrides', async () => {
|
||||
mockFs(existingObjectStore);
|
||||
const volumeCmd = await setupServerlessVolumes(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
resources: ['./relative/path/users', '/absolute/path/users_roles'],
|
||||
});
|
||||
|
|
@ -578,6 +589,7 @@ describe('setupServerlessVolumes()', () => {
|
|||
|
||||
await expect(async () => {
|
||||
await setupServerlessVolumes(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
resources: ['/absolute/path/invalid'],
|
||||
});
|
||||
|
|
@ -626,7 +638,7 @@ describe('runServerlessCluster()', () => {
|
|||
});
|
||||
execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
|
||||
|
||||
await runServerlessCluster(log, { basePath: baseEsPath });
|
||||
await runServerlessCluster(log, { projectType, basePath: baseEsPath });
|
||||
|
||||
// setupDocker execa calls then run three nodes and attach logger
|
||||
expect(execa.mock.calls).toHaveLength(8);
|
||||
|
|
@ -639,7 +651,7 @@ describe('runServerlessCluster()', () => {
|
|||
});
|
||||
execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
|
||||
|
||||
await runServerlessCluster(log, { basePath: baseEsPath, waitForReady: true });
|
||||
await runServerlessCluster(log, { projectType, basePath: baseEsPath, waitForReady: true });
|
||||
expect(waitUntilClusterReadyMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitUntilClusterReadyMock.mock.calls[0][0].expectedStatus).toEqual('green');
|
||||
expect(waitUntilClusterReadyMock.mock.calls[0][0].readyTimeout).toEqual(undefined);
|
||||
|
|
@ -655,6 +667,7 @@ describe('runServerlessCluster()', () => {
|
|||
createMockIdpMetadataMock.mockResolvedValue('<xml/>');
|
||||
|
||||
await runServerlessCluster(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
waitForReady: true,
|
||||
ssl: true,
|
||||
|
|
@ -672,7 +685,7 @@ describe('runServerlessCluster()', () => {
|
|||
});
|
||||
execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
|
||||
|
||||
await runServerlessCluster(log, { basePath: baseEsPath, waitForReady: true });
|
||||
await runServerlessCluster(log, { projectType, basePath: baseEsPath, waitForReady: true });
|
||||
expect(waitForSecurityIndexMock).toHaveBeenCalledTimes(1);
|
||||
expect(waitForSecurityIndexMock.mock.calls[0][0].readyTimeout).toEqual(undefined);
|
||||
});
|
||||
|
|
@ -685,6 +698,7 @@ describe('runServerlessCluster()', () => {
|
|||
execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
|
||||
|
||||
await runServerlessCluster(log, {
|
||||
projectType,
|
||||
basePath: baseEsPath,
|
||||
waitForReady: true,
|
||||
esArgs: ['xpack.security.enabled=false'],
|
||||
|
|
@ -708,7 +722,7 @@ describe('stopServerlessCluster()', () => {
|
|||
});
|
||||
|
||||
describe('teardownServerlessClusterSync()', () => {
|
||||
const defaultOptions = { basePath: 'foo/bar' };
|
||||
const defaultOptions = { projectType, basePath: 'foo/bar' };
|
||||
|
||||
test('should kill running serverless nodes', () => {
|
||||
const nodes = ['es01', 'es02', 'es03'];
|
||||
|
|
|
|||
|
|
@ -59,8 +59,8 @@ interface BaseOptions extends ImageOptions {
|
|||
files?: string | string[];
|
||||
}
|
||||
|
||||
const serverlessProjectTypes = new Set<string>(['es', 'oblt', 'security']);
|
||||
const isServerlessProjectType = (value: string): value is ServerlessProjectType => {
|
||||
export const serverlessProjectTypes = new Set<string>(['es', 'oblt', 'security']);
|
||||
export const isServerlessProjectType = (value: string): value is ServerlessProjectType => {
|
||||
return serverlessProjectTypes.has(value);
|
||||
};
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ export interface ServerlessOptions extends EsClusterExecOptions, BaseOptions {
|
|||
/** Publish ES docker container on additional host IP */
|
||||
host?: string;
|
||||
/** Serverless project type */
|
||||
projectType?: ServerlessProjectType;
|
||||
projectType: ServerlessProjectType;
|
||||
/** Clean (or delete) all data created by the ES cluster after it is stopped */
|
||||
clean?: boolean;
|
||||
/** Path to the directory where the ES cluster will store data */
|
||||
|
|
@ -599,16 +599,8 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
|
|||
}, {} as Record<string, string>)
|
||||
: {};
|
||||
|
||||
// Check if projectType is valid
|
||||
if (projectType && !isServerlessProjectType(projectType)) {
|
||||
throw new Error(
|
||||
`Incorrect serverless project type: ${projectType}, use one of ${Array.from(
|
||||
serverlessProjectTypes
|
||||
).join(', ')}`
|
||||
);
|
||||
}
|
||||
// Read roles for the specified projectType, 'es' if it is not defined
|
||||
const rolesResourcePath = resolve(SERVERLESS_ROLES_ROOT_PATH, projectType ?? 'es', 'roles.yml');
|
||||
// Read roles for the specified projectType
|
||||
const rolesResourcePath = resolve(SERVERLESS_ROLES_ROOT_PATH, projectType, 'roles.yml');
|
||||
|
||||
const resourcesPaths = [...SERVERLESS_RESOURCES_PATHS, rolesResourcePath];
|
||||
|
||||
|
|
|
|||
|
|
@ -242,6 +242,11 @@ export function createTestEsCluster<
|
|||
} else if (esFrom === 'snapshot') {
|
||||
installPath = (await firstNode.installSnapshot(config)).installPath;
|
||||
} else if (esFrom === 'serverless') {
|
||||
if (!esServerlessOptions) {
|
||||
throw new Error(
|
||||
`'esServerlessOptions' must be defined to start Elasticsearch in serverless mode`
|
||||
);
|
||||
}
|
||||
await firstNode.runServerless({
|
||||
basePath,
|
||||
esArgs: customEsArgs,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type { ToolingLog } from '@kbn/tooling-log';
|
|||
import getPort from 'get-port';
|
||||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import type { ArtifactLicense, ServerlessProjectType } from '@kbn/es';
|
||||
import { isServerlessProjectType } from '@kbn/es/src/utils';
|
||||
import type { Config } from '../../functional_test_runner';
|
||||
import { createTestEsCluster, esTestConfig } from '../../es';
|
||||
|
||||
|
|
@ -53,7 +54,9 @@ function getEsConfig({
|
|||
const serverless: boolean = config.get('serverless');
|
||||
const files: string[] | undefined = config.get('esTestCluster.files');
|
||||
|
||||
const esServerlessOptions = getESServerlessOptions(esServerlessImage, config);
|
||||
const esServerlessOptions = serverless
|
||||
? getESServerlessOptions(esServerlessImage, config)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
ssl,
|
||||
|
|
@ -162,6 +165,7 @@ async function startEsNode({
|
|||
}
|
||||
|
||||
interface EsServerlessOptions {
|
||||
projectType: ServerlessProjectType;
|
||||
host?: string;
|
||||
resources: string[];
|
||||
kibanaUrl: string;
|
||||
|
|
@ -189,19 +193,20 @@ function getESServerlessOptions(
|
|||
(config.get('kbnTestServer.serverArgs') as string[])) ||
|
||||
[];
|
||||
|
||||
const projectTypeFromArgs = kbnServerArgs
|
||||
const projectType = kbnServerArgs
|
||||
.filter((arg) => arg.startsWith('--serverless'))
|
||||
.reduce((acc, arg) => {
|
||||
const match = arg.match(/--serverless[=\s](\w+)/);
|
||||
return acc + (match ? match[1] : '');
|
||||
}, '');
|
||||
const projectType = projectTypeFromArgs.length
|
||||
? (projectTypeFromArgs as ServerlessProjectType)
|
||||
: undefined;
|
||||
}, '') as ServerlessProjectType;
|
||||
|
||||
if (!isServerlessProjectType(projectType)) {
|
||||
throw new Error(`Unsupported serverless projectType: ${projectType}`);
|
||||
}
|
||||
|
||||
const commonOptions = {
|
||||
host: serverlessHost,
|
||||
projectType,
|
||||
host: serverlessHost,
|
||||
resources: serverlessResources,
|
||||
kibanaUrl: Url.format({
|
||||
protocol: config.get('servers.kibana.protocol'),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue