mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[ES Archiver][Load Action] Add perf option override to cli (#175781)
## Summary With the merge of https://github.com/elastic/kibana/pull/174631, we've need of a follow up pr, to extend the performance override, to the command line interface. Also, modify the type of the performance option override, by making both the batch size and the concurrency optional. - With this change, if one or the other of the two options (batch size & concurrency) are not provided, the defaults are used. ### How to Test locally Start the server in one terminal, such as: `$ node scripts/functional_tests_server.js --config test/functional/apps/console/config.ts` Then in a second terminal: `$ node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --batch-size 300 --concurrency 2` #### Additional Testing (regarding only using one option or the other) _Just batch size_ `$ node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --batch-size 300` _Just concurrency_ `$ node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --concurrency 2` #### To Slow the Load Action Down (good for testing) Load an archive with very low numbers `node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --concurrency 1 --batch-size 10` You should see the progress indicator due to the forced latency ``` info [x-pack/test/functional/es_archives/logstash_functional] Created index "logstash-2015.09.20" info [x-pack/test/functional/es_archives/logstash_functional] Deleted existing index "logstash-2015.09.21" info [x-pack/test/functional/es_archives/logstash_functional] Created index "logstash-2015.09.21" info progress: 2940 info progress: 6040 info progress: 9380 info progress: 12451 info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4634 docs into "logstash-2015.09.22" info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4757 docs into "logstash-2015.09.20" info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4614 docs into "logstash-2015.09.21" ``` --------- Co-authored-by: Robert Oskamp <traeluki@gmail.com>
This commit is contained in:
parent
e9dc10e97d
commit
b76b5c3b64
2 changed files with 20 additions and 14 deletions
|
@ -18,11 +18,10 @@ import readline from 'readline';
|
|||
import Fs from 'fs';
|
||||
|
||||
import { CA_CERT_PATH } from '@kbn/dev-utils';
|
||||
import { RunWithCommands } from '@kbn/dev-cli-runner';
|
||||
import { FlagsReader, RunWithCommands } from '@kbn/dev-cli-runner';
|
||||
import { createFlagError } from '@kbn/dev-cli-errors';
|
||||
import { readConfigFile, KbnClient, EsVersion } from '@kbn/test';
|
||||
import { Client, HttpConnection } from '@elastic/elasticsearch';
|
||||
|
||||
import { EsArchiver } from './es_archiver';
|
||||
|
||||
const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v);
|
||||
|
@ -202,12 +201,20 @@ export function runCli() {
|
|||
WARNING: If the indices exist already they will be deleted!
|
||||
|
||||
$ node scripts/es_archiver load my_test_data --config ../config.js
|
||||
|
||||
The same with overrides for batch size and concurrency default values:
|
||||
|
||||
$ node scripts/es_archiver load my_test_data --config ../config.js --batch-size 300 --concurrency 2
|
||||
|
||||
`,
|
||||
flags: {
|
||||
boolean: ['use-create', 'docs-only'],
|
||||
string: ['batch-size', 'concurrency'],
|
||||
help: `
|
||||
--use-create use create instead of index for loading documents
|
||||
--docs-only load only documents, not indices
|
||||
--batch-size the "high water mark"; the number of records processed per bulk request
|
||||
--concurrency number of bulk requests made by the api
|
||||
`,
|
||||
},
|
||||
async run({ flags, esArchiver, statsMeta }) {
|
||||
|
@ -221,17 +228,16 @@ export function runCli() {
|
|||
|
||||
statsMeta.set('esArchiverPath', path);
|
||||
|
||||
const useCreate = flags['use-create'];
|
||||
if (typeof useCreate !== 'boolean') {
|
||||
throw createFlagError('--use-create does not take a value');
|
||||
}
|
||||
const flagsReader = new FlagsReader(flags);
|
||||
|
||||
const docsOnly = flags['docs-only'];
|
||||
if (typeof docsOnly !== 'boolean') {
|
||||
throw createFlagError('--docs-only does not take a value');
|
||||
}
|
||||
|
||||
await esArchiver.load(path, { useCreate, docsOnly });
|
||||
await esArchiver.load(path, {
|
||||
useCreate: flagsReader.boolean('use-create'),
|
||||
docsOnly: flagsReader.boolean('docs-only'),
|
||||
performance: {
|
||||
batchSize: flagsReader.number('batch-size'),
|
||||
concurrency: flagsReader.number('concurrency'),
|
||||
},
|
||||
});
|
||||
},
|
||||
})
|
||||
.command({
|
||||
|
|
|
@ -96,8 +96,8 @@ export function createIndexDocRecordsStream(
|
|||
}
|
||||
|
||||
export interface LoadActionPerfOptions {
|
||||
batchSize: number;
|
||||
concurrency: number;
|
||||
batchSize?: number;
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_PERFORMANCE_OPTIONS: LoadActionPerfOptions = {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue