[ES Archiver][Load Action] Add perf option override to cli (#175781)

## Summary

With the merge of https://github.com/elastic/kibana/pull/174631, we've
need of a follow up pr,
to extend the performance override, to the command line interface.

Also, modify the type of the performance option override, by making both
the batch size and the concurrency optional.
- With this change, if one or the other of the two options (batch size &
concurrency) are not provided, the defaults are used.

### How to Test locally

Start the server in one terminal, such as:

`$ node scripts/functional_tests_server.js --config
test/functional/apps/console/config.ts`

Then in a second terminal:

`$ node scripts/es_archiver.js load
x-pack/test/functional/es_archives/logstash_functional --batch-size 300
--concurrency 2`

#### Additional Testing (regarding only using one option or the other)

_Just batch size_
`$ node scripts/es_archiver.js load
x-pack/test/functional/es_archives/logstash_functional --batch-size 300`

_Just concurrency_
`$ node scripts/es_archiver.js load
x-pack/test/functional/es_archives/logstash_functional --concurrency 2`

#### To Slow the Load Action Down (good for testing)

Load an archive with very low numbers

`node scripts/es_archiver.js load
x-pack/test/functional/es_archives/logstash_functional --concurrency 1
--batch-size 10`

You should see the progress indicator due to the forced latency

```
 info [x-pack/test/functional/es_archives/logstash_functional] Created index "logstash-2015.09.20"
 info [x-pack/test/functional/es_archives/logstash_functional] Deleted existing index "logstash-2015.09.21"
 info [x-pack/test/functional/es_archives/logstash_functional] Created index "logstash-2015.09.21"
 info progress: 2940
 info progress: 6040
 info progress: 9380
 info progress: 12451
 info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4634 docs into "logstash-2015.09.22"
 info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4757 docs into "logstash-2015.09.20"
 info [x-pack/test/functional/es_archives/logstash_functional] Indexed 4614 docs into "logstash-2015.09.21"
```

---------

Co-authored-by: Robert Oskamp <traeluki@gmail.com>
This commit is contained in:
Tre 2024-02-02 14:24:04 +00:00 committed by GitHub
parent e9dc10e97d
commit b76b5c3b64
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 20 additions and 14 deletions

View file

@ -18,11 +18,10 @@ import readline from 'readline';
import Fs from 'fs';
import { CA_CERT_PATH } from '@kbn/dev-utils';
import { RunWithCommands } from '@kbn/dev-cli-runner';
import { FlagsReader, RunWithCommands } from '@kbn/dev-cli-runner';
import { createFlagError } from '@kbn/dev-cli-errors';
import { readConfigFile, KbnClient, EsVersion } from '@kbn/test';
import { Client, HttpConnection } from '@elastic/elasticsearch';
import { EsArchiver } from './es_archiver';
const resolveConfigPath = (v: string) => Path.resolve(process.cwd(), v);
@ -202,12 +201,20 @@ export function runCli() {
WARNING: If the indices exist already they will be deleted!
$ node scripts/es_archiver load my_test_data --config ../config.js
The same with overrides for batch size and concurrency default values:
$ node scripts/es_archiver load my_test_data --config ../config.js --batch-size 300 --concurrency 2
`,
flags: {
boolean: ['use-create', 'docs-only'],
string: ['batch-size', 'concurrency'],
help: `
--use-create use create instead of index for loading documents
--docs-only load only documents, not indices
--batch-size the "high water mark"; the number of records processed per bulk request
--concurrency number of bulk requests made by the api
`,
},
async run({ flags, esArchiver, statsMeta }) {
@ -221,17 +228,16 @@ export function runCli() {
statsMeta.set('esArchiverPath', path);
const useCreate = flags['use-create'];
if (typeof useCreate !== 'boolean') {
throw createFlagError('--use-create does not take a value');
}
const flagsReader = new FlagsReader(flags);
const docsOnly = flags['docs-only'];
if (typeof docsOnly !== 'boolean') {
throw createFlagError('--docs-only does not take a value');
}
await esArchiver.load(path, { useCreate, docsOnly });
await esArchiver.load(path, {
useCreate: flagsReader.boolean('use-create'),
docsOnly: flagsReader.boolean('docs-only'),
performance: {
batchSize: flagsReader.number('batch-size'),
concurrency: flagsReader.number('concurrency'),
},
});
},
})
.command({

View file

@ -96,8 +96,8 @@ export function createIndexDocRecordsStream(
}
export interface LoadActionPerfOptions {
batchSize: number;
concurrency: number;
batchSize?: number;
concurrency?: number;
}
const DEFAULT_PERFORMANCE_OPTIONS: LoadActionPerfOptions = {