github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[testGroupRunOrder] allow defining a queue name for each ftr config (#135349)

Browse source
This commit is contained in:
Spencer 2022-06-30 16:12:06 -05:00 • committed by GitHub
parent 7fd3beee4e
commit ed51e0fe0a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 190 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/ftr_configs.yml
View file

@ -56,6 +56,7 @@ disabled:
- x-pack/test/plugin_api_perf/config.js
- x-pack/test/screenshot_creation/config.ts
defaultQueue: 'n2-4-spot'
enabled:
- test/accessibility/config.ts
- test/analytics/config.ts

2
.buildkite/pipeline-utils/ci-stats/client.ts
View file

@ -43,6 +43,7 @@ export interface TestGroupRunOrderResponse {
types: Array<{
type: string;
count: number;
queue?: string;
groups: Array<{
durationMin: number;
names: string[];
@ -159,6 +160,7 @@ export class CiStatsClient {
>;
groups: Array<{
type: string;
queue?: string;
defaultMin?: number;
maxMin: number;
minimumIsolationMin?: number;

231
.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
View file

@ -25,52 +25,58 @@ const getRequiredEnv = (name: string) => {
return value;
};
function getRunGroup(bk: BuildkiteClient, types: RunGroup[], typeName: string): RunGroup {
const type = types.find((t) => t.type === typeName);
if (!type) {
function getRunGroups(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup[] {
const types = allTypes.filter((t) => t.type === typeName);
if (!types.length) {
throw new Error(`missing test group run order for group [${typeName}]`);
}
const misses = type.namesWithoutDurations.length;
if (misses > 0) {
const misses = types.flatMap((t) => t.namesWithoutDurations);
if (misses.length > 0) {
bk.setAnnotation(
`test-group-missing-durations:${typeName}`,
'warning',
[
misses === 1
misses.length === 1
? `The following "${typeName}" config doesn't have a recorded time in ci-stats so the automatically-determined test groups might be a little unbalanced.`
: `The following "${typeName}" configs don't have recorded times in ci-stats so the automatically-determined test groups might be a little unbalanced.`,
misses === 1
misses.length === 1
? `If this is a new config then this warning can be ignored as times will be reported soon.`
: `If these are new configs then this warning can be ignored as times will be reported soon.`,
misses === 1
misses.length === 1
? `The other possibility is that there aren't any tests in this config, so times are never reported.`
: `The other possibility is that there aren't any tests in these configs, so times are never reported.`,
'Empty test configs should be removed',
'',
...type.namesWithoutDurations.map((n) => ` - ${n}`),
...misses.map((n) => ` - ${n}`),
].join('\n')
);
}
const tooLongs = type.tooLong?.length ?? 0;
if (tooLongs > 0) {
const tooLongs = types.flatMap((t) => t.tooLong ?? []);
if (tooLongs.length > 0) {
bk.setAnnotation(
`test-group-too-long:${typeName}`,
'error',
[
tooLongs === 1
tooLongs.length === 1
? `The following "${typeName}" config has a duration that exceeds the maximum amount of time desired for a single CI job. Please split it up.`
: `The following "${typeName}" configs have durations that exceed the maximum amount of time desired for a single CI job. Please split them up.`,
'',
...(type.tooLong ?? []).map(
({ config, durationMin }) => ` - ${config}: ${durationMin} minutes`
),
...tooLongs.map(({ config, durationMin }) => ` - ${config}: ${durationMin} minutes`),
].join('\n')
);
}
return type;
return types;
}
function getRunGroup(bk: BuildkiteClient, allTypes: RunGroup[], typeName: string): RunGroup {
const groups = getRunGroups(bk, allTypes, typeName);
if (groups.length !== 1) {
throw new Error(`expected to find exactly 1 "${typeName}" run group`);
}
return groups[0];
}
function getTrackedBranch(): string {
@ -105,16 +111,39 @@ function getEnabledFtrConfigs(patterns?: string[]) {
}
if (
!Array.isArray(configs.enabled) ||
!configs.enabled.every((p): p is string => typeof p === 'string')
!configs.enabled.every(
(p): p is string | { [configPath: string]: { queue: string } } =>
typeof p === 'string' ||
(isObj(p) && Object.values(p).every((v) => isObj(v) && typeof v.queue === 'string'))
)
) {
throw new Error('expected "enabled" value to be an array of strings');
throw new Error(`expected "enabled" value to be an array of strings or objects shaped as:\n
- {configPath}:
queue: {queueName}`);
}
if (typeof configs.defaultQueue !== 'string') {
throw new Error('expected yaml file to have a string "defaultQueue" key');
}
if (!patterns) {
return configs.enabled;
const defaultQueue = configs.defaultQueue;
const ftrConfigsByQueue = new Map<string, string[]>();
for (const enabled of configs.enabled) {
const path = typeof enabled === 'string' ? enabled : Object.keys(enabled)[0];
const queue = isObj(enabled) ? enabled[path].queue : defaultQueue;
if (patterns && !patterns.some((pattern) => minimatch(path, pattern))) {
continue;
}
const group = ftrConfigsByQueue.get(queue);
if (group) {
group.push(path);
} else {
ftrConfigsByQueue.set(queue, [path]);
}
}
return configs.enabled.filter((path) => patterns.some((pattern) => minimatch(path, pattern)));
return { defaultQueue, ftrConfigsByQueue };
} catch (_) {
const error = _ instanceof Error ? _ : new Error(`${_} thrown`);
throw new Error(`unable to parse ftr_configs.yml file: ${error.message}`);
@ -182,9 +211,10 @@ export async function pickTestGroupRunOrder() {
.filter(Boolean)
: ['build'];
const ftrConfigs = LIMIT_CONFIG_TYPE.includes('functional')
? getEnabledFtrConfigs(FTR_CONFIG_PATTERNS)
: [];
const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
if (!LIMIT_CONFIG_TYPE.includes('functional')) {
ftrConfigsByQueue.clear();
}
const jestUnitConfigs = LIMIT_CONFIG_TYPE.includes('unit')
? globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], {
@ -200,7 +230,7 @@ export async function pickTestGroupRunOrder() {
})
: [];
if (!ftrConfigs.length && !jestUnitConfigs.length && !jestIntegrationConfigs.length) {
if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) {
throw new Error('unable to find any unit, integration, or FTR configs');
}
@ -268,14 +298,15 @@ export async function pickTestGroupRunOrder() {
overheadMin: 0.2,
names: jestIntegrationConfigs,
},
{
...Array.from(ftrConfigsByQueue).map(([queue, names]) => ({
type: FUNCTIONAL_TYPE,
defaultMin: 60,
queue,
maxMin: FUNCTIONAL_MAX_MINUTES,
minimumIsolationMin: FUNCTIONAL_MINIMUM_ISOLATION_MIN,
overheadMin: 1.5,
names: ftrConfigs,
},
names,
})),
],
});
@ -284,21 +315,62 @@ export async function pickTestGroupRunOrder() {
const unit = getRunGroup(bk, types, UNIT_TYPE);
const integration = getRunGroup(bk, types, INTEGRATION_TYPE);
const functional = getRunGroup(bk, types, FUNCTIONAL_TYPE);
let configCounter = 0;
let groupCounter = 0;
// the relevant data we will use to define the pipeline steps
const functionalGroups: Array<{
title: string;
key: string;
sortBy: number | string;
queue: string;
}> = [];
// the map that we will write to the artifacts for informing ftr config jobs of what they should do
const ftrRunOrder: Record<
string,
{ title: string; expectedDurationMin: number; names: string[] }
> = {};
for (const { groups, queue } of getRunGroups(bk, types, FUNCTIONAL_TYPE)) {
for (const group of groups) {
if (!group.names.length) {
continue;
}
const key = `ftr_configs_${configCounter++}`;
let sortBy;
let title;
if (group.names.length === 1) {
title = group.names[0];
sortBy = title;
} else {
sortBy = ++groupCounter;
title = `FTR Configs #${sortBy}`;
}
functionalGroups.push({
title,
key,
sortBy,
queue: queue ?? defaultQueue,
});
ftrRunOrder[key] = {
title,
expectedDurationMin: group.durationMin,
names: group.names,
};
}
}
// write the config for each step to an artifact that can be used by the individual jest jobs
Fs.writeFileSync('jest_run_order.json', JSON.stringify({ unit, integration }, null, 2));
bk.uploadArtifacts('jest_run_order.json');
// write the config for functional steps to an artifact that can be used by the individual functional jobs
Fs.writeFileSync('ftr_run_order.json', JSON.stringify(functional, null, 2));
Fs.writeFileSync('ftr_run_order.json', JSON.stringify(ftrRunOrder, null, 2));
bk.uploadArtifacts('ftr_run_order.json');
let smallFtrConfigsCounter = 0;
const getSmallFtrConfigsLabel = () => {
return `Super Quick FTR Configs #${++smallFtrConfigsCounter}`;
};
// upload the step definitions to Buildkite
bk.uploadSteps(
[
@ -342,55 +414,46 @@ export async function pickTestGroupRunOrder() {
},
}
: [],
functional.count > 0
? FUNCTIONAL_MINIMUM_ISOLATION_MIN === undefined
? {
label: 'FTR Configs',
key: 'ftr-configs',
depends_on: FTR_CONFIGS_DEPS,
parallelism: functional.count,
command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
timeout_in_minutes: 150,
agents: {
queue: 'n2-4-spot-2',
},
retry: {
automatic: [
{ exit_status: '-1', limit: 3 },
...(FTR_CONFIGS_RETRY_COUNT > 0
? [{ exit_status: '*', limit: FTR_CONFIGS_RETRY_COUNT }]
: []),
],
},
}
: {
group: 'FTR Configs',
key: 'ftr-configs',
depends_on: FTR_CONFIGS_DEPS,
steps: functional.groups
.map(
(group, i): BuildkiteStep => ({
label: group.names.length === 1 ? group.names[0] : getSmallFtrConfigsLabel(),
command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
timeout_in_minutes: 150,
agents: {
queue: 'n2-4-spot-2',
},
env: {
FTR_CONFIG_GROUP_INDEX: `${i}`,
},
retry: {
automatic: [
{ exit_status: '-1', limit: 3 },
...(FTR_CONFIGS_RETRY_COUNT > 0
? [{ exit_status: '*', limit: FTR_CONFIGS_RETRY_COUNT }]
: []),
],
},
})
)
.sort((a, b) => a.label.localeCompare(b.label)),
}
functionalGroups.length
? {
group: 'FTR Configs',
key: 'ftr-configs',
depends_on: FTR_CONFIGS_DEPS,
steps: functionalGroups
.sort((a, b) =>
// if both groups are sorted by number then sort by that
typeof a.sortBy === 'number' && typeof b.sortBy === 'number'
? a.sortBy - b.sortBy
: // if both groups are sorted by string, sort by that
typeof a.sortBy === 'string' && typeof b.sortBy === 'string'
? a.sortBy.localeCompare(b.sortBy)
: // if a is sorted by number then order it later than b
typeof a.sortBy === 'number'
? 1
: -1
)
.map(
({ title, key, queue = defaultQueue }): BuildkiteStep => ({
label: title,
command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
timeout_in_minutes: 150,
agents: {
queue,
},
env: {
FTR_CONFIG_GROUP_KEY: key,
},
retry: {
automatic: [
{ exit_status: '-1', limit: 3 },
...(FTR_CONFIGS_RETRY_COUNT > 0
? [{ exit_status: '*', limit: FTR_CONFIGS_RETRY_COUNT }]
: []),
],
},
})
),
}
: [],
].flat()
);

16
.buildkite/scripts/steps/code_coverage/ftr_configs.sh
View file

@ -13,20 +13,23 @@ echo "--- KIBANA_DIR: $KIBANA_DIR"
buildPlatformPlugins
is_test_execution_step
export JOB_NUM=$BUILDKITE_PARALLEL_JOB
export JOB=ftr-configs-${JOB_NUM}
FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
if [ "$FTR_CONFIG_GROUP_KEY" == "" ]; then
echo "Missing FTR_CONFIG_GROUP_KEY env var"
exit 1
fi
export JOB="$FTR_CONFIG_GROUP_KEY"
functionalTarget="$KIBANA_DIR/target/kibana-coverage/functional"
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}"
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
# a FTR failure will result in the script returning an exit code of 10
exitCode=0
configs="${FTR_CONFIG:-}"
if [[ "$configs" == "" ]]; then
echo "--- Downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi
echo "--- Config(s) for this FTR Group:"
@ -77,7 +80,6 @@ while read -r config; do
result: ${lastCode}")
if [ $lastCode -ne 0 ]; then
exitCode=10
echo "FTR exited with code $lastCode"
echo "^^^ +++"

14
.buildkite/scripts/steps/test/ftr_configs.sh
View file

@ -4,10 +4,16 @@ set -euo pipefail
source .buildkite/scripts/steps/functional/common.sh
export JOB_NUM=${BUILDKITE_PARALLEL_JOB:-0}
export JOB=ftr-configs-${JOB_NUM}
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${JOB_NUM}"
FTR_CONFIG_GROUP_KEY=${FTR_CONFIG_GROUP_KEY:-}
if [ "$FTR_CONFIG_GROUP_KEY" == "" ]; then
echo "Missing FTR_CONFIG_GROUP_KEY env var"
exit 1
fi
export JOB="$FTR_CONFIG_GROUP_KEY"
FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
# a FTR failure will result in the script returning an exit code of 10
exitCode=0
@ -27,7 +33,7 @@ fi
if [[ "$configs" == "" ]]; then
echo "--- downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
configs=$(jq -r '.groups[env.JOB_NUM | tonumber].names | .[]' ftr_run_order.json)
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi
failedConfigs=""

24
packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts
View file

@ -14,10 +14,28 @@ import JsYaml from 'js-yaml';
export const FTR_CONFIGS_MANIFEST_REL = '.buildkite/ftr_configs.yml';
const ftrConfigsManifest = JsYaml.safeLoad(
interface FtrConfigWithOptions {
[configPath: string]: {
queue: string;
};
}
interface FtrConfigsManifest {
defaultQueue: string;
disabled: string[];
enabled: Array<string | FtrConfigWithOptions>;
}
const ftrConfigsManifest: FtrConfigsManifest = JsYaml.safeLoad(
Fs.readFileSync(Path.resolve(REPO_ROOT, FTR_CONFIGS_MANIFEST_REL), 'utf8')
);
export const FTR_CONFIGS_MANIFEST_PATHS = (Object.values(ftrConfigsManifest) as string[][])
export const FTR_CONFIGS_MANIFEST_PATHS = [
Object.values(ftrConfigsManifest.enabled),
Object.values(ftrConfigsManifest.disabled),
]
.flat()
.map((rel) => Path.resolve(REPO_ROOT, rel));
.map((config) => {
const rel = typeof config === 'string' ? config : Object.keys(config)[0];
return Path.resolve(REPO_ROOT, rel);
});