github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Attempt to fix flaky tests after split migration (#159397)

Browse source 
## Summary
Addresses root cause of https://github.com/elastic/kibana/issues/158918

Underlying cause is that _esArchiver_ is messing up with the SO indices
whilst Kibana is already running.

This can cause some asynchronous calls made by Kibana (e.g. `GET
/.kibana_8.8.0/telemetry:telemetry`) to hit ES at the exact time where
the underlying SO indices are **just** recreated, causing the error
described in the related issue.

The idea of the fix is to delete `mappings.json`, used by _esArchiver_
to create the SO indices. This way, _esArchiver_ will use existing SO
indices instead (aka the "official" ones, created by Kibana at startup),
thus avoiding the problem altogether.

As a side effect:

- Documents in `data.json` must be updated so that they are correctly
inserted.
- The different FTR tests must make sure the SO indices are empty before
inserting those documents (done in the `before(), beforeEach()`
statements).
This commit is contained in:
Gerard Soldevila 2023-07-07 17:05:42 +02:00 • committed by GitHub
parent d9c0c554f8
commit 79f7bb45fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 118 additions and 321 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/kbn-es-archiver/src/actions/unload.ts
View file

@ -46,7 +46,7 @@ export async function unloadAction({
await createPromiseFromStreams([
createReadStream(resolve(inputDir, filename)) as Readable,
...createParseArchiveStreams({ gzip: isGzip(filename) }),
createFilterRecordsStream((record) => ['index', 'data_stream'].includes(record.type)),
createFilterRecordsStream((record) => ['index', 'data_stream', 'doc'].includes(record.type)),
createDeleteIndexStream(client, stats, log),
] as [Readable, ...Writable[]]);
}

46
packages/kbn-es-archiver/src/lib/indices/delete_index_stream.test.ts
View file

@ -20,6 +20,7 @@ import {
createStubIndexRecord,
createStubDataStreamRecord,
createStubLogger,
createStubDocRecord,
} from './__mocks__/stubs';
const log = createStubLogger();
@ -79,4 +80,49 @@ describe('esArchiver: createDeleteIndexStream()', () => {
name: 'foo-template',
});
});
describe('saved object cleanup', () => {
describe('when saved object documents are found', () => {
it('cleans the corresponding saved object indices', async () => {
const client = createStubClient();
const stats = createStubStats();
await createPromiseFromStreams([
createListStream([
createStubDocRecord('.kibana_task_manager', 1),
createStubDocRecord('.kibana_alerting_cases', 2),
createStubDocRecord('.kibana', 3),
]),
createDeleteIndexStream(client, stats, log),
]);
expect(mockCleanSavedObjectIndices).toHaveBeenCalledTimes(2);
expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
1,
expect.objectContaining({ index: '.kibana_task_manager' })
);
expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
2,
expect.not.objectContaining({ index: expect.any(String) })
);
});
});
describe('when saved object documents are not found', () => {
it('does not clean any indices', async () => {
const client = createStubClient();
const stats = createStubStats();
await createPromiseFromStreams([
createListStream([
createStubDocRecord('.foo', 1),
createStubDocRecord('.bar', 2),
createStubDocRecord('.baz', 3),
]),
createDeleteIndexStream(client, stats, log),
]);
expect(mockCleanSavedObjectIndices).not.toHaveBeenCalled();
});
});
});
});

17
packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts
View file

@ -60,7 +60,22 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli
await deleteDataStream(client, dataStream, name);
stats.deletedDataStream(dataStream, name);
} else {
this.push(record);
if (record.type === 'doc') {
const index = record.value.index;
if (index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX)) {
if (!kibanaTaskManagerIndexAlreadyCleaned) {
await cleanSavedObjectIndices({ client, stats, index, log });
kibanaTaskManagerIndexAlreadyCleaned = true;
log.debug(`Cleaned saved object index [${index}]`);
}
} else if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
if (!kibanaIndicesAlreadyCleaned) {
await cleanSavedObjectIndices({ client, stats, log });
kibanaIndicesAlreadyCleaned = kibanaTaskManagerIndexAlreadyCleaned = true;
log.debug(`Cleaned all saved object indices`);
}
}
}
}
callback();
} catch (err) {

3
packages/kbn-es-archiver/src/lib/indices/kibana_index.ts
View file

@ -121,6 +121,7 @@ export async function cleanSavedObjectIndices({
const resp = await client.deleteByQuery(
{
index,
refresh: true,
body: {
query: {
bool: {
@ -139,7 +140,7 @@ export async function cleanSavedObjectIndices({
}
);
if (resp.total !== resp.deleted) {
if (resp.total !== resp.deleted && resp.total && resp.total > 1) {
log.warning(
'delete by query deleted %d of %d total documents, trying again',
resp.deleted,

79
x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json
View file

@ -1,22 +1,3 @@
{
"type": "_doc",
"value": {
"id": "space:default",
"index": ".kibana",
"source": {
"space": {
"_reserved": true,
"description": "This is the default space",
"disabledFeatures": [],
"name": "Default Space"
},
"type": "space",
"updated_at": "2017-09-21T18:49:16.270Z"
},
"type": "_doc"
}
}
{
"type": "_doc",
"value": {
@ -29,7 +10,11 @@
"name": "Space 1"
},
"type": "space",
"updated_at": "2017-09-21T18:49:16.270Z"
"updated_at": "2017-09-21T18:49:16.270Z",
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "6.6.0",
"managed": false,
"references": []
},
"type": "_doc"
}
@ -47,7 +32,11 @@
"name": "Space 2"
},
"type": "space",
"updated_at": "2017-09-21T18:49:16.270Z"
"updated_at": "2017-09-21T18:49:16.270Z",
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "6.6.0",
"managed": false,
"references": []
},
"type": "_doc"
}
@ -120,7 +109,9 @@
}],
"type": "dashboard",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["default"]
"namespaces": ["default"],
"typeMigrationVersion": "8.7.0",
"coreMigrationVersion": "8.8.0"
},
"type": "_doc"
}
@ -149,7 +140,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["default"]
"namespaces": ["default"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -178,7 +171,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["default"]
"namespaces": ["default"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -208,7 +203,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["default"]
"namespaces": ["default"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -244,7 +241,9 @@
],
"type": "dashboard",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["space_1"]
"namespaces": ["space_1"],
"typeMigrationVersion": "8.7.0",
"coreMigrationVersion": "8.8.0"
},
"type": "_doc"
}
@ -273,7 +272,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["space_1"]
"namespaces": ["space_1"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -302,7 +303,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["space_1"]
"namespaces": ["space_1"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -332,7 +335,9 @@
],
"type": "visualization",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["space_1"]
"namespaces": ["space_1"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.5.0"
},
"type": "_doc"
}
@ -351,7 +356,9 @@
"references": [],
"type": "index-pattern",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["default"]
"namespaces": ["default"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.0.0"
},
"type": "_doc"
}
@ -370,7 +377,9 @@
"references": [],
"type": "index-pattern",
"updated_at": "2017-09-21T18:49:16.270Z",
"namespaces": ["space_1"]
"namespaces": ["space_1"],
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.0.0"
},
"type": "_doc"
}
@ -409,7 +418,9 @@
"targetNamespace": "default",
"targetType": "sharedtype",
"targetId": "default_only"
}
},
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.2.0"
}
}
}
@ -505,7 +516,9 @@
"targetNamespace": "space_2",
"targetType": "sharedtype",
"targetId": "space_2_only"
}
},
"coreMigrationVersion": "8.8.0",
"typeMigrationVersion": "8.2.0"
}
}
}

277
x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json
View file

@ -1,277 +0,0 @@
{
"type": "index",
"value": {
"aliases": {
".kibana_$KIBANA_PACKAGE_VERSION": {},
".kibana": {}
},
"index": ".kibana_$KIBANA_PACKAGE_VERSION_001",
"mappings": {
"dynamic": "false",
"_meta": {
"indexTypesMap": {
".kibana_task_manager": [
"task"
],
".kibana": [
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"config",
"config-global",
"core-usage-stats",
"enterprise_search_telemetry",
"event_loop_delays_daily",
"file",
"file-upload-usage-collection-telemetry",
"fileShare",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"inventory-view",
"legacy-url-alias",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"sample-data-telemetry",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"workplace_search_telemetry"
],
".kibana_ingest": [
"epm-packages",
"epm-packages-assets",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"ingest_manager_settings"
],
".kibana_security_solution": [
"csp-rule-template",
"endpoint:user-artifact-manifest",
"exception-list",
"exception-list-agnostic",
"osquery-manager-usage-metric",
"osquery-pack",
"osquery-pack-asset",
"osquery-saved-query",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event"
],
".kibana_alerting_cases": [
"action",
"action_task_params",
"alert",
"api_key_pending_invalidation",
"cases",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"connector_token",
"maintenance-window",
"rules-settings"
],
".kibana_analytics": [
"canvas-element",
"canvas-workpad",
"canvas-workpad-template",
"dashboard",
"graph-workspace",
"index-pattern",
"kql-telemetry",
"lens",
"lens-ui-telemetry",
"map",
"query",
"search",
"search-session",
"search-telemetry",
"visualization"
]
}
}
},
"settings": {
"index": {
"auto_expand_replicas": "0-1",
"number_of_replicas": "0",
"number_of_shards": "1",
"mapping": {
"total_fields": {
"limit": 1500
}
}
}
}
}
}
{
"type": "index",
"value": {
"aliases": {
".kibana_analytics_$KIBANA_PACKAGE_VERSION": {},
".kibana_analytics": {}
},
"index": ".kibana_analytics_$KIBANA_PACKAGE_VERSION_001",
"mappings": {
"dynamic": "false",
"_meta": {
"indexTypesMap": {
".kibana_task_manager": [
"task"
],
".kibana": [
"apm-indices",
"apm-server-schema",
"apm-service-group",
"apm-telemetry",
"app_search_telemetry",
"application_usage_daily",
"application_usage_totals",
"config",
"config-global",
"core-usage-stats",
"enterprise_search_telemetry",
"event_loop_delays_daily",
"file",
"file-upload-usage-collection-telemetry",
"fileShare",
"guided-onboarding-guide-state",
"guided-onboarding-plugin-state",
"infrastructure-monitoring-log-view",
"infrastructure-ui-source",
"inventory-view",
"legacy-url-alias",
"metrics-explorer-view",
"ml-job",
"ml-module",
"ml-trained-model",
"monitoring-telemetry",
"sample-data-telemetry",
"slo",
"space",
"spaces-usage-stats",
"synthetics-monitor",
"synthetics-param",
"synthetics-privates-locations",
"tag",
"telemetry",
"ui-metric",
"upgrade-assistant-ml-upgrade-operation",
"upgrade-assistant-reindex-operation",
"uptime-dynamic-settings",
"uptime-synthetics-api-key",
"url",
"usage-counters",
"workplace_search_telemetry"
],
".kibana_ingest": [
"epm-packages",
"epm-packages-assets",
"fleet-fleet-server-host",
"fleet-message-signing-keys",
"fleet-preconfiguration-deletion-record",
"fleet-proxy",
"ingest-agent-policies",
"ingest-download-sources",
"ingest-outputs",
"ingest-package-policies",
"ingest_manager_settings"
],
".kibana_security_solution": [
"csp-rule-template",
"endpoint:user-artifact-manifest",
"exception-list",
"exception-list-agnostic",
"osquery-manager-usage-metric",
"osquery-pack",
"osquery-pack-asset",
"osquery-saved-query",
"security-rule",
"security-solution-signals-migration",
"siem-detection-engine-rule-actions",
"siem-ui-timeline",
"siem-ui-timeline-note",
"siem-ui-timeline-pinned-event"
],
".kibana_alerting_cases": [
"action",
"action_task_params",
"alert",
"api_key_pending_invalidation",
"cases",
"cases-comments",
"cases-configure",
"cases-connector-mappings",
"cases-telemetry",
"cases-user-actions",
"connector_token",
"maintenance-window",
"rules-settings"
],
".kibana_analytics": [
"canvas-element",
"canvas-workpad",
"canvas-workpad-template",
"dashboard",
"graph-workspace",
"index-pattern",
"kql-telemetry",
"lens",
"lens-ui-telemetry",
"map",
"query",
"search",
"search-session",
"search-telemetry",
"visualization"
]
}
}
},
"settings": {
"index": {
"auto_expand_replicas": "0-1",
"number_of_replicas": "0",
"number_of_shards": "1",
"mapping": {
"total_fields": {
"limit": 1500
}
}
}
}
}
}

5
x-pack/test/spaces_api_integration/common/suites/get.ts
View file

@ -53,8 +53,9 @@ export function getTestSuiteFactory(esArchiver: any, supertest: SuperAgent<any>)
const allSpaces = [
{
id: 'default',
name: 'Default Space',
description: 'This is the default space',
name: 'Default',
color: '#00bfb3',
description: 'This is your default space!',
_reserved: true,
disabledFeatures: [],
},

5
x-pack/test/spaces_api_integration/common/suites/get_all.ts
View file

@ -38,8 +38,9 @@ interface AuthorizedPurposes {
const ALL_SPACE_RESULTS = [
{
id: 'default',
name: 'Default Space',
description: 'This is the default space',
name: 'Default',
color: '#00bfb3',
description: 'This is your default space!',
_reserved: true,
disabledFeatures: [],
},

5
x-pack/test/spaces_api_integration/spaces_only/apis/index.ts
View file

@ -19,9 +19,6 @@ export default function spacesOnlyTestSuite({ loadTestFile }: FtrProviderContext
loadTestFile(require.resolve('./get'));
loadTestFile(require.resolve('./update'));
loadTestFile(require.resolve('./update_objects_spaces'));
// FLAKY: https://github.com/elastic/kibana/issues/158711, https://github.com/elastic/kibana/issues/158366
// Also, https://github.com/elastic/kibana/issues/158918
// esArchiver fails with no_shard_available_action_exception after deleting indexes
// loadTestFile(require.resolve('./disable_legacy_url_aliases'));
loadTestFile(require.resolve('./disable_legacy_url_aliases'));
});
}