github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.16] [8.x] [Entity Store] Fix and re-enable entity store integration tests (#196296) (#196656) (#196840)

Browse source 
# Backport

This will backport the following commits from `8.x` to `8.16`:
- [[8.x] [Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)](https://github.com/elastic/kibana/pull/196656)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mark
Hopkin","email":"mark.hopkin@elastic.co"},"sourceCommit":{"committedDate":"2024-10-18T10:03:50Z","message":"[8.x]
[Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)\n\n# Backport\n\nThis will backport the following
commits from `main` to `8.x`:\n- [[Entity Store] Fix and re-enable
entity store integration
tests\n(#196296)](https://github.com/elastic/kibana/pull/196296)\n\n<!---
Backport version: 8.9.8 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Mark\nHopkin\",\"email\":\"mark.hopkin@elastic.co\"},\"sourceCommit\":{\"committedDate\":\"2024-10-16T09:22:18Z\",\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.16.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:skip\",\"backport\nmissing\",\"v9.0.0\",\"Team:\nSecuritySolution\",\"backport:prev-minor\",\"Team:Entity\nAnalytics\"],\"number\":196296,\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"labelRegex\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"number\":196296,\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}}]}]
BACKPORT-->\n\n---------\n\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"cb119aca152821bb41efe0b47de398960a818c1a","branchLabelMapping":{"^v8.16.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["backport"],"number":196656,"url":"https://github.com/elastic/kibana/pull/196656","mergeCommit":{"message":"[8.x]
[Entity Store] Fix and re-enable entity store integration tests
(#196296) (#196656)\n\n# Backport\n\nThis will backport the following
commits from `main` to `8.x`:\n- [[Entity Store] Fix and re-enable
entity store integration
tests\n(#196296)](https://github.com/elastic/kibana/pull/196296)\n\n<!---
Backport version: 8.9.8 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Mark\nHopkin\",\"email\":\"mark.hopkin@elastic.co\"},\"sourceCommit\":{\"committedDate\":\"2024-10-16T09:22:18Z\",\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.16.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:skip\",\"backport\nmissing\",\"v9.0.0\",\"Team:\nSecuritySolution\",\"backport:prev-minor\",\"Team:Entity\nAnalytics\"],\"number\":196296,\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"labelRegex\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/196296\",\"number\":196296,\"mergeCommit\":{\"message\":\"[Entity\nStore]
Fix and re-enable entity store integration tests
(#196296)\\n\\n##\nSummary\\r\\n\\r\\nWe previously had to skip these
tests due to permissions\nissues, I have\\r\\nnow fixed and
re-enabled\nthem.\",\"sha\":\"002b757123a12172a4d5d114843bc67d063ebdf8\"}}]}]
BACKPORT-->\n\n---------\n\nCo-authored-by: kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"cb119aca152821bb41efe0b47de398960a818c1a"}},"sourceBranch":"8.x","suggestedTargetBranches":[],"targetPullRequestStates":[]}]
BACKPORT-->

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Mark Hopkin 2024-10-18 19:10:24 +01:00 • committed by GitHub
parent 806796f1f7
commit f929851176
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 80 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts
View file

@ -48,9 +48,7 @@ export const getUnitedEntityDefinition = memoize(
namespace,
indexPatterns,
});
},
({ entityType, namespace, fieldHistoryLength }: Options) =>
`${entityType}-${namespace}-${fieldHistoryLength}`
}
);
export const getUnitedEntityDefinitionVersion = (entityType: EntityType): string =>

8
x-pack/test/functional/es_archives/security_solution/entity_store/data.json
View file

@ -2,7 +2,7 @@
"type": "doc",
"value": {
"id": "a4cf452c1e0375c3d4412cb550ad1783358468a3b3b777da4829d72c7d6fb74f",
"index": ".entities.v1.latest.ea_default_user_entity_store",
"index": ".entities.v1.latest.security_user_default",
"source": {
"event": {
"ingested": "2024-09-11T11:26:49.706875Z"
@ -27,7 +27,7 @@
"id": "LBQAgKHGmpup0Kg9nlKmeQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "ea_default_user_entity_store"
"definitionId": "security_user_default"
}
}
}
@ -37,7 +37,7 @@
"type": "doc",
"value": {
"id": "a2cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb71f",
"index": ".entities.v1.latest.ea_default_host_entity_store",
"index": ".entities.v1.latest.security_host_default",
"source": {
"event": {
"ingested": "2024-09-11T11:26:49.641707Z"
@ -78,7 +78,7 @@
"id": "ZXKm6GEcUJY6NHkMgPPmGQ==",
"type": "node",
"firstSeenTimestamp": "2024-09-11T10:46:00.000Z",
"definitionId": "ea_default_host_entity_store"
"definitionId": "security_host_default"
}
}
}

38
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts
View file

@ -7,26 +7,16 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../ftr_provider_context';
import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
export default ({ getService }: FtrProviderContext) => {
const api = getService('securitySolutionApi');
const supertest = getService('supertest');
const {
expectTransformExists,
expectTransformNotFound,
expectEnrichPolicyExists,
expectEnrichPolicyNotFound,
expectComponentTemplateExists,
expectComponentTemplateNotFound,
expectIngestPipelineExists,
expectIngestPipelineNotFound,
} = elasticAssetCheckerFactory(getService);
const utils = EntityStoreUtils(getService);
// TODO: unskip once permissions issue is resolved
describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => {
// Failing: See https://github.com/elastic/kibana/issues/196526
describe.skip('@ess @skipInServerlessMKI Entity Store Engine APIs', () => {
const dataView = dataViewRouteHelpersFactory(supertest);
before(async () => {
@ -45,20 +35,12 @@ export default ({ getService }: FtrProviderContext) => {
it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
await expectTransformExists('entities-v1-latest-ea_default_user_entity_store');
await expectEnrichPolicyExists('entity_store_field_retention_user_default_v1');
await expectComponentTemplateExists(`ea_default_user_entity_store-latest@platform`);
await expectIngestPipelineExists(`ea_default_user_entity_store-latest@platform`);
await utils.expectEngineAssetsExist('user');
});
it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
await expectTransformExists('entities-v1-latest-ea_default_host_entity_store');
await expectEnrichPolicyExists('entity_store_field_retention_host_default_v1');
await expectComponentTemplateExists(`ea_default_host_entity_store-latest@platform`);
await expectIngestPipelineExists(`ea_default_host_entity_store-latest@platform`);
await utils.expectEngineAssetsExist('host');
});
});
@ -188,10 +170,7 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);
await expectTransformNotFound('entities-v1-latest-ea_default_host_entity_store');
await expectEnrichPolicyNotFound('entity_store_field_retention_host_default_v1');
await expectComponentTemplateNotFound(`ea_default_host_entity_store-latest@platform`);
await expectIngestPipelineNotFound(`ea_default_host_entity_store-latest@platform`);
await utils.expectEngineAssetsDoNotExist('host');
});
it('should delete the user entity engine', async () => {
@ -204,10 +183,7 @@ export default ({ getService }: FtrProviderContext) => {
})
.expect(200);
await expectTransformNotFound('entities-v1-latest-ea_default_user_entity_store');
await expectEnrichPolicyNotFound('entity_store_field_retention_user_default_v1');
await expectComponentTemplateNotFound(`ea_default_user_entity_store-latest@platform`);
await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`);
await utils.expectEngineAssetsDoNotExist('user');
});
});

46
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine_nondefault_spaces.ts
View file

@ -9,15 +9,19 @@ import expect from '@kbn/expect';
import { v4 as uuidv4 } from 'uuid';
import { FtrProviderContextWithSpaces } from '../../../../ftr_provider_context_with_spaces';
import { EntityStoreUtils } from '../../utils';
import { dataViewRouteHelpersFactory } from '../../utils/data_view';
export default ({ getService }: FtrProviderContextWithSpaces) => {
const api = getService('securitySolutionApi');
const spaces = getService('spaces');
const namespace = uuidv4().substring(0, 8);
const supertest = getService('supertest');
const utils = EntityStoreUtils(getService, namespace);
// TODO: unskip once kibana system user has entity index privileges
// Failing: See https://github.com/elastic/kibana/issues/196546
describe.skip('@ess Entity Store Engine APIs in non-default space', () => {
const dataView = dataViewRouteHelpersFactory(supertest, namespace);
before(async () => {
await utils.cleanEngines();
await spaces.create({
@ -25,9 +29,11 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
name: namespace,
disabledFeatures: [],
});
await dataView.create('security-solution');
});
after(async () => {
await dataView.delete('security-solution');
await spaces.delete(namespace);
});
@ -38,18 +44,12 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
it('should have installed the expected user resources', async () => {
await utils.initEntityEngineForEntityType('user');
const expectedTransforms = [`entities-v1-latest-ea_${namespace}_user_entity_store`];
await utils.expectTransformsExist(expectedTransforms);
await utils.expectEngineAssetsExist('user');
});
it('should have installed the expected host resources', async () => {
await utils.initEntityEngineForEntityType('host');
const expectedTransforms = [`entities-v1-latest-ea_${namespace}_host_entity_store`];
await utils.expectTransformsExist(expectedTransforms);
await utils.expectEngineAssetsExist('host');
});
});
@ -79,9 +79,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
});
});
@ -98,9 +98,9 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
expect(getResponse.body).to.eql({
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
});
});
});
@ -116,16 +116,16 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
{
status: 'started',
type: 'host',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
},
{
status: 'started',
type: 'user',
indexPattern:
'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*',
filter: '',
fieldHistoryLength: 10,
indexPattern: '',
},
]);
});
@ -200,10 +200,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
)
.expect(200);
await utils.expectTransformNotFound(
`entities-v1-history-ea_${namespace}_host_entity_store`
);
await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_host_entity_store`);
await utils.expectEngineAssetsDoNotExist('host');
});
it('should delete the user entity engine', async () => {
@ -219,10 +216,7 @@ export default ({ getService }: FtrProviderContextWithSpaces) => {
)
.expect(200);
await utils.expectTransformNotFound(
`entities-v1-history-ea_${namespace}_user_entity_store`
);
await utils.expectTransformNotFound(`entities-v1-latest-ea_${namespace}_user_entity_store`);
await utils.expectEngineAssetsDoNotExist('user');
});
});
});

3
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/entities_list.ts
View file

@ -11,8 +11,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
export default ({ getService }: FtrProviderContext) => {
const securitySolutionApi = getService('securitySolutionApi');
// TODO: unskip once permissions issue is resolved
describe.skip('@ess Entity store - Entities list API', () => {
describe('@ess @skipInServerlessMKI Entity store - Entities list API', () => {
describe('when the entity store is disable', () => {
it("should return response with success status when the index doesn't exist", async () => {
const { body } = await securitySolutionApi.listEntities({

6
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts
View file

@ -12,7 +12,7 @@ export const dataViewRouteHelpersFactory = (
) => ({
create: (name: string) => {
return supertest
.post(`/api/data_views/data_view`)
.post(`/s/${namespace}/api/data_views/data_view`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {
@ -26,13 +26,13 @@ export const dataViewRouteHelpersFactory = (
},
delete: (name: string) => {
return supertest
.delete(`/api/data_views/data_view/${name}-${namespace}`)
.delete(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.expect(200);
},
updateIndexPattern: (name: string, indexPattern: string) => {
return supertest
.post(`/api/data_views/data_view/${name}-${namespace}`)
.post(`/s/${namespace}/api/data_views/data_view/${name}-${namespace}`)
.set('kbn-xsrf', 'foo')
.send({
data_view: {

67
x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts
View file

@ -6,16 +6,27 @@
*/
import { EntityType } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/common.gen';
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../../api_integration/ftr_provider_context';
import { elasticAssetCheckerFactory } from './elastic_asset_checker';
export const EntityStoreUtils = (
getService: FtrProviderContext['getService'],
namespace?: string
namespace: string = 'default'
) => {
const api = getService('securitySolutionApi');
const es = getService('es');
const log = getService('log');
const {
expectTransformExists,
expectTransformNotFound,
expectEnrichPolicyExists,
expectEnrichPolicyNotFound,
expectComponentTemplateExists,
expectComponentTemplateNotFound,
expectIngestPipelineExists,
expectIngestPipelineNotFound,
} = elasticAssetCheckerFactory(getService);
log.debug(`EntityStoreUtils namespace: ${namespace}`);
@ -37,17 +48,24 @@ export const EntityStoreUtils = (
}
};
const initEntityEngineForEntityType = (entityType: EntityType) => {
log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`);
return api
.initEntityEngine(
{
params: { entityType },
body: {},
},
namespace
)
.expect(200);
const initEntityEngineForEntityType = async (entityType: EntityType) => {
log.info(
`Initializing engine for entity type ${entityType} in namespace ${namespace || 'default'}`
);
const res = await api.initEntityEngine(
{
params: { entityType },
body: {},
},
namespace
);
if (res.status !== 200) {
log.error(`Failed to initialize engine for entity type ${entityType}`);
log.error(JSON.stringify(res.body));
}
expect(res.status).to.eql(200);
};
const expectTransformStatus = async (
@ -78,22 +96,25 @@ export const EntityStoreUtils = (
}
};
const expectTransformNotFound = async (transformId: string, attempts: number = 5) => {
return expectTransformStatus(transformId, false);
};
const expectTransformExists = async (transformId: string) => {
return expectTransformStatus(transformId, true);
const expectEngineAssetsExist = async (entityType: EntityType) => {
await expectTransformExists(`entities-v1-latest-security_${entityType}_${namespace}`);
await expectEnrichPolicyExists(`entity_store_field_retention_${entityType}_${namespace}_v1`);
await expectComponentTemplateExists(`security_${entityType}_${namespace}-latest@platform`);
await expectIngestPipelineExists(`security_${entityType}_${namespace}-latest@platform`);
};
const expectTransformsExist = async (transformIds: string[]) =>
Promise.all(transformIds.map((id) => expectTransformExists(id)));
const expectEngineAssetsDoNotExist = async (entityType: EntityType) => {
await expectTransformNotFound(`entities-v1-latest-security_${entityType}_${namespace}`);
await expectEnrichPolicyNotFound(`entity_store_field_retention_${entityType}_${namespace}_v1`);
await expectComponentTemplateNotFound(`security_${entityType}_${namespace}-latest@platform`);
await expectIngestPipelineNotFound(`security_${entityType}_${namespace}-latest@platform`);
};
return {
cleanEngines,
initEntityEngineForEntityType,
expectTransformStatus,
expectTransformNotFound,
expectTransformExists,
expectTransformsExist,
expectEngineAssetsExist,
expectEngineAssetsDoNotExist,
};
};