github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution] add handling for endpoint package spec v2 (#169901)

Browse source
This commit is contained in:
Joey F. Poon 2023-10-26 13:08:06 -07:00 • committed by GitHub
parent b3955ce0c1
commit 82880fed8e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 120 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

6
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -29,14 +29,20 @@ export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
/** index that the metadata transform writes to (destination) and that is used by endpoint APIs */
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*';
// endpoint package V2 has an additional prefix in the transform names
const PACKAGE_V2_PREFIX = 'logs-';
/** The metadata Transform Name prefix with NO (package) version) */
export const metadataTransformPrefix = 'endpoint.metadata_current-default';
export const METADATA_CURRENT_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${metadataTransformPrefix}`;
// metadata transforms pattern for matching all metadata transform ids
export const METADATA_TRANSFORMS_PATTERN = 'endpoint.metadata_*';
export const METADATA_TRANSFORMS_PATTERN_V2 = `${PACKAGE_V2_PREFIX}${METADATA_TRANSFORMS_PATTERN}`;
// united metadata transform id
export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default';
export const METADATA_UNITED_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${METADATA_UNITED_TRANSFORM}`;
// united metadata transform destination index
export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default';

7
x-pack/plugins/security_solution/common/endpoint/index_data.ts
View file

@ -111,8 +111,8 @@ export const indexHostsAndAlerts = usageTracker.track(
const shouldWaitForEndpointMetadataDocs = fleet;
if (shouldWaitForEndpointMetadataDocs) {
await waitForMetadataTransformsReady(client);
await stopMetadataTransforms(client);
await waitForMetadataTransformsReady(client, epmEndpointPackage.version);
await stopMetadataTransforms(client, epmEndpointPackage.version);
}
for (let i = 0; i < numHosts; i++) {
@ -147,7 +147,8 @@ export const indexHostsAndAlerts = usageTracker.track(
if (shouldWaitForEndpointMetadataDocs) {
await startMetadataTransforms(
client,
response.agents.map((agent) => agent.agent?.id ?? '')
response.agents.map((agent) => agent.agent?.id ?? ''),
epmEndpointPackage.version
);
}

46
x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import semverLte from 'semver/functions/lte';
import type { Client } from '@elastic/elasticsearch';
import type { TransformGetTransformStatsTransformStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
@ -12,21 +14,24 @@ import { usageTracker } from '../data_loaders/usage_tracker';
import {
metadataCurrentIndexPattern,
metadataTransformPrefix,
METADATA_CURRENT_TRANSFORM_V2,
METADATA_TRANSFORMS_PATTERN,
METADATA_TRANSFORMS_PATTERN_V2,
METADATA_UNITED_TRANSFORM,
METADATA_UNITED_TRANSFORM_V2,
} from '../constants';
export const waitForMetadataTransformsReady = usageTracker.track(
'waitForMetadataTransformsReady',
async (esClient: Client): Promise<void> => {
await waitFor(() => areMetadataTransformsReady(esClient));
async (esClient: Client, version: string): Promise<void> => {
await waitFor(() => areMetadataTransformsReady(esClient, version));
}
);
export const stopMetadataTransforms = usageTracker.track(
'stopMetadataTransforms',
async (esClient: Client): Promise<void> => {
const transformIds = await getMetadataTransformIds(esClient);
async (esClient: Client, version: string): Promise<void> => {
const transformIds = await getMetadataTransformIds(esClient, version);
await Promise.all(
transformIds.map((transformId) =>
@ -46,14 +51,18 @@ export const startMetadataTransforms = usageTracker.track(
async (
esClient: Client,
// agentIds to wait for
agentIds: string[]
agentIds: string[],
version: string
): Promise<void> => {
const transformIds = await getMetadataTransformIds(esClient);
const transformIds = await getMetadataTransformIds(esClient, version);
const isV2 = isEndpointPackageV2(version);
const currentTransformPrefix = isV2 ? METADATA_CURRENT_TRANSFORM_V2 : metadataTransformPrefix;
const currentTransformId = transformIds.find((transformId) =>
transformId.startsWith(metadataTransformPrefix)
transformId.startsWith(currentTransformPrefix)
);
const unitedTransformPrefix = isV2 ? METADATA_UNITED_TRANSFORM_V2 : METADATA_UNITED_TRANSFORM;
const unitedTransformId = transformIds.find((transformId) =>
transformId.startsWith(METADATA_UNITED_TRANSFORM)
transformId.startsWith(unitedTransformPrefix)
);
if (!currentTransformId || !unitedTransformId) {
// eslint-disable-next-line no-console
@ -88,22 +97,26 @@ export const startMetadataTransforms = usageTracker.track(
);
async function getMetadataTransformStats(
esClient: Client
esClient: Client,
version: string
): Promise<TransformGetTransformStatsTransformStats[]> {
const transformId = isEndpointPackageV2(version)
? METADATA_TRANSFORMS_PATTERN_V2
: METADATA_TRANSFORMS_PATTERN;
return (
await esClient.transform.getTransformStats({
transform_id: METADATA_TRANSFORMS_PATTERN,
transform_id: transformId,
allow_no_match: true,
})
).transforms;
}
async function getMetadataTransformIds(esClient: Client): Promise<string[]> {
return (await getMetadataTransformStats(esClient)).map((transform) => transform.id);
async function getMetadataTransformIds(esClient: Client, version: string): Promise<string[]> {
return (await getMetadataTransformStats(esClient, version)).map((transform) => transform.id);
}
async function areMetadataTransformsReady(esClient: Client): Promise<boolean> {
const transforms = await getMetadataTransformStats(esClient);
async function areMetadataTransformsReady(esClient: Client, version: string): Promise<boolean> {
const transforms = await getMetadataTransformStats(esClient, version);
return !transforms.some(
// TODO TransformGetTransformStatsTransformStats type needs to be updated to include health
(transform: TransformGetTransformStatsTransformStats & { health?: { status: string } }) =>
@ -159,3 +172,8 @@ async function waitFor(
}
}
}
const MIN_ENDPOINT_PACKAGE_V2_VERSION = '8.12.0';
export function isEndpointPackageV2(version: string) {
return semverLte(MIN_ENDPOINT_PACKAGE_V2_VERSION, version);
}

1
x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts
View file

@ -70,6 +70,7 @@ describe('check metadata transforms task', () => {
{ type: ElasticsearchAssetType.transform } as EsAssetReference,
{ type: ElasticsearchAssetType.transform } as EsAssetReference,
],
version: '8.11.0',
} as Installation);
});

24
x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts
View file

@ -19,10 +19,14 @@ import type {
import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server';
import { ElasticsearchAssetType, FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
import type { EndpointAppContext } from '../../types';
import { METADATA_TRANSFORMS_PATTERN } from '../../../../common/endpoint/constants';
import {
METADATA_TRANSFORMS_PATTERN,
METADATA_TRANSFORMS_PATTERN_V2,
} from '../../../../common/endpoint/constants';
import { WARNING_TRANSFORM_STATES } from '../../../../common/constants';
import { wrapErrorIfNeeded } from '../../utils';
import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state';
import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
const SCOPE = ['securitySolution'];
const INTERVAL = '2h';
@ -108,11 +112,21 @@ export class CheckMetadataTransformsTask {
const [{ elasticsearch }] = await core.getStartServices();
const esClient = elasticsearch.client.asInternalUser;
const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
if (!installation) {
this.logger.info('no endpoint installation found');
return { state: taskInstance.state };
}
const transformName = isEndpointPackageV2(installation.version)
? METADATA_TRANSFORMS_PATTERN_V2
: METADATA_TRANSFORMS_PATTERN;
let transformStatsResponse: TransportResult<TransformGetTransformStatsResponse>;
try {
transformStatsResponse = await esClient?.transform.getTransformStats(
{
transform_id: METADATA_TRANSFORMS_PATTERN,
transform_id: transformName,
},
{ meta: true }
);
@ -124,12 +138,6 @@ export class CheckMetadataTransformsTask {
return { state: taskInstance.state };
}
const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
if (!installation) {
this.logger.info('no endpoint installation found');
return { state: taskInstance.state };
}
const expectedTransforms = installation.installed_es.filter(
(asset) => asset.type === ElasticsearchAssetType.transform
);

14
x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts
View file

@ -7,6 +7,8 @@
import type { TypeOf } from '@kbn/config-schema';
import type { Logger, RequestHandler } from '@kbn/core/server';
import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
import type {
MetadataListResponse,
EndpointSortableField,
@ -25,7 +27,9 @@ import {
ENDPOINT_DEFAULT_SORT_DIRECTION,
ENDPOINT_DEFAULT_SORT_FIELD,
METADATA_TRANSFORMS_PATTERN,
METADATA_TRANSFORMS_PATTERN_V2,
} from '../../../../common/endpoint/constants';
import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
export const getLogger = (endpointAppContext: EndpointAppContext): Logger => {
return endpointAppContext.logFactory.get('metadata');
@ -99,13 +103,21 @@ export const getMetadataRequestHandler = function (
};
export function getMetadataTransformStatsHandler(
endpointAppContext: EndpointAppContext,
logger: Logger
): RequestHandler<unknown, unknown, unknown, SecuritySolutionRequestHandlerContext> {
return async (context, _, response) => {
const esClient = (await context.core).elasticsearch.client.asInternalUser;
const packageClient = endpointAppContext.service.getInternalFleetServices().packages;
const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
const transformName =
installation?.version && !isEndpointPackageV2(installation.version)
? METADATA_TRANSFORMS_PATTERN
: METADATA_TRANSFORMS_PATTERN_V2;
try {
const transformStats = await esClient.transform.getTransformStats({
transform_id: METADATA_TRANSFORMS_PATTERN,
transform_id: transformName,
allow_no_match: true,
});
return response.ok({

2
x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts
View file

@ -103,7 +103,7 @@ export function registerEndpointRoutes(
withEndpointAuthz(
{ all: ['canReadSecuritySolution'] },
logger,
getMetadataTransformStatsHandler(logger)
getMetadataTransformStatsHandler(endpointAppContext, logger)
)
);
}

29
x-pack/test/security_solution_endpoint/services/endpoint.ts
View file

@ -12,8 +12,10 @@ import { Client } from '@elastic/elasticsearch';
import {
metadataCurrentIndexPattern,
metadataTransformPrefix,
METADATA_CURRENT_TRANSFORM_V2,
METADATA_UNITED_INDEX,
METADATA_UNITED_TRANSFORM,
METADATA_UNITED_TRANSFORM_V2,
HOST_METADATA_GET_ROUTE,
METADATA_DATASTREAM,
} from '@kbn/security-solution-plugin/common/endpoint/constants';
@ -22,6 +24,8 @@ import {
IndexedHostsAndAlertsResponse,
indexHostsAndAlerts,
} from '@kbn/security-solution-plugin/common/endpoint/index_data';
import { getEndpointPackageInfo } from '@kbn/security-solution-plugin/common/endpoint/utils/package';
import { isEndpointPackageV2 } from '@kbn/security-solution-plugin/common/endpoint/utils/transforms';
import { installOrUpgradeEndpointFleetPackage } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/setup_fleet_for_endpoint';
import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors';
import { STARTED_TRANSFORM_STATES } from '@kbn/security-solution-plugin/common/constants';
@ -116,11 +120,23 @@ export class EndpointTestResources extends FtrService {
customIndexFn,
} = options;
let currentTransformName = metadataTransformPrefix;
let unitedTransformName = METADATA_UNITED_TRANSFORM;
if (waitUntilTransformed && customIndexFn) {
const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
const isV2 = isEndpointPackageV2(endpointPackage.version);
if (isV2) {
currentTransformName = METADATA_CURRENT_TRANSFORM_V2;
unitedTransformName = METADATA_UNITED_TRANSFORM_V2;
}
}
if (waitUntilTransformed && customIndexFn) {
// need this before indexing docs so that the united transform doesn't
// create a checkpoint with a timestamp after the doc timestamps
await this.stopTransform(metadataTransformPrefix);
await this.stopTransform(METADATA_UNITED_TRANSFORM);
await this.stopTransform(currentTransformName);
await this.stopTransform(unitedTransformName);
}
// load data into the system
@ -147,10 +163,10 @@ export class EndpointTestResources extends FtrService {
);
if (waitUntilTransformed && customIndexFn) {
await this.startTransform(metadataTransformPrefix);
await this.startTransform(currentTransformName);
const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id)));
await this.waitForEndpoints(metadataIds, waitTimeout);
await this.startTransform(METADATA_UNITED_TRANSFORM);
await this.startTransform(unitedTransformName);
}
if (waitUntilTransformed) {
@ -342,4 +358,9 @@ export class EndpointTestResources extends FtrService {
return response;
}
async isEndpointPackageV2(): Promise<boolean> {
const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
return isEndpointPackageV2(endpointPackage.version);
}
}

33
x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts
View file

@ -16,7 +16,9 @@ import {
METADATA_TRANSFORMS_STATUS_ROUTE,
METADATA_UNITED_INDEX,
METADATA_UNITED_TRANSFORM,
METADATA_UNITED_TRANSFORM_V2,
metadataTransformPrefix,
METADATA_CURRENT_TRANSFORM_V2,
} from '@kbn/security-solution-plugin/common/endpoint/constants';
import { AGENTS_INDEX } from '@kbn/fleet-plugin/common';
import { indexFleetEndpointPolicy } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_fleet_endpoint_policy';
@ -44,8 +46,7 @@ export default function ({ getService }: FtrProviderContext) {
const endpointTestResources = getService('endpointTestResources');
const log = getService('log');
// FLAKY: https://github.com/elastic/kibana/issues/151854
describe.skip('test metadata apis', () => {
describe('test metadata apis', () => {
describe('list endpoints GET route', () => {
const numberOfHostsInFixture = 2;
let agent1Timestamp: number;
@ -400,7 +401,17 @@ export default function ({ getService }: FtrProviderContext) {
});
describe('get metadata transforms', () => {
const testRegex = /endpoint\.metadata_(united|current)-default-*/;
const testRegex = /(endpoint|logs-endpoint)\.metadata_(united|current)-default-*/;
let currentTransformName = metadataTransformPrefix;
let unitedTransformName = METADATA_UNITED_TRANSFORM;
before(async () => {
const isPackageV2 = await endpointTestResources.isEndpointPackageV2();
if (isPackageV2) {
currentTransformName = METADATA_CURRENT_TRANSFORM_V2;
unitedTransformName = METADATA_UNITED_TRANSFORM_V2;
}
});
it('should respond forbidden if no fleet access', async () => {
await getService('supertestWithoutAuth')
@ -411,8 +422,8 @@ export default function ({ getService }: FtrProviderContext) {
});
it('correctly returns stopped transform stats', async () => {
await stopTransform(getService, `${metadataTransformPrefix}*`);
await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`);
await stopTransform(getService, `${currentTransformName}*`);
await stopTransform(getService, `${unitedTransformName}*`);
const { body } = await supertest
.get(METADATA_TRANSFORMS_STATUS_ROUTE)
@ -428,17 +439,17 @@ export default function ({ getService }: FtrProviderContext) {
expect(transforms.length).to.eql(2);
const currentTransform = transforms.find((transform) =>
transform.id.startsWith(metadataTransformPrefix)
transform.id.startsWith(currentTransformName)
);
expect(currentTransform).to.be.ok();
const unitedTransform = transforms.find((transform) =>
transform.id.startsWith(METADATA_UNITED_TRANSFORM)
transform.id.startsWith(unitedTransformName)
);
expect(unitedTransform).to.be.ok();
await startTransform(getService, metadataTransformPrefix);
await startTransform(getService, METADATA_UNITED_TRANSFORM);
await startTransform(getService, currentTransformName);
await startTransform(getService, unitedTransformName);
});
it('correctly returns started transform stats', async () => {
@ -456,12 +467,12 @@ export default function ({ getService }: FtrProviderContext) {
expect(transforms.length).to.eql(2);
const currentTransform = transforms.find((transform) =>
transform.id.startsWith(metadataTransformPrefix)
transform.id.startsWith(currentTransformName)
);
expect(currentTransform).to.be.ok();
const unitedTransform = transforms.find((transform) =>
transform.id.startsWith(METADATA_UNITED_TRANSFORM)
transform.id.startsWith(unitedTransformName)
);
expect(unitedTransform).to.be.ok();
});