github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[APM] Migrate fleet sourcemaps to APM managed index (#147208)

Browse source
This commit is contained in:
Søren Louv-Jansen 2022-12-16 04:24:23 +01:00 • committed by GitHub
parent 5a6bf1dacd
commit f6491f6140
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 1222 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

17
x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap generated
View file

@ -845,16 +845,6 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the
}
}
},
"sourcemap": {
"properties": {
"1d": {
"type": "long"
},
"all": {
"type": "long"
}
}
},
"onboarding": {
"properties": {
"1d": {
@ -1011,13 +1001,6 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the
}
}
},
"sourcemap": {
"properties": {
"ms": {
"type": "long"
}
}
},
"onboarding": {
"properties": {
"ms": {

2
x-pack/plugins/apm/ftr_e2e/cypress/e2e/power_user/no_data_screen.cy.ts
View file

@ -10,7 +10,6 @@ describe('No data screen', () => {
before(() => {
// Change indices
setApmIndices({
sourcemap: 'foo-*',
error: 'foo-*',
onboarding: 'foo-*',
span: 'foo-*',
@ -37,7 +36,6 @@ describe('No data screen', () => {
after(() => {
// reset to default indices
setApmIndices({
sourcemap: '',
error: '',
onboarding: '',
span: '',

7
x-pack/plugins/apm/public/components/app/settings/apm_indices/index.tsx
View file

@ -33,13 +33,6 @@ import {
} from '../../../../services/rest/create_call_apm_api';
const APM_INDEX_LABELS = [
{
configurationName: 'sourcemap',
label: i18n.translate(
'xpack.apm.settings.apmIndices.sourcemapIndicesLabel',
{ defaultMessage: 'Sourcemap Indices' }
),
},
{
configurationName: 'error',
label: i18n.translate('xpack.apm.settings.apmIndices.errorIndicesLabel', {

1
x-pack/plugins/apm/scripts/shared/read_kibana_config.ts
View file

@ -43,7 +43,6 @@ export const readKibanaConfig = () => {
'xpack.apm.indices.error': 'logs-apm*,apm-*',
'xpack.apm.indices.span': 'traces-apm*,apm-*',
'xpack.apm.indices.onboarding': 'apm-*',
'xpack.apm.indices.sourcemap': 'apm-*',
'elasticsearch.hosts': 'http://localhost:9200',
...loadedKibanaConfig,
...cliEsCredentials,

3
x-pack/plugins/apm/server/index.ts
View file

@ -51,7 +51,6 @@ const configSchema = schema.object({
span: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
error: schema.string({ defaultValue: 'logs-apm*,apm-*' }),
metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }),
sourcemap: schema.string({ defaultValue: 'apm-*' }),
onboarding: schema.string({ defaultValue: 'apm-*' }),
}),
forceSyntheticSource: schema.boolean({ defaultValue: false }),
@ -61,10 +60,12 @@ const configSchema = schema.object({
export const config: PluginConfigDescriptor<APMConfig> = {
deprecations: ({
rename,
unused,
renameFromRoot,
deprecateFromRoot,
unusedFromRoot,
}) => [
unused('indices.sourcemap', { level: 'warning' }),
rename('autocreateApmIndexPattern', 'autoCreateApmDataView', {
level: 'warning',
}),

7
x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts
View file

@ -298,10 +298,6 @@ describe('data telemetry collection tasks', () => {
'1d': 1,
all: 1,
},
sourcemap: {
'1d': 1,
all: 1,
},
span: {
'1d': 1,
all: 1,
@ -321,9 +317,6 @@ describe('data telemetry collection tasks', () => {
onboarding: {
ms: 0,
},
sourcemap: {
ms: 0,
},
span: {
ms: 0,
},

11
x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts
View file

@ -54,7 +54,10 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
import { Span } from '../../../../typings/es_schemas/ui/span';
import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
import { APMTelemetry, APMPerService, APMDataTelemetry } from '../types';
import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_indices';
import {
ApmIndicesConfig,
APM_AGENT_CONFIGURATION_INDEX,
} from '../../../routes/settings/apm_indices/get_apm_indices';
import { TelemetryClient } from '../telemetry_client';
type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
@ -464,7 +467,6 @@ export const tasks: TelemetryTask[] = [
span: indices.span,
transaction: indices.transaction,
onboarding: indices.onboarding,
sourcemap: indices.sourcemap,
};
type ProcessorEvent = keyof typeof indicesByProcessorEvent;
@ -558,7 +560,7 @@ export const tasks: TelemetryTask[] = [
name: 'agent_configuration',
executor: async ({ indices, telemetryClient }) => {
const agentConfigurationCount = await telemetryClient.search({
index: indices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
body: {
size: 0,
timeout,
@ -1033,11 +1035,10 @@ export const tasks: TelemetryTask[] = [
executor: async ({ indices, telemetryClient }) => {
const response = await telemetryClient.indicesStats({
index: [
indices.apmAgentConfigurationIndex,
APM_AGENT_CONFIGURATION_INDEX,
indices.error,
indices.metric,
indices.onboarding,
indices.sourcemap,
indices.span,
indices.transaction,
],

2
x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts
View file

@ -193,7 +193,6 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
span: timeframeMapSchema,
error: timeframeMapSchema,
metric: timeframeMapSchema,
sourcemap: timeframeMapSchema,
onboarding: timeframeMapSchema,
agent_configuration: timeframeMapAllSchema,
max_transaction_groups_per_service: timeframeMapSchema,
@ -221,7 +220,6 @@ export const apmSchema: MakeSchemaFrom<APMUsage> = {
transaction: { ms: long },
error: { ms: long },
metric: { ms: long },
sourcemap: { ms: long },
onboarding: { ms: long },
},
integrations: { ml: { all_jobs_count: long } },

3
x-pack/plugins/apm/server/lib/apm_telemetry/types.ts
View file

@ -101,7 +101,6 @@ export interface APMUsage {
span: TimeframeMap;
error: TimeframeMap;
metric: TimeframeMap;
sourcemap: TimeframeMap;
onboarding: TimeframeMap;
agent_configuration: TimeframeMapAll;
max_transaction_groups_per_service: TimeframeMap;
@ -125,7 +124,7 @@ export interface APMUsage {
};
};
retainment: Record<
'span' | 'transaction' | 'error' | 'metric' | 'sourcemap' | 'onboarding',
'span' | 'transaction' | 'error' | 'metric' | 'onboarding',
{ ms: number }
>;
integrations: {

1
x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/unpack_processor_events.test.ts
View file

@ -27,7 +27,6 @@ describe('unpackProcessorEvents', () => {
error: 'my-apm-*-error-*',
span: 'my-apm-*-span-*',
onboarding: 'my-apm-*-onboarding-*',
sourcemap: 'my-apm-*-sourcemap-*',
} as ApmIndicesConfig;
res = unpackProcessorEvents(request, indices);

61
x-pack/plugins/apm/server/plugin.ts
View file

@ -56,6 +56,8 @@ import {
} from '../common/es_fields/apm';
import { tutorialProvider } from './tutorial';
import { migrateLegacyAPMIndicesToSpaceAware } from './saved_objects/migrations/migrate_legacy_apm_indices_to_space_aware';
import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source_map_migration';
import { createApmSourceMapIndexTemplate } from './routes/source_maps/create_apm_source_map_index_template';
export class APMPlugin
implements
@ -110,6 +112,9 @@ export class APMPlugin
const getCoreStart = () =>
core.getStartServices().then(([coreStart]) => coreStart);
const getPluginStart = () =>
core.getStartServices().then(([coreStart, pluginStart]) => pluginStart);
const { ruleDataService } = plugins.ruleRegistry;
const ruleDataClient = ruleDataService.initializeIndex({
feature: APM_SERVER_FEATURE_ID,
@ -220,6 +225,21 @@ export class APMPlugin
kibanaVersion: this.initContext.env.packageInfo.version,
});
const fleetStartPromise = resourcePlugins.fleet?.start();
const taskManager = plugins.taskManager;
// create source map index and run migrations
scheduleSourceMapMigration({
coreStartPromise: getCoreStart(),
pluginStartPromise: getPluginStart(),
fleetStartPromise,
taskManager,
logger: this.logger,
}).catch((e) => {
this.logger?.error('Failed to schedule APM source map migration');
this.logger?.error(e);
});
return {
config$,
getApmIndices: boundGetApmIndices,
@ -254,28 +274,39 @@ export class APMPlugin
};
}
public start(core: CoreStart) {
public start(core: CoreStart, plugins: APMPluginStartDependencies) {
if (this.currentConfig == null || this.logger == null) {
throw new Error('APMPlugin needs to be setup before calling start()');
}
// create agent configuration index without blocking start lifecycle
createApmAgentConfigurationIndex({
client: core.elasticsearch.client.asInternalUser,
config: this.currentConfig,
logger: this.logger,
});
// create custom action index without blocking start lifecycle
createApmCustomLinkIndex({
client: core.elasticsearch.client.asInternalUser,
config: this.currentConfig,
logger: this.logger,
const logger = this.logger;
const client = core.elasticsearch.client.asInternalUser;
// create .apm-agent-configuration index without blocking start lifecycle
createApmAgentConfigurationIndex({ client, logger }).catch((e) => {
logger.error('Failed to create .apm-agent-configuration index');
logger.error(e);
});
migrateLegacyAPMIndicesToSpaceAware({
coreStart: core,
logger: this.logger,
// create .apm-custom-link index without blocking start lifecycle
createApmCustomLinkIndex({ client, logger }).catch((e) => {
logger.error('Failed to create .apm-custom-link index');
logger.error(e);
});
// create .apm-source-map index without blocking start lifecycle
createApmSourceMapIndexTemplate({ client, logger }).catch((e) => {
logger.error('Failed to create apm-source-map index template');
logger.error(e);
});
// TODO: remove in 9.0
migrateLegacyAPMIndicesToSpaceAware({ coreStart: core, logger }).catch(
(e) => {
logger.error('Failed to run migration making APM indices space aware');
logger.error(e);
}
);
}
public stop() {}

45
x-pack/plugins/apm/server/routes/fleet/source_maps.ts
View file

@ -18,32 +18,26 @@ import { APMPluginStartDependencies } from '../../types';
import { getApmPackagePolicies } from './get_apm_package_policies';
import { APM_SERVER, PackagePolicy } from './register_fleet_policy_callbacks';
export interface ApmArtifactBody {
const doUnzip = promisify(unzip);
interface ApmSourceMapArtifactBody {
serviceName: string;
serviceVersion: string;
bundleFilepath: string;
sourceMap: SourceMap;
}
export type ArtifactSourceMap = Omit<Artifact, 'body'> & {
body: ApmArtifactBody;
body: ApmSourceMapArtifactBody;
};
export type FleetPluginStart = NonNullable<APMPluginStartDependencies['fleet']>;
const doUnzip = promisify(unzip);
async function unzipArtifactBody(
artifact: Artifact
): Promise<ArtifactSourceMap> {
const body = await doUnzip(Buffer.from(artifact.body, 'base64'));
return {
...artifact,
body: JSON.parse(body.toString()) as ApmArtifactBody,
};
export async function getUnzippedArtifactBody(artifactBody: string) {
const unzippedBody = await doUnzip(Buffer.from(artifactBody, 'base64'));
return JSON.parse(unzippedBody.toString()) as ApmSourceMapArtifactBody;
}
function getApmArtifactClient(fleetPluginStart: FleetPluginStart) {
export function getApmArtifactClient(fleetPluginStart: FleetPluginStart) {
return fleetPluginStart.createArtifactsClient('apm');
}
@ -66,17 +60,20 @@ export async function listSourceMapArtifacts({
});
const artifacts = await Promise.all(
artifactsResponse.items.map(unzipArtifactBody)
artifactsResponse.items.map(async (item) => {
const body = await getUnzippedArtifactBody(item.body);
return { ...item, body };
})
);
return { artifacts, total: artifactsResponse.total };
}
export async function createApmArtifact({
export async function createFleetSourceMapArtifact({
apmArtifactBody,
fleetPluginStart,
}: {
apmArtifactBody: ApmArtifactBody;
apmArtifactBody: ApmSourceMapArtifactBody;
fleetPluginStart: FleetPluginStart;
}) {
const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
@ -89,7 +86,7 @@ export async function createApmArtifact({
});
}
export async function deleteApmArtifact({
export async function deleteFleetSourcemapArtifact({
id,
fleetPluginStart,
}: {
@ -141,12 +138,12 @@ export async function updateSourceMapsOnFleetPolicies({
core,
fleetPluginStart,
savedObjectsClient,
elasticsearchClient,
internalESClient,
}: {
core: { setup: CoreSetup; start: () => Promise<CoreStart> };
fleetPluginStart: FleetPluginStart;
savedObjectsClient: SavedObjectsClientContract;
elasticsearchClient: ElasticsearchClient;
internalESClient: ElasticsearchClient;
}) {
const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart });
const apmFleetPolicies = await getApmPackagePolicies({
@ -171,7 +168,7 @@ export async function updateSourceMapsOnFleetPolicies({
await fleetPluginStart.packagePolicyService.update(
savedObjectsClient,
elasticsearchClient,
internalESClient,
id,
updatedPackagePolicy
);
@ -179,11 +176,11 @@ export async function updateSourceMapsOnFleetPolicies({
);
}
export function getCleanedBundleFilePath(bundleFilePath: string) {
export function getCleanedBundleFilePath(bundleFilepath: string) {
try {
const cleanedBundleFilepath = new URL(bundleFilePath);
const cleanedBundleFilepath = new URL(bundleFilepath);
return cleanedBundleFilepath.href;
} catch (e) {
return bundleFilePath;
return bundleFilepath;
}
}

224
x-pack/plugins/apm/server/routes/settings/agent_configuration/__snapshots__/queries.test.ts.snap generated
View file

@ -1,10 +1,88 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`agent configuration queries findExactConfiguration find configuration by service.environment 1`] = `undefined`;
exports[`agent configuration queries findExactConfiguration find configuration by service.environment 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"filter": Array [
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.name",
},
},
],
},
},
Object {
"term": Object {
"service.environment": "bar",
},
},
],
},
},
},
"index": ".apm-agent-configuration",
}
`;
exports[`agent configuration queries findExactConfiguration find configuration by service.name 1`] = `undefined`;
exports[`agent configuration queries findExactConfiguration find configuration by service.name 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"filter": Array [
Object {
"term": Object {
"service.name": "foo",
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.environment",
},
},
],
},
},
],
},
},
},
"index": ".apm-agent-configuration",
}
`;
exports[`agent configuration queries findExactConfiguration find configuration by service.name and service.environment 1`] = `undefined`;
exports[`agent configuration queries findExactConfiguration find configuration by service.name and service.environment 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"filter": Array [
Object {
"term": Object {
"service.name": "foo",
},
},
Object {
"term": Object {
"service.environment": "bar",
},
},
],
},
},
},
"index": ".apm-agent-configuration",
}
`;
exports[`agent configuration queries getAllEnvironments fetches all environments 1`] = `
Object {
@ -42,10 +120,142 @@ Object {
}
`;
exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = `undefined`;
exports[`agent configuration queries getExistingEnvironmentsForService fetches unavailable environments 1`] = `
Object {
"body": Object {
"aggs": Object {
"environments": Object {
"terms": Object {
"field": "service.environment",
"missing": "ALL_OPTION_VALUE",
"size": 50,
},
},
},
"query": Object {
"bool": Object {
"filter": Array [
Object {
"term": Object {
"service.name": "foo",
},
},
],
},
},
"size": 0,
},
"index": ".apm-agent-configuration",
}
`;
exports[`agent configuration queries listConfigurations fetches configurations 1`] = `undefined`;
exports[`agent configuration queries listConfigurations fetches configurations 1`] = `
Object {
"index": ".apm-agent-configuration",
"size": 200,
}
`;
exports[`agent configuration queries searchConfigurations fetches filtered configurations with an environment 1`] = `undefined`;
exports[`agent configuration queries searchConfigurations fetches filtered configurations with an environment 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"minimum_should_match": 2,
"should": Array [
Object {
"constant_score": Object {
"boost": 2,
"filter": Object {
"term": Object {
"service.name": "foo",
},
},
},
},
Object {
"constant_score": Object {
"boost": 1,
"filter": Object {
"term": Object {
"service.environment": "bar",
},
},
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.name",
},
},
],
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.environment",
},
},
],
},
},
],
},
},
},
"index": ".apm-agent-configuration",
}
`;
exports[`agent configuration queries searchConfigurations fetches filtered configurations without an environment 1`] = `undefined`;
exports[`agent configuration queries searchConfigurations fetches filtered configurations without an environment 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"minimum_should_match": 2,
"should": Array [
Object {
"constant_score": Object {
"boost": 2,
"filter": Object {
"term": Object {
"service.name": "foo",
},
},
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.name",
},
},
],
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.environment",
},
},
],
},
},
],
},
},
},
"index": ".apm-agent-configuration",
}
`;

8
x-pack/plugins/apm/server/routes/settings/agent_configuration/create_agent_config_index.ts
View file

@ -10,21 +10,17 @@ import {
createOrUpdateIndex,
Mappings,
} from '@kbn/observability-plugin/server';
import { APMConfig } from '../../..';
import { getApmIndicesConfig } from '../apm_indices/get_apm_indices';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
export async function createApmAgentConfigurationIndex({
client,
config,
logger,
}: {
client: ElasticsearchClient;
config: APMConfig;
logger: Logger;
}) {
const index = getApmIndicesConfig(config).apmAgentConfigurationIndex;
return createOrUpdateIndex({
index,
index: APM_AGENT_CONFIGURATION_INDEX,
client,
logger,
mappings,

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/create_or_update_configuration.ts
View file

@ -14,6 +14,7 @@ import {
APMIndexDocumentParams,
APMInternalESClient,
} from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
export function createOrUpdateConfiguration({
configurationId,
@ -26,7 +27,7 @@ export function createOrUpdateConfiguration({
}) {
const params: APMIndexDocumentParams<AgentConfiguration> = {
refresh: true,
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
body: {
agent_name: configurationIntake.agent_name,
service: {

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/delete_configuration.ts
View file

@ -6,6 +6,7 @@
*/
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
export async function deleteConfiguration({
configurationId,
@ -16,7 +17,7 @@ export async function deleteConfiguration({
}) {
const params = {
refresh: 'wait_for' as const,
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
id: configurationId,
};

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/find_exact_configuration.ts
View file

@ -12,6 +12,7 @@ import {
SERVICE_NAME,
} from '../../../../common/es_fields/apm';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
import { convertConfigSettingsToString } from './convert_settings_to_string';
import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet';
@ -31,7 +32,7 @@ export async function findExactConfiguration({
: { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } };
const params = {
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
body: {
query: {
bool: { filter: [serviceNameFilter, environmentFilter] },

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts
View file

@ -11,6 +11,7 @@ import {
} from '../../../../../common/es_fields/apm';
import { ALL_OPTION_VALUE } from '../../../../../common/agent_configuration/all_option';
import { APMInternalESClient } from '../../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../../apm_indices/get_apm_indices';
export async function getExistingEnvironmentsForService({
serviceName,
@ -26,7 +27,7 @@ export async function getExistingEnvironmentsForService({
: { must_not: [{ exists: { field: SERVICE_NAME } }] };
const params = {
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
body: {
size: 0,
query: { bool },

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/list_configurations.ts
View file

@ -9,12 +9,13 @@ import { AgentConfiguration } from '../../../../common/agent_configuration/confi
import { convertConfigSettingsToString } from './convert_settings_to_string';
import { getConfigsAppliedToAgentsThroughFleet } from './get_config_applied_to_agent_through_fleet';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
export async function listConfigurations(
internalESClient: APMInternalESClient
) {
const params = {
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
size: 200,
};

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/mark_applied_by_agent.ts
View file

@ -7,6 +7,7 @@
import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
// We're not wrapping this function with a span as it is not blocking the request
export async function markAppliedByAgent({
@ -19,7 +20,7 @@ export async function markAppliedByAgent({
internalESClient: APMInternalESClient;
}) {
const params = {
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
id, // by specifying the `id` elasticsearch will do an "upsert"
body: {
...body,

3
x-pack/plugins/apm/server/routes/settings/agent_configuration/search_configurations.ts
View file

@ -13,6 +13,7 @@ import {
import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types';
import { convertConfigSettingsToString } from './convert_settings_to_string';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_AGENT_CONFIGURATION_INDEX } from '../apm_indices/get_apm_indices';
export async function searchConfigurations({
service,
@ -47,7 +48,7 @@ export async function searchConfigurations({
: [];
const params = {
index: internalESClient.apmIndices.apmAgentConfigurationIndex,
index: APM_AGENT_CONFIGURATION_INDEX,
body: {
query: {
bool: {

11
x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts
View file

@ -16,16 +16,17 @@ import { withApmSpan } from '../../../utils/with_apm_span';
import { APMIndices } from '../../../saved_objects/apm_indices';
export type ApmIndicesConfig = Readonly<{
sourcemap: string;
error: string;
onboarding: string;
span: string;
transaction: string;
metric: string;
apmAgentConfigurationIndex: string;
apmCustomLinkIndex: string;
}>;
export const APM_AGENT_CONFIGURATION_INDEX = '.apm-agent-configuration';
export const APM_CUSTOM_LINK_INDEX = '.apm-custom-link';
export const APM_SOURCE_MAP_INDEX = '.apm-source-map';
type ISavedObjectsClient = Pick<SavedObjectsClient, 'get'>;
async function getApmIndicesSavedObject(
@ -44,15 +45,11 @@ async function getApmIndicesSavedObject(
export function getApmIndicesConfig(config: APMConfig): ApmIndicesConfig {
return {
sourcemap: config.indices.sourcemap,
error: config.indices.error,
onboarding: config.indices.onboarding,
span: config.indices.span,
transaction: config.indices.transaction,
metric: config.indices.metric,
// system indices, not configurable
apmAgentConfigurationIndex: '.apm-agent-configuration',
apmCustomLinkIndex: '.apm-custom-link',
};
}

2
x-pack/plugins/apm/server/routes/settings/apm_indices/route.ts
View file

@ -25,7 +25,6 @@ const apmIndexSettingsRoute = createApmServerRoute({
| 'span'
| 'error'
| 'metric'
| 'sourcemap'
| 'onboarding';
defaultValue: string;
savedValue: string | undefined;
@ -66,7 +65,6 @@ const saveApmIndicesRoute = createApmServerRoute({
},
params: t.type({
body: t.partial({
sourcemap: t.string,
error: t.string,
onboarding: t.string,
span: t.string,

89
x-pack/plugins/apm/server/routes/settings/custom_link/__snapshots__/list_custom_links.test.ts.snap generated
View file

@ -1,5 +1,90 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`List Custom Links fetches all custom links 1`] = `undefined`;
exports[`List Custom Links fetches all custom links 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"filter": Array [],
},
},
"sort": Array [
Object {
"label.keyword": Object {
"order": "asc",
},
},
],
},
"index": ".apm-custom-link",
"size": 500,
}
`;
exports[`List Custom Links filters custom links 1`] = `undefined`;
exports[`List Custom Links filters custom links 1`] = `
Object {
"body": Object {
"query": Object {
"bool": Object {
"filter": Array [
Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"term": Object {
"service.name": "foo",
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "service.name",
},
},
],
},
},
],
},
},
Object {
"bool": Object {
"minimum_should_match": 1,
"should": Array [
Object {
"term": Object {
"transaction.name": "bar",
},
},
Object {
"bool": Object {
"must_not": Array [
Object {
"exists": Object {
"field": "transaction.name",
},
},
],
},
},
],
},
},
],
},
},
"sort": Array [
Object {
"label.keyword": Object {
"order": "asc",
},
},
],
},
"index": ".apm-custom-link",
"size": 500,
}
`;

8
x-pack/plugins/apm/server/routes/settings/custom_link/create_custom_link_index.ts
View file

@ -11,21 +11,17 @@ import {
createOrUpdateIndex,
Mappings,
} from '@kbn/observability-plugin/server';
import { APMConfig } from '../../..';
import { getApmIndicesConfig } from '../apm_indices/get_apm_indices';
import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices';
export const createApmCustomLinkIndex = async ({
client,
config,
logger,
}: {
client: ElasticsearchClient;
config: APMConfig;
logger: Logger;
}) => {
const index = getApmIndicesConfig(config).apmCustomLinkIndex;
return createOrUpdateIndex({
index,
index: APM_CUSTOM_LINK_INDEX,
client,
logger,
mappings,

10
x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.test.ts
View file

@ -8,18 +8,12 @@
import { mockNow } from '../../../utils/test_helpers';
import { CustomLink } from '../../../../common/custom_link/custom_link_types';
import { createOrUpdateCustomLink } from './create_or_update_custom_link';
import { ApmIndicesConfig } from '../apm_indices/get_apm_indices';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
describe('Create or Update Custom link', () => {
const internalClientIndexMock = jest.fn();
const mockIndices = {
apmCustomLinkIndex: 'apmCustomLinkIndex',
} as unknown as ApmIndicesConfig;
const mockInternalESClient = {
apmIndices: mockIndices,
index: internalClientIndexMock,
} as unknown as APMInternalESClient;
@ -48,7 +42,7 @@ describe('Create or Update Custom link', () => {
'create_or_update_custom_link',
{
refresh: true,
index: 'apmCustomLinkIndex',
index: '.apm-custom-link',
body: {
'@timestamp': 1570737000000,
label: 'foo',
@ -69,7 +63,7 @@ describe('Create or Update Custom link', () => {
'create_or_update_custom_link',
{
refresh: true,
index: 'apmCustomLinkIndex',
index: '.apm-custom-link',
id: 'bar',
body: {
'@timestamp': 1570737000000,

3
x-pack/plugins/apm/server/routes/settings/custom_link/create_or_update_custom_link.ts
View file

@ -14,6 +14,7 @@ import {
APMIndexDocumentParams,
APMInternalESClient,
} from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices';
export function createOrUpdateCustomLink({
customLinkId,
@ -26,7 +27,7 @@ export function createOrUpdateCustomLink({
}) {
const params: APMIndexDocumentParams<CustomLinkES> = {
refresh: true,
index: internalESClient.apmIndices.apmCustomLinkIndex,
index: APM_CUSTOM_LINK_INDEX,
body: {
'@timestamp': Date.now(),
...toESFormat(customLink),

3
x-pack/plugins/apm/server/routes/settings/custom_link/delete_custom_link.ts
View file

@ -6,6 +6,7 @@
*/
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices';
export function deleteCustomLink({
customLinkId,
@ -16,7 +17,7 @@ export function deleteCustomLink({
}) {
const params = {
refresh: 'wait_for' as const,
index: internalESClient.apmIndices.apmCustomLinkIndex,
index: APM_CUSTOM_LINK_INDEX,
id: customLinkId,
};

3
x-pack/plugins/apm/server/routes/settings/custom_link/list_custom_links.ts
View file

@ -14,6 +14,7 @@ import {
import { fromESFormat } from './helper';
import { filterOptionsRt } from './custom_link_types';
import { APMInternalESClient } from '../../../lib/helpers/create_es_client/create_internal_es_client';
import { APM_CUSTOM_LINK_INDEX } from '../apm_indices/get_apm_indices';
export async function listCustomLinks({
internalESClient,
@ -35,7 +36,7 @@ export async function listCustomLinks({
});
const params = {
index: internalESClient.apmIndices.apmCustomLinkIndex,
index: APM_CUSTOM_LINK_INDEX,
size: 500,
body: {
query: {

57
x-pack/plugins/apm/server/routes/source_maps/bulk_create_apm_source_maps.ts Normal file
View file

@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { Artifact } from '@kbn/fleet-plugin/server';
import { getUnzippedArtifactBody } from '../fleet/source_maps';
import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
import { ApmSourceMap } from './create_apm_source_map_index_template';
import { getEncodedContent, getSourceMapId } from './sourcemap_utils';
export async function bulkCreateApmSourceMaps({
artifacts,
internalESClient,
}: {
artifacts: Artifact[];
internalESClient: ElasticsearchClient;
}) {
const docs = await Promise.all(
artifacts.map(async (artifact): Promise<ApmSourceMap> => {
const { serviceName, serviceVersion, bundleFilepath, sourceMap } =
await getUnzippedArtifactBody(artifact.body);
const { contentEncoded, contentHash } = await getEncodedContent(
sourceMap
);
return {
fleet_id: artifact.id,
created: artifact.created,
content: contentEncoded,
content_sha256: contentHash,
file: {
path: bundleFilepath,
},
service: {
name: serviceName,
version: serviceVersion,
},
};
})
);
return internalESClient.bulk<ApmSourceMap>({
body: docs.flatMap((doc) => {
const id = getSourceMapId({
serviceName: doc.service.name,
serviceVersion: doc.service.version,
bundleFilepath: doc.file.path,
});
return [{ create: { _index: APM_SOURCE_MAP_INDEX, _id: id } }, doc];
}),
});
}

61
x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map.ts Normal file
View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { isElasticsearchVersionConflictError } from '@kbn/fleet-plugin/server/errors/utils';
import { Logger } from '@kbn/core/server';
import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
import { ApmSourceMap } from './create_apm_source_map_index_template';
import { SourceMap } from './route';
import { getEncodedContent, getSourceMapId } from './sourcemap_utils';
export async function createApmSourceMap({
internalESClient,
logger,
fleetId,
created,
sourceMapContent,
bundleFilepath,
serviceName,
serviceVersion,
}: {
internalESClient: ElasticsearchClient;
logger: Logger;
fleetId: string;
created: string;
sourceMapContent: SourceMap;
bundleFilepath: string;
serviceName: string;
serviceVersion: string;
}) {
const { contentEncoded, contentHash } = await getEncodedContent(
sourceMapContent
);
const doc: ApmSourceMap = {
fleet_id: fleetId,
created,
content: contentEncoded,
content_sha256: contentHash,
file: { path: bundleFilepath },
service: { name: serviceName, version: serviceVersion },
};
try {
const id = getSourceMapId({ serviceName, serviceVersion, bundleFilepath });
logger.debug(`Create APM source map: "${id}"`);
return await internalESClient.create<ApmSourceMap>({
index: APM_SOURCE_MAP_INDEX,
id,
body: doc,
});
} catch (e) {
// we ignore 409 errors from the create (document already exists)
if (!isElasticsearchVersionConflictError(e)) {
throw e;
}
}
}

88
x-pack/plugins/apm/server/routes/source_maps/create_apm_source_map_index_template.ts Normal file
View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient, Logger } from '@kbn/core/server';
import { createOrUpdateIndexTemplate } from '@kbn/observability-plugin/server';
import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
const indexTemplate: IndicesPutIndexTemplateRequest = {
name: 'apm-source-map',
body: {
version: 1,
index_patterns: [APM_SOURCE_MAP_INDEX],
template: {
settings: {
number_of_shards: 1,
index: {
hidden: true,
},
},
mappings: {
dynamic: 'strict',
properties: {
fleet_id: {
type: 'keyword',
},
created: {
type: 'date',
},
content: {
type: 'binary',
},
content_sha256: {
type: 'keyword',
},
'file.path': {
type: 'keyword',
},
'service.name': {
type: 'keyword',
},
'service.version': {
type: 'keyword',
},
},
},
},
},
};
export async function createApmSourceMapIndexTemplate({
client,
logger,
}: {
client: ElasticsearchClient;
logger: Logger;
}) {
// create index template
await createOrUpdateIndexTemplate({ indexTemplate, client, logger });
// create index if it doesn't exist
const indexExists = await client.indices.exists({
index: APM_SOURCE_MAP_INDEX,
});
if (!indexExists) {
logger.debug(`Create index: "${APM_SOURCE_MAP_INDEX}"`);
await client.indices.create({ index: APM_SOURCE_MAP_INDEX });
}
}
export interface ApmSourceMap {
fleet_id: string;
created: string;
content: string;
content_sha256: string;
file: {
path: string;
};
service: {
name: string;
version: string;
};
}

26
x-pack/plugins/apm/server/routes/source_maps/delete_apm_sourcemap.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
export async function deleteApmSourceMap({
internalESClient,
fleetId,
}: {
internalESClient: ElasticsearchClient;
fleetId: string;
}) {
return internalESClient.deleteByQuery({
index: APM_SOURCE_MAP_INDEX,
query: {
bool: {
filter: [{ term: { fleet_id: fleetId } }],
},
},
});
}

65
x-pack/plugins/apm/server/routes/source_maps/route.ts
View file

@ -10,8 +10,8 @@ import { SavedObjectsClientContract } from '@kbn/core/server';
import { jsonRt, toNumberRt } from '@kbn/io-ts-utils';
import { Artifact } from '@kbn/fleet-plugin/server';
import {
createApmArtifact,
deleteApmArtifact,
createFleetSourceMapArtifact,
deleteFleetSourcemapArtifact,
listSourceMapArtifacts,
updateSourceMapsOnFleetPolicies,
getCleanedBundleFilePath,
@ -20,6 +20,9 @@ import {
import { getInternalSavedObjectsClient } from '../../lib/helpers/get_internal_saved_objects_client';
import { createApmServerRoute } from '../apm_routes/create_apm_server_route';
import { stringFromBufferRt } from '../../utils/string_from_buffer_rt';
import { createApmSourceMap } from './create_apm_source_map';
import { deleteApmSourceMap } from './delete_apm_sourcemap';
import { runFleetSourcemapArtifactsMigration } from './schedule_source_map_migration';
export const sourceMapRt = t.intersection([
t.type({
@ -89,35 +92,55 @@ const uploadSourceMapRoute = createApmServerRoute({
.pipe(sourceMapRt),
}),
}),
handler: async ({ params, plugins, core }): Promise<Artifact | undefined> => {
handler: async ({
params,
plugins,
core,
logger,
}): Promise<Artifact | undefined> => {
const {
service_name: serviceName,
service_version: serviceVersion,
bundle_filepath: bundleFilepath,
sourcemap: sourceMap,
sourcemap: sourceMapContent,
} = params.body;
const cleanedBundleFilepath = getCleanedBundleFilePath(bundleFilepath);
const fleetPluginStart = await plugins.fleet?.start();
const coreStart = await core.start();
const esClient = coreStart.elasticsearch.client.asInternalUser;
const internalESClient = coreStart.elasticsearch.client.asInternalUser;
const savedObjectsClient = await getInternalSavedObjectsClient(core.setup);
try {
if (fleetPluginStart) {
const artifact = await createApmArtifact({
// create source map as fleet artifact
const artifact = await createFleetSourceMapArtifact({
fleetPluginStart,
apmArtifactBody: {
serviceName,
serviceVersion,
bundleFilepath: cleanedBundleFilepath,
sourceMap,
sourceMap: sourceMapContent,
},
});
// sync source map to APM managed index
await createApmSourceMap({
internalESClient,
logger,
fleetId: artifact.id,
created: artifact.created,
sourceMapContent,
bundleFilepath: cleanedBundleFilepath,
serviceName,
serviceVersion,
});
// sync source map to fleet policy
await updateSourceMapsOnFleetPolicies({
core,
fleetPluginStart,
savedObjectsClient:
savedObjectsClient as unknown as SavedObjectsClientContract,
elasticsearchClient: esClient,
internalESClient,
});
return artifact;
@ -143,17 +166,18 @@ const deleteSourceMapRoute = createApmServerRoute({
const fleetPluginStart = await plugins.fleet?.start();
const { id } = params.path;
const coreStart = await core.start();
const esClient = coreStart.elasticsearch.client.asInternalUser;
const internalESClient = coreStart.elasticsearch.client.asInternalUser;
const savedObjectsClient = await getInternalSavedObjectsClient(core.setup);
try {
if (fleetPluginStart) {
await deleteApmArtifact({ id, fleetPluginStart });
await deleteFleetSourcemapArtifact({ id, fleetPluginStart });
await deleteApmSourceMap({ internalESClient, fleetId: id });
await updateSourceMapsOnFleetPolicies({
core,
fleetPluginStart,
savedObjectsClient:
savedObjectsClient as unknown as SavedObjectsClientContract,
elasticsearchClient: esClient,
internalESClient,
});
}
} catch (e) {
@ -165,8 +189,27 @@ const deleteSourceMapRoute = createApmServerRoute({
},
});
const migrateFleetArtifactsSourceMapRoute = createApmServerRoute({
endpoint: 'POST /internal/apm/sourcemaps/migrate_fleet_artifacts',
options: { tags: ['access:apm', 'access:apm_write'] },
handler: async ({ plugins, core, logger }): Promise<void> => {
const fleet = await plugins.fleet?.start();
const coreStart = await core.start();
const internalESClient = coreStart.elasticsearch.client.asInternalUser;
if (fleet) {
return runFleetSourcemapArtifactsMigration({
fleet,
internalESClient,
logger,
});
}
},
});
export const sourceMapsRouteRepository = {
...listSourceMapRoute,
...uploadSourceMapRoute,
...deleteSourceMapRoute,
...migrateFleetArtifactsSourceMapRoute,
};

225
x-pack/plugins/apm/server/routes/source_maps/schedule_source_map_migration.ts Normal file
View file

@ -0,0 +1,225 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { FleetStartContract } from '@kbn/fleet-plugin/server';
import { FleetArtifactsClient } from '@kbn/fleet-plugin/server/services';
import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server';
import { CoreStart, Logger } from '@kbn/core/server';
import { getApmArtifactClient } from '../fleet/source_maps';
import { bulkCreateApmSourceMaps } from './bulk_create_apm_source_maps';
import { APM_SOURCE_MAP_INDEX } from '../settings/apm_indices/get_apm_indices';
import { ApmSourceMap } from './create_apm_source_map_index_template';
import { APMPluginStartDependencies } from '../../types';
import { createApmSourceMapIndexTemplate } from './create_apm_source_map_index_template';
const PER_PAGE = 10;
const TASK_ID = 'apm-source-map-migration-task-id';
const TASK_TYPE = 'apm-source-map-migration-task';
export async function scheduleSourceMapMigration({
coreStartPromise,
pluginStartPromise,
fleetStartPromise,
taskManager,
logger,
}: {
coreStartPromise: Promise<CoreStart>;
pluginStartPromise: Promise<APMPluginStartDependencies>;
fleetStartPromise?: Promise<FleetStartContract>;
taskManager?: TaskManagerSetupContract;
logger: Logger;
}) {
if (!taskManager) {
return;
}
logger.debug(`Register task "${TASK_TYPE}"`);
taskManager.registerTaskDefinitions({
[TASK_TYPE]: {
title: 'Migrate fleet source map artifacts',
description:
'Migrates fleet source map artifacts to `.apm-source-map` index',
timeout: '1h',
maxAttempts: 5,
maxConcurrency: 1,
createTaskRunner() {
const taskState: TaskState = { isAborted: false };
return {
async run() {
logger.debug(`Run task: "${TASK_TYPE}"`);
const coreStart = await coreStartPromise;
const internalESClient =
coreStart.elasticsearch.client.asInternalUser;
// ensure that the index template has been created before running migration
await createApmSourceMapIndexTemplate({
client: internalESClient,
logger,
});
const fleet = await fleetStartPromise;
if (fleet) {
await runFleetSourcemapArtifactsMigration({
taskState,
fleet,
internalESClient,
logger,
});
}
},
async cancel() {
taskState.isAborted = true;
logger.debug(`Task cancelled: "${TASK_TYPE}"`);
},
};
},
},
});
const pluginStart = await pluginStartPromise;
const taskManagerStart = pluginStart.taskManager;
if (taskManagerStart) {
logger.debug(`Task scheduled: "${TASK_TYPE}"`);
await pluginStart.taskManager?.ensureScheduled({
id: TASK_ID,
taskType: TASK_TYPE,
scope: ['apm'],
params: {},
state: {},
});
}
}
interface TaskState {
isAborted: boolean;
}
export async function runFleetSourcemapArtifactsMigration({
taskState,
fleet,
internalESClient,
logger,
}: {
taskState?: TaskState;
fleet: FleetStartContract;
internalESClient: ElasticsearchClient;
logger: Logger;
}) {
try {
const latestApmSourceMapTimestamp = await getLatestApmSourceMap(
internalESClient
);
const createdDateFilter = latestApmSourceMapTimestamp
? ` AND created:>${asLuceneEncoding(latestApmSourceMapTimestamp)}`
: '';
await paginateArtifacts({
taskState,
page: 1,
apmArtifactClient: getApmArtifactClient(fleet),
kuery: `type: sourcemap${createdDateFilter}`,
logger,
internalESClient,
});
} catch (e) {
logger.error('Failed to migrate APM fleet source map artifacts');
logger.error(e);
}
}
// will convert "2022-12-12T21:21:51.203Z" to "2022-12-12T21\:21\:51.203Z" because colons are not allowed when using Lucene syntax
function asLuceneEncoding(timestamp: string) {
return timestamp.replaceAll(':', '\\:');
}
async function getArtifactsForPage({
page,
apmArtifactClient,
kuery,
}: {
page: number;
apmArtifactClient: FleetArtifactsClient;
kuery: string;
}) {
return await apmArtifactClient.listArtifacts({
kuery,
perPage: PER_PAGE,
page,
sortOrder: 'asc',
sortField: 'created',
});
}
async function paginateArtifacts({
taskState,
page,
apmArtifactClient,
kuery,
logger,
internalESClient,
}: {
taskState?: TaskState;
page: number;
apmArtifactClient: FleetArtifactsClient;
kuery: string;
logger: Logger;
internalESClient: ElasticsearchClient;
}) {
if (taskState?.isAborted) {
return;
}
const { total, items: artifacts } = await getArtifactsForPage({
page,
apmArtifactClient,
kuery,
});
if (artifacts.length === 0) {
logger.debug('No source maps need to be migrated');
return;
}
const migratedCount = (page - 1) * PER_PAGE + artifacts.length;
logger.info(`Migrating ${migratedCount} of ${total} source maps`);
await bulkCreateApmSourceMaps({ artifacts, internalESClient });
const hasMorePages = total > migratedCount;
if (hasMorePages) {
await paginateArtifacts({
taskState,
page: page + 1,
apmArtifactClient,
kuery,
logger,
internalESClient,
});
} else {
logger.info(`Successfully migrated ${total} source maps`);
}
}
async function getLatestApmSourceMap(internalESClient: ElasticsearchClient) {
const params = {
index: APM_SOURCE_MAP_INDEX,
track_total_hits: false,
size: 1,
_source: ['created'],
sort: [{ created: { order: 'desc' } }],
body: {
query: { match_all: {} },
},
};
const res = await internalESClient.search<ApmSourceMap>(params);
return res.hits.hits[0]?._source?.created;
}

37
x-pack/plugins/apm/server/routes/source_maps/sourcemap_utils.ts Normal file
View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { deflate } from 'zlib';
import { BinaryLike, createHash } from 'crypto';
import { promisify } from 'util';
import { SourceMap } from './route';
const deflateAsync = promisify(deflate);
function asSha256Encoded(content: BinaryLike): string {
return createHash('sha256').update(content).digest('hex');
}
export async function getEncodedContent(sourceMapContent: SourceMap) {
const contentBuffer = Buffer.from(JSON.stringify(sourceMapContent));
const contentZipped = await deflateAsync(contentBuffer);
const contentEncoded = contentZipped.toString('base64');
const contentHash = asSha256Encoded(contentZipped);
return { contentEncoded, contentHash };
}
export function getSourceMapId({
serviceName,
serviceVersion,
bundleFilepath,
}: {
serviceName: string;
serviceVersion: string;
bundleFilepath: string;
}) {
return [serviceName, serviceVersion, bundleFilepath].join('-');
}

1
x-pack/plugins/apm/server/routes/storage_explorer/is_cross_cluster_search.test.ts
View file

@ -44,7 +44,6 @@ describe('isCrossClusterSearch', () => {
metric: '',
error: '',
onboarding: 'apm-*,remote_cluster:apm-*',
sourcemap: 'apm-*,remote_cluster:apm-*',
} as ApmIndicesConfig,
} as unknown as APMEventClient;

1
x-pack/plugins/apm/server/saved_objects/apm_indices.ts
View file

@ -11,7 +11,6 @@ import { updateApmOssIndexPaths } from './migrations/update_apm_oss_index_paths'
export interface APMIndices {
apmIndices?: {
sourcemap?: string;
error?: string;
onboarding?: string;
span?: string;

1
x-pack/plugins/apm/server/saved_objects/migrations/update_apm_oss_index_paths.ts
View file

@ -6,7 +6,6 @@
*/
const apmIndexConfigs = [
['sourcemap', 'apm_oss.sourcemapIndices'],
['error', 'apm_oss.errorIndices'],
['onboarding', 'apm_oss.onboardingIndices'],
['span', 'apm_oss.spanIndices'],

9
x-pack/plugins/apm/server/utils/test_helpers.tsx
View file

@ -52,7 +52,6 @@ export async function inspectSearchParams(
const indices: {
[Property in keyof APMConfig['indices']]: string;
} = {
sourcemap: 'myIndex',
error: 'myIndex',
onboarding: 'myIndex',
span: 'myIndex',
@ -86,14 +85,10 @@ export async function inspectSearchParams(
}
) as APMConfig;
const mockInternalESClient = { search: spy } as any;
const mockIndices = {
...indices,
apmAgentConfigurationIndex: 'myIndex',
apmCustomLinkIndex: 'myIndex',
};
try {
response = await fn({
mockIndices,
mockIndices: indices,
mockApmEventClient,
mockConfig,
mockInternalESClient,

3
x-pack/plugins/observability/common/typings.ts
View file

@ -19,14 +19,11 @@ export const alertWorkflowStatusRt = t.keyof({
export type AlertWorkflowStatus = t.TypeOf<typeof alertWorkflowStatusRt>;
export interface ApmIndicesConfig {
sourcemap: string;
error: string;
onboarding: string;
span: string;
transaction: string;
metric: string;
apmAgentConfigurationIndex: string;
apmCustomLinkIndex: string;
}
export type AlertStatus =

8
x-pack/plugins/observability/server/index.ts
View file

@ -12,6 +12,7 @@ import { schema, TypeOf } from '@kbn/config-schema';
import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server';
import { ObservabilityPlugin, ObservabilityPluginSetup } from './plugin';
import { createOrUpdateIndex, Mappings } from './utils/create_or_update_index';
import { createOrUpdateIndexTemplate } from './utils/create_or_update_index_template';
import { ScopedAnnotationsClient } from './lib/annotations/bootstrap_annotations';
import {
unwrapEsResponse,
@ -61,6 +62,11 @@ export const plugin = (initContext: PluginInitializerContext) =>
new ObservabilityPlugin(initContext);
export type { Mappings, ObservabilityPluginSetup, ScopedAnnotationsClient };
export { createOrUpdateIndex, unwrapEsResponse, WrappedElasticsearchClientError };
export {
createOrUpdateIndex,
createOrUpdateIndexTemplate,
unwrapEsResponse,
WrappedElasticsearchClientError,
};
export { uiSettings } from './ui_settings';

55
x-pack/plugins/observability/server/utils/create_or_update_index_template.ts Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import pRetry from 'p-retry';
import { Logger, ElasticsearchClient } from '@kbn/core/server';
import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
export async function createOrUpdateIndexTemplate({
indexTemplate,
client,
logger,
}: {
indexTemplate: IndicesPutIndexTemplateRequest;
client: ElasticsearchClient;
logger: Logger;
}) {
try {
/*
* In some cases we could be trying to create the index template before ES is ready.
* When this happens, we retry creating the template with exponential backoff.
* We use retry's default formula, meaning that the first retry happens after 2s,
* the 5th after 32s, and the final attempt after around 17m. If the final attempt fails,
* the error is logged to the console.
* See https://github.com/sindresorhus/p-retry and https://github.com/tim-kos/node-retry.
*/
return await pRetry(
async () => {
logger.debug(
`Create index template: "${indexTemplate.name}" for index pattern "${indexTemplate.body?.index_patterns}"`
);
const result = await client.indices.putIndexTemplate(indexTemplate);
if (!result.acknowledged) {
// @ts-expect-error
const resultError = JSON.stringify(result?.body?.error);
throw new Error(resultError);
}
return result;
},
{
onFailedAttempt: (e) => {
logger.warn(`Could not create index template: '${indexTemplate.name}'. Retrying...`);
logger.warn(e);
},
}
);
} catch (e) {
logger.error(`Could not create index template: '${indexTemplate.name}'. Error: ${e.message}.`);
}
}

17
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -3876,16 +3876,6 @@
}
}
},
"sourcemap": {
"properties": {
"1d": {
"type": "long"
},
"all": {
"type": "long"
}
}
},
"onboarding": {
"properties": {
"1d": {
@ -4042,13 +4032,6 @@
}
}
},
"sourcemap": {
"properties": {
"ms": {
"type": "long"
}
}
},
"onboarding": {
"properties": {
"ms": {

1
x-pack/plugins/translations/translations/fr-FR.json
View file

@ -7823,7 +7823,6 @@
"xpack.apm.settings.apmIndices.metricsIndicesLabel": "Index des indicateurs",
"xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "Votre rôle d'utilisateur ne dispose pas d'autorisations pour changer les index APM",
"xpack.apm.settings.apmIndices.onboardingIndicesLabel": "Intégration des index",
"xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "Index des source maps",
"xpack.apm.settings.apmIndices.spanIndicesLabel": "Index des intervalles",
"xpack.apm.settings.apmIndices.title": "Index",
"xpack.apm.settings.apmIndices.transactionIndicesLabel": "Index des transactions",

1
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -7812,7 +7812,6 @@
"xpack.apm.settings.apmIndices.metricsIndicesLabel": "メトリックインデックス",
"xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "ユーザーロールには、APMインデックスを変更する権限がありません",
"xpack.apm.settings.apmIndices.onboardingIndicesLabel": "オンボーディングインデックス",
"xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "ソースマップインデックス",
"xpack.apm.settings.apmIndices.spanIndicesLabel": "スパンインデックス",
"xpack.apm.settings.apmIndices.title": "インデックス",
"xpack.apm.settings.apmIndices.transactionIndicesLabel": "トランザクションインデックス",

1
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -7826,7 +7826,6 @@
"xpack.apm.settings.apmIndices.metricsIndicesLabel": "指标索引",
"xpack.apm.settings.apmIndices.noPermissionTooltipLabel": "您的用户角色无权更改 APM 索引",
"xpack.apm.settings.apmIndices.onboardingIndicesLabel": "载入索引",
"xpack.apm.settings.apmIndices.sourcemapIndicesLabel": "源地图索引",
"xpack.apm.settings.apmIndices.spanIndicesLabel": "跨度索引",
"xpack.apm.settings.apmIndices.title": "索引",
"xpack.apm.settings.apmIndices.transactionIndicesLabel": "事务索引",

2
x-pack/test/apm_api_integration/common/config.ts
View file

@ -28,7 +28,7 @@ import { RegistryProvider } from './registry';
export interface ApmFtrConfig {
name: APMFtrConfigName;
license: 'basic' | 'trial';
kibanaConfig?: Record<string, string | string[]>;
kibanaConfig?: Record<string, any>;
}
async function getApmApiClient({

9
x-pack/test/apm_api_integration/configs/index.ts
View file

@ -8,17 +8,25 @@
import { mapValues } from 'lodash';
import { createTestConfig, CreateTestConfig } from '../common/config';
const apmDebugLogger = {
name: 'plugins.apm',
level: 'debug',
appenders: ['console'],
};
const apmFtrConfigs = {
basic: {
license: 'basic' as const,
kibanaConfig: {
'xpack.apm.forceSyntheticSource': 'true',
'logging.loggers': [apmDebugLogger],
},
},
trial: {
license: 'trial' as const,
kibanaConfig: {
'xpack.apm.forceSyntheticSource': 'true',
'logging.loggers': [apmDebugLogger],
},
},
rules: {
@ -26,6 +34,7 @@ const apmFtrConfigs = {
kibanaConfig: {
'xpack.ruleRegistry.write.enabled': 'true',
'xpack.apm.forceSyntheticSource': 'true',
'logging.loggers': [apmDebugLogger],
},
},
};

219
x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts
View file

@ -4,7 +4,8 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import type { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
import type { ApmSourceMap } from '@kbn/apm-plugin/server/routes/source_maps/create_apm_source_map_index_template';
import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route';
import expect from '@kbn/expect';
import { first, last, times } from 'lodash';
@ -13,6 +14,65 @@ import { FtrProviderContext } from '../../common/ftr_provider_context';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const apmApiClient = getService('apmApiClient');
const esClient = getService('es');
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function waitFor(
cb: () => Promise<boolean>,
{ retries = 50, delay = 100 }: { retries?: number; delay?: number } = {}
): Promise<void> {
if (retries === 0) {
throw new Error(`Maximum number of retries reached`);
}
const res = await cb();
if (!res) {
await sleep(delay);
return waitFor(cb, { retries: retries - 1, delay });
}
}
async function waitForSourceMapCount(
count: number,
{ retries, delay }: { retries?: number; delay?: number } = {}
) {
await waitFor(
async () => {
const res = await esClient.search({
index: '.apm-source-map',
size: 0,
track_total_hits: true,
});
// @ts-expect-error
return res.hits.total.value === count;
},
{ retries, delay }
);
return count;
}
async function deleteAllApmSourceMaps() {
await esClient.deleteByQuery({
index: '.apm-source-map*',
refresh: true,
query: { match_all: {} },
});
}
async function deleteAllFleetSourceMaps() {
return esClient.deleteByQuery({
index: '.fleet-artifacts*',
refresh: true,
query: {
bool: {
filter: [{ term: { type: 'sourcemap' } }, { term: { package_name: 'apm' } }],
},
},
});
}
async function uploadSourcemap({
bundleFilePath,
@ -40,6 +100,12 @@ export default function ApiTest({ getService }: FtrProviderContext) {
return response.body;
}
async function runSourceMapMigration() {
await apmApiClient.writeUser({
endpoint: 'POST /internal/apm/sourcemaps/migrate_fleet_artifacts',
});
}
async function deleteSourcemap(id: string) {
await apmApiClient.writeUser({
endpoint: 'DELETE /api/apm/sourcemaps/{id}',
@ -58,6 +124,10 @@ export default function ApiTest({ getService }: FtrProviderContext) {
}
registry.when('source maps', { config: 'basic', archives: [] }, () => {
before(async () => {
await Promise.all([deleteAllFleetSourceMaps(), deleteAllApmSourceMaps()]);
});
let resp: APIReturnType<'POST /api/apm/sourcemaps'>;
describe('upload source map', () => {
after(async () => {
@ -67,9 +137,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});
it('can upload a source map', async () => {
before(async () => {
resp = await uploadSourcemap({
serviceName: 'my_service',
serviceName: 'uploading-test',
serviceVersion: '1.0.0',
bundleFilePath: 'bar',
sourcemap: {
@ -78,17 +148,50 @@ export default function ApiTest({ getService }: FtrProviderContext) {
mappings: '',
},
});
expect(resp).to.not.empty();
await waitForSourceMapCount(1);
});
it('is uploaded as a fleet artifact', async () => {
const res = await esClient.search({
index: '.fleet-artifacts',
size: 1,
query: {
bool: {
filter: [{ term: { type: 'sourcemap' } }, { term: { package_name: 'apm' } }],
},
},
});
// @ts-expect-error
expect(res.hits.hits[0]._source.identifier).to.be('uploading-test-1.0.0');
});
it('is added to .apm-source-map index', async () => {
const res = await esClient.search({
index: '.apm-source-map',
});
const doc = res.hits.hits[0]._source as ApmSourceMap;
expect(doc.content).to.be(
'eJyrVipLLSrOzM9TsjI0MtZRKs4vLUpOLVayilZSitVRyk0sKMjMSwfylZRqAURLDgo='
);
expect(doc.content_sha256).to.be(
'02dd950aa88a66183d312a7a5f44d72fc9e3914cdbbe5e3a04f1509a8a3d7d83'
);
expect(doc.file.path).to.be('bar');
expect(doc.service.name).to.be('uploading-test');
expect(doc.service.version).to.be('1.0.0');
});
});
describe('list source maps', async () => {
const uploadedSourcemapIds: string[] = [];
before(async () => {
const sourcemapCount = times(15);
const totalCount = 6;
const sourcemapCount = times(totalCount);
for (const i of sourcemapCount) {
const sourcemap = await uploadSourcemap({
serviceName: 'my_service',
await uploadSourcemap({
serviceName: 'list-test',
serviceVersion: `1.0.${i}`,
bundleFilePath: 'bar',
sourcemap: {
@ -97,44 +200,44 @@ export default function ApiTest({ getService }: FtrProviderContext) {
mappings: '',
},
});
uploadedSourcemapIds.push(sourcemap.id);
}
await waitForSourceMapCount(totalCount);
});
after(async () => {
await Promise.all(uploadedSourcemapIds.map((id) => deleteSourcemap(id)));
await Promise.all([deleteAllFleetSourceMaps(), deleteAllApmSourceMaps()]);
});
describe('pagination', () => {
it('can retrieve the first page', async () => {
const firstPageItems = await listSourcemaps({ page: 1, perPage: 5 });
expect(first(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.14');
expect(last(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.10');
expect(firstPageItems.artifacts.length).to.be(5);
expect(firstPageItems.total).to.be(15);
const res = await listSourcemaps({ page: 1, perPage: 2 });
expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.5');
expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.4');
expect(res.artifacts.length).to.be(2);
expect(res.total).to.be(6);
});
it('can retrieve the second page', async () => {
const secondPageItems = await listSourcemaps({ page: 2, perPage: 5 });
expect(first(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.9');
expect(last(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.5');
expect(secondPageItems.artifacts.length).to.be(5);
expect(secondPageItems.total).to.be(15);
const res = await listSourcemaps({ page: 2, perPage: 2 });
expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.3');
expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.2');
expect(res.artifacts.length).to.be(2);
expect(res.total).to.be(6);
});
it('can retrieve the third page', async () => {
const thirdPageItems = await listSourcemaps({ page: 3, perPage: 5 });
expect(first(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.4');
expect(last(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.0');
expect(thirdPageItems.artifacts.length).to.be(5);
expect(thirdPageItems.total).to.be(15);
const res = await listSourcemaps({ page: 3, perPage: 2 });
expect(first(res.artifacts)?.identifier).to.eql('list-test-1.0.1');
expect(last(res.artifacts)?.identifier).to.eql('list-test-1.0.0');
expect(res.artifacts.length).to.be(2);
expect(res.total).to.be(6);
});
});
it('can list source maps', async () => {
it('can list source maps without specifying pagination options', async () => {
const sourcemaps = await listSourcemaps();
expect(sourcemaps.artifacts.length).to.be(15);
expect(sourcemaps.total).to.be(15);
expect(sourcemaps.artifacts.length).to.be(6);
expect(sourcemaps.total).to.be(6);
});
it('returns newest source maps first', async () => {
@ -144,10 +247,14 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});
});
function getRandomString() {
return Math.random().toString(36).substring(7);
}
describe('delete source maps', () => {
it('can delete a source map', async () => {
before(async () => {
const sourcemap = await uploadSourcemap({
serviceName: 'my_service',
serviceName: `delete-test_${getRandomString()}`,
serviceVersion: '1.0.0',
bundleFilePath: 'bar',
sourcemap: {
@ -157,11 +264,63 @@ export default function ApiTest({ getService }: FtrProviderContext) {
},
});
// wait for the sourcemap to be indexed in .apm-source-map index
await waitForSourceMapCount(1);
// delete sourcemap
await deleteSourcemap(sourcemap.id);
// wait for the sourcemap to be deleted from .apm-source-map index
await waitForSourceMapCount(0);
});
it('can delete a fleet source map artifact', async () => {
const { artifacts, total } = await listSourcemaps();
expect(artifacts).to.be.empty();
expect(total).to.be(0);
});
it('can delete an apm source map', async () => {
// check that the sourcemap is deleted from .apm-source-map index
const res = await esClient.search({ index: '.apm-source-map' });
// @ts-expect-error
expect(res.hits.total.value).to.be(0);
});
});
describe('source map migration from fleet artifacts to `.apm-source-map`', () => {
const totalCount = 100;
before(async () => {
await Promise.all(
times(totalCount).map(async (i) => {
await uploadSourcemap({
serviceName: `migration-test`,
serviceVersion: `1.0.${i}`,
bundleFilePath: 'bar',
sourcemap: {
version: 123,
sources: [''],
mappings: '',
},
});
})
);
// wait for sourcemaps to be indexed in .apm-source-map index
await waitForSourceMapCount(totalCount);
});
it('it will migrate fleet artifacts to `.apm-source-map`', async () => {
await deleteAllApmSourceMaps();
// wait for source maps to be deleted before running migration
await waitForSourceMapCount(0);
await runSourceMapMigration();
expect(await waitForSourceMapCount(totalCount)).to.be(totalCount);
});
});
});
}

1
x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts
View file

@ -106,6 +106,7 @@ export default function ({ getService }: FtrProviderContext) {
'alerting_health_check',
'alerting_telemetry',
'alerts_invalidate_api_keys',
'apm-source-map-migration-task',
'apm-telemetry-task',
'cases-telemetry-task',
'cleanup_failed_action_executions',