github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Osquery] Refactor telemetry to use EBT (#138221)

Browse source
This commit is contained in:
Patryk Kopyciński 2022-08-09 17:50:20 +02:00 • committed by GitHub
parent 26a4783553
commit 3de9eda441
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 529 additions and 921 deletions
Download patch file Download diff file Expand all files Collapse all files

15
x-pack/plugins/osquery/server/common/types.ts
View file

@ -34,3 +34,18 @@ export interface PackSavedObjectAttributes {
}
export type PackSavedObject = SavedObject<PackSavedObjectAttributes>;
export interface SavedQuerySavedObjectAttributes {
id: string;
description: string | undefined;
query: string;
interval: number | string;
platform: string;
ecs_mapping?: Array<Record<string, unknown>>;
created_at: string;
created_by: string | undefined;
updated_at: string;
updated_by: string | undefined;
}
export type SavedQuerySavedObject = SavedObject<PackSavedObjectAttributes>;

19
x-pack/plugins/osquery/server/lib/saved_query/saved_object_mappings.ts
View file

@ -11,8 +11,27 @@ import {
savedQuerySavedObjectType,
packSavedObjectType,
packAssetSavedObjectType,
usageMetricSavedObjectType,
} from '../../../common/types';
export const usageMetricSavedObjectMappings: SavedObjectsType['mappings'] = {
properties: {
count: {
type: 'long',
},
errors: {
type: 'long',
},
},
};
export const usageMetricType: SavedObjectsType = {
name: usageMetricSavedObjectType,
hidden: false,
namespaceType: 'agnostic',
mappings: usageMetricSavedObjectMappings,
};
export const savedQuerySavedObjectMappings: SavedObjectsType['mappings'] = {
properties: {
description: {

2
x-pack/plugins/osquery/server/lib/telemetry/__mocks__/index.ts
View file

@ -19,9 +19,7 @@ export const createMockTelemetryEventsSender = (
setup: jest.fn(),
start: jest.fn(),
stop: jest.fn(),
getClusterID: jest.fn(),
fetchTelemetryUrl: jest.fn(),
queueTelemetryEvents: jest.fn(),
processEvents: jest.fn(),
isTelemetryOptedIn: jest.fn().mockReturnValue(enableTelemetry ?? jest.fn()),
sendIfDue: jest.fn(),

12
x-pack/plugins/osquery/server/lib/telemetry/constants.ts
View file

@ -5,11 +5,7 @@
* 2.0.
*/
export const TELEMETRY_MAX_BUFFER_SIZE = 100;
export const MAX_PACK_TELEMETRY_BATCH = 100;
export const TELEMETRY_CHANNEL_CONFIGS = 'osquery-configs';
export const TELEMETRY_CHANNEL_LIVE_QUERIES = 'osquery-live-queries-test';
export const TELEMETRY_CHANNEL_PACKS = 'osquery-packs';
export const TELEMETRY_CHANNEL_SAVED_QUERIES = 'osquery-saved-queries';
export const TELEMETRY_EBT_LIVE_QUERY_EVENT = 'osquery_live_query';
export const TELEMETRY_EBT_PACK_EVENT = 'osquery_pack';
export const TELEMETRY_EBT_SAVED_QUERY_EVENT = 'osquery_saved_query';
export const TELEMETRY_EBT_CONFIG_EVENT = 'osquery_config';

107
x-pack/plugins/osquery/server/lib/telemetry/helpers.ts
View file

@ -5,85 +5,64 @@
* 2.0.
*/
import moment from 'moment';
import { filter, find, isEmpty, pick, isString } from 'lodash';
import type { SavedObjectsFindResponse } from '@kbn/core/server';
import type { PackagePolicy } from '@kbn/fleet-plugin/common';
import type { ESClusterInfo, ESLicense, ListTemplate, TelemetryEvent } from './types';
import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import type {
PackSavedObjectAttributes,
SavedQuerySavedObjectAttributes,
} from '../../common/types';
/**
* Constructs the configs telemetry schema from a collection of config saved objects
*/
export const templateConfigs = (
configsData: PackagePolicy[],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
) =>
configsData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};
return {
...template,
...item,
};
});
export const templateConfigs = (configsData: PackagePolicy[]) =>
configsData.map((item) => ({
id: item.id,
version: item.package?.version,
enabled: item.enabled,
config: find(item.inputs, ['type', 'osquery'])?.config?.osquery.value,
}));
/**
* Constructs the packs telemetry schema from a collection of packs saved objects
*/
export const templatePacks = (
packsData: SavedObjectsFindResponse['saved_objects'],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
) =>
packsData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};
packsData: SavedObjectsFindResponse<PackSavedObjectAttributes>['saved_objects']
) => {
const nonEmptyQueryPacks = filter(packsData, (pack) => !isEmpty(pack.attributes.queries));
return {
...template,
id: item.id,
...(item.attributes as TelemetryEvent),
};
});
return nonEmptyQueryPacks.map((item) =>
pick(
{
name: item.attributes.name,
enabled: item.attributes.enabled,
queries: item.attributes.queries,
policies: (filter(item.references, ['type', AGENT_POLICY_SAVED_OBJECT_TYPE]), 'id')?.length,
prebuilt:
!!filter(item.references, ['type', 'osquery-pack-asset']) &&
item.attributes.version !== undefined,
},
['name', 'queries', 'policies', 'prebuilt', 'enabled']
)
);
};
/**
* Constructs the packs telemetry schema from a collection of packs saved objects
*/
export const templateSavedQueries = (
savedQueriesData: SavedObjectsFindResponse['saved_objects'],
clusterInfo: ESClusterInfo,
licenseInfo: ESLicense | undefined
savedQueriesData: SavedObjectsFindResponse<SavedQuerySavedObjectAttributes>['saved_objects'],
prebuiltSavedQueryIds: string[]
) =>
savedQueriesData.map((item) => {
const template: ListTemplate = {
'@timestamp': moment().toISOString(),
cluster_uuid: clusterInfo.cluster_uuid,
cluster_name: clusterInfo.cluster_name,
license_id: licenseInfo?.uid,
};
return {
...template,
id: item.id,
...(item.attributes as TelemetryEvent),
};
});
/**
* Convert counter label list to kebab case
*
* @param label_list the list of labels to create standardized UsageCounter from
* @returns a string label for usage in the UsageCounter
*/
export function createUsageCounterLabel(labelList: string[]): string {
return labelList.join('-');
}
savedQueriesData.map((item) => ({
id: item.attributes.id,
query: item.attributes.query,
platform: item.attributes.platform,
interval: isString(item.attributes.interval)
? parseInt(item.attributes.interval, 10)
: item.attributes.interval,
...(!isEmpty(item.attributes.ecs_mapping) ? { ecs_mapping: item.attributes.ecs_mapping } : {}),
prebuilt: prebuiltSavedQueryIds.includes(item.id),
}));

64
x-pack/plugins/osquery/server/lib/telemetry/receiver.ts
View file

@ -15,22 +15,28 @@ import type {
import type {
AgentClient,
AgentPolicyServiceInterface,
PackageService,
PackagePolicyServiceInterface,
} from '@kbn/fleet-plugin/server';
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import { OSQUERY_INTEGRATION_NAME } from '../../../common';
import { packSavedObjectType, savedQuerySavedObjectType } from '../../../common/types';
import type { ESLicense, ESClusterInfo } from './types';
import type { OsqueryAppContextService } from '../osquery_app_context_services';
import type {
PackSavedObjectAttributes,
SavedQuerySavedObjectAttributes,
} from '../../common/types';
import { getPrebuiltSavedQueryIds } from '../../routes/saved_query/utils';
export class TelemetryReceiver {
// @ts-expect-error used as part of this
private readonly logger: Logger;
private agentClient?: AgentClient;
private agentPolicyService?: AgentPolicyServiceInterface;
private packageService?: PackageService;
private packagePolicyService?: PackagePolicyServiceInterface;
private esClient?: ElasticsearchClient;
private soClient?: SavedObjectsClientContract;
private clusterInfo?: ESClusterInfo;
private readonly max_records = 100;
constructor(logger: Logger) {
@ -40,19 +46,15 @@ export class TelemetryReceiver {
public async start(core: CoreStart, osqueryContextService?: OsqueryAppContextService) {
this.agentClient = osqueryContextService?.getAgentService()?.asInternalUser;
this.agentPolicyService = osqueryContextService?.getAgentPolicyService();
this.packageService = osqueryContextService?.getPackageService();
this.packagePolicyService = osqueryContextService?.getPackagePolicyService();
this.esClient = core.elasticsearch.client.asInternalUser;
this.soClient =
core.savedObjects.createInternalRepository() as unknown as SavedObjectsClientContract;
this.clusterInfo = await this.fetchClusterInfo();
}
public getClusterInfo(): ESClusterInfo | undefined {
return this.clusterInfo;
}
public async fetchPacks() {
return this.soClient?.find({
return this.soClient?.find<PackSavedObjectAttributes>({
type: packSavedObjectType,
page: 1,
perPage: this.max_records,
@ -62,7 +64,7 @@ export class TelemetryReceiver {
}
public async fetchSavedQueries() {
return this.soClient?.find({
return this.soClient?.find<SavedQuerySavedObjectAttributes>({
type: savedQuerySavedObjectType,
page: 1,
perPage: this.max_records,
@ -83,6 +85,10 @@ export class TelemetryReceiver {
throw Error('elasticsearch client is unavailable: cannot retrieve fleet policy responses');
}
public async fetchPrebuiltSavedQueryIds() {
return getPrebuiltSavedQueryIds(this.packageService?.asInternalUser);
}
public async fetchFleetAgents() {
if (this.esClient === undefined || this.soClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve fleet policy responses');
@ -105,44 +111,4 @@ export class TelemetryReceiver {
return this.agentPolicyService?.get(this.soClient, id);
}
public async fetchClusterInfo(): Promise<ESClusterInfo> {
if (this.esClient === undefined || this.esClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve cluster infomation');
}
return this.esClient.info();
}
public async fetchLicenseInfo(): Promise<ESLicense | undefined> {
if (this.esClient === undefined || this.esClient === null) {
throw Error('elasticsearch client is unavailable: cannot retrieve license information');
}
try {
const ret = await this.esClient.transport.request<{ license: ESLicense }>({
method: 'GET',
path: '/_license',
querystring: {
local: true,
},
});
return ret.license;
} catch (err) {
this.logger.debug(`failed retrieving license: ${err}`);
return undefined;
}
}
public copyLicenseFields(lic: ESLicense) {
return {
uid: lic.uid,
status: lic.status,
type: lic.type,
...(lic.issued_to ? { issued_to: lic.issued_to } : {}),
...(lic.issuer ? { issuer: lic.issuer } : {}),
};
}
}

254
x-pack/plugins/osquery/server/lib/telemetry/sender.test.ts
View file

@ -1,254 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable dot-notation */
import { TelemetryEventsSender } from './sender';
import { loggingSystemMock } from '@kbn/core/server/mocks';
import { usageCountersServiceMock } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counters_service.mock';
import { URL } from 'url';
describe('TelemetryEventsSender', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
const usageCountersServiceSetup = usageCountersServiceMock.createSetupContract();
const telemetryUsageCounter = usageCountersServiceSetup.createUsageCounter(
'testTelemetryUsageCounter'
);
beforeEach(() => {
logger = loggingSystemMock.createLogger();
});
describe('processEvents', () => {
it('returns empty array when empty array is passed', () => {
const sender = new TelemetryEventsSender(logger);
const result = sender.processEvents([]);
expect(result).toStrictEqual([]);
});
it('applies the allowlist', () => {
const sender = new TelemetryEventsSender(logger);
const input = [
{
event: {
kind: 'alert',
},
dns: {
question: {
name: 'test-dns',
},
},
agent: {
name: 'test',
},
rule: {
id: 'X',
name: 'Y',
ruleset: 'Z',
version: '100',
},
file: {
extension: '.exe',
size: 3,
created: 0,
path: 'X',
Ext: {
code_signature: {
key1: 'X',
key2: 'Y',
},
malware_classification: {
key1: 'X',
},
malware_signature: {
key1: 'X',
},
header_bytes: 'data in here',
quarantine_result: true,
quarantine_message: 'this file is bad',
},
},
host: {
os: {
name: 'windows',
},
},
process: {
name: 'foo.exe',
working_directory: '/some/usr/dir',
entity_id: 'some_entity_id',
},
Responses: '{ "result": 0 }', // >= 7.15
Target: {
process: {
name: 'bar.exe',
thread: {
id: 1234,
},
},
},
},
];
const result = sender.processEvents(input);
expect(result).toStrictEqual([
{
event: {
kind: 'alert',
},
dns: {
question: {
name: 'test-dns',
},
},
agent: {
name: 'test',
},
rule: {
id: 'X',
name: 'Y',
ruleset: 'Z',
version: '100',
},
file: {
extension: '.exe',
size: 3,
created: 0,
path: 'X',
Ext: {
code_signature: {
key1: 'X',
key2: 'Y',
},
header_bytes: 'data in here',
malware_classification: {
key1: 'X',
},
malware_signature: {
key1: 'X',
},
quarantine_result: true,
quarantine_message: 'this file is bad',
},
},
host: {
os: {
name: 'windows',
},
},
process: {
name: 'foo.exe',
working_directory: '/some/usr/dir',
entity_id: 'some_entity_id',
},
Responses: '{ "result": 0 }',
Target: {
process: {
name: 'bar.exe',
thread: {
id: 1234,
},
},
},
},
]);
});
});
describe('queueTelemetryEvents', () => {
it('queues two events', () => {
const sender = new TelemetryEventsSender(logger);
sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
expect(sender['queue'].length).toBe(2);
});
it('queues more than maxQueueSize events', () => {
const sender = new TelemetryEventsSender(logger);
sender['maxQueueSize'] = 5;
sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]);
sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]);
sender.queueTelemetryEvents([{ 'event.kind': '7' }, { 'event.kind': '8' }]);
expect(sender['queue'].length).toBe(5);
});
it('empties the queue when sending', async () => {
const sender = new TelemetryEventsSender(logger);
sender['telemetryStart'] = {
getIsOptedIn: jest.fn(async () => true),
};
sender['telemetrySetup'] = {
getTelemetryUrl: jest.fn(async () => new URL('https://telemetry.elastic.co')),
};
sender['telemetryUsageCounter'] = telemetryUsageCounter;
sender['sendEvents'] = jest.fn(async () => {
sender['telemetryUsageCounter']?.incrementCounter({
counterName: 'test_counter',
counterType: 'invoked',
incrementBy: 1,
});
});
sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
expect(sender['queue'].length).toBe(2);
await sender['sendIfDue']();
expect(sender['queue'].length).toBe(0);
expect(sender['sendEvents']).toBeCalledTimes(1);
sender.queueTelemetryEvents([{ 'event.kind': '3' }, { 'event.kind': '4' }]);
sender.queueTelemetryEvents([{ 'event.kind': '5' }, { 'event.kind': '6' }]);
expect(sender['queue'].length).toBe(4);
await sender['sendIfDue']();
expect(sender['queue'].length).toBe(0);
expect(sender['sendEvents']).toBeCalledTimes(2);
expect(sender['telemetryUsageCounter'].incrementCounter).toBeCalledTimes(2);
});
it("shouldn't send when telemetry is disabled", async () => {
const sender = new TelemetryEventsSender(logger);
sender['sendEvents'] = jest.fn();
const telemetryStart = {
getIsOptedIn: jest.fn(async () => false),
};
sender['telemetryStart'] = telemetryStart;
sender.queueTelemetryEvents([{ 'event.kind': '1' }, { 'event.kind': '2' }]);
expect(sender['queue'].length).toBe(2);
await sender['sendIfDue']();
expect(sender['queue'].length).toBe(0);
expect(sender['sendEvents']).toBeCalledTimes(0);
});
});
});
describe('getV3UrlFromV2', () => {
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
beforeEach(() => {
logger = loggingSystemMock.createLogger();
});
it('should return prod url', () => {
const sender = new TelemetryEventsSender(logger);
expect(
sender.getV3UrlFromV2('https://telemetry.elastic.co/xpack/v2/send', 'alerts-endpoint')
).toBe('https://telemetry.elastic.co/v3/send/alerts-endpoint');
});
it('should return staging url', () => {
const sender = new TelemetryEventsSender(logger);
expect(
sender.getV3UrlFromV2('https://telemetry-staging.elastic.co/xpack/v2/send', 'alerts-endpoint')
).toBe('https://telemetry-staging.elastic.co/v3-dev/send/alerts-endpoint');
});
it('should support ports and auth', () => {
const sender = new TelemetryEventsSender(logger);
expect(
sender.getV3UrlFromV2('http://user:pass@myproxy.local:1337/xpack/v2/send', 'alerts-endpoint')
).toBe('http://user:pass@myproxy.local:1337/v3/send/alerts-endpoint');
});
});

407
x-pack/plugins/osquery/server/lib/telemetry/sender.ts
View file

@ -5,41 +5,30 @@
* 2.0.
*/
import axios from 'axios';
import { URL } from 'url';
import { transformDataToNdjson } from '@kbn/securitysolution-utils';
import type { Logger } from '@kbn/core/server';
import type { TelemetryPluginStart, TelemetryPluginSetup } from '@kbn/telemetry-plugin/server';
import type { UsageCounter } from '@kbn/usage-collection-plugin/server';
import type { AnalyticsServiceSetup, Logger } from '@kbn/core/server';
import type {
TaskManagerSetupContract,
TaskManagerStartContract,
} from '@kbn/task-manager-plugin/server';
import type { TelemetryReceiver } from './receiver';
import { createTelemetryTaskConfigs } from './tasks';
import { createUsageCounterLabel } from './helpers';
import type { TelemetryEvent } from './types';
import { TELEMETRY_MAX_BUFFER_SIZE } from './constants';
import {
TELEMETRY_EBT_LIVE_QUERY_EVENT,
TELEMETRY_EBT_SAVED_QUERY_EVENT,
TELEMETRY_EBT_CONFIG_EVENT,
TELEMETRY_EBT_PACK_EVENT,
} from './constants';
import type { OsqueryTelemetryTaskConfig } from './task';
import { OsqueryTelemetryTask } from './task';
const usageLabelPrefix: string[] = ['osquery_telemetry', 'sender'];
export class TelemetryEventsSender {
private readonly initialCheckDelayMs = 10 * 1000;
private readonly checkIntervalMs = 60 * 1000;
private readonly logger: Logger;
private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE;
private telemetryStart?: TelemetryPluginStart;
private telemetrySetup?: TelemetryPluginSetup;
private intervalId?: NodeJS.Timeout;
private isSending = false;
// @ts-expect-error used as part of this
private receiver: TelemetryReceiver | undefined;
private queue: TelemetryEvent[] = [];
private isOptedIn?: boolean = true; // Assume true until the first check
public analyticsReportEvent: AnalyticsServiceSetup['reportEvent'] | undefined;
private telemetryUsageCounter?: UsageCounter;
private telemetryTasks?: OsqueryTelemetryTask[];
constructor(logger: Logger) {
@ -48,47 +37,34 @@ export class TelemetryEventsSender {
public setup(
telemetryReceiver: TelemetryReceiver,
telemetrySetup?: TelemetryPluginSetup,
taskManager?: TaskManagerSetupContract,
telemetryUsageCounter?: UsageCounter
analytics?: AnalyticsServiceSetup
) {
this.telemetrySetup = telemetrySetup;
this.telemetryUsageCounter = telemetryUsageCounter;
if (analytics) {
this.analyticsReportEvent = analytics.reportEvent;
if (taskManager) {
this.telemetryTasks = createTelemetryTaskConfigs().map(
(config: OsqueryTelemetryTaskConfig) => {
const task = new OsqueryTelemetryTask(config, this.logger, this, telemetryReceiver);
task.register(taskManager);
this.registerEvents(analytics.registerEventType);
return task;
}
);
if (taskManager) {
this.telemetryTasks = createTelemetryTaskConfigs().map(
(config: OsqueryTelemetryTaskConfig) => {
const task = new OsqueryTelemetryTask(config, this.logger, this, telemetryReceiver);
task.register(taskManager);
return task;
}
);
}
}
}
public getClusterID(): string | undefined {
return this.receiver?.getClusterInfo()?.cluster_uuid;
}
public start(
telemetryStart?: TelemetryPluginStart,
taskManager?: TaskManagerStartContract,
receiver?: TelemetryReceiver
) {
this.telemetryStart = telemetryStart;
public start(taskManager?: TaskManagerStartContract, receiver?: TelemetryReceiver) {
this.receiver = receiver;
if (taskManager && this.telemetryTasks) {
this.logger.debug(`Starting osquery telemetry tasks`);
this.telemetryTasks.forEach((task) => task.start(taskManager));
}
this.logger.debug(`Starting local task`);
setTimeout(() => {
this.sendIfDue();
this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs);
}, this.initialCheckDelayMs);
}
public stop() {
@ -97,207 +73,146 @@ export class TelemetryEventsSender {
}
}
public queueTelemetryEvents(events: TelemetryEvent[]) {
const qlength = this.queue.length;
if (events.length === 0) {
return;
}
this.logger.debug(`Queue events`);
if (qlength >= this.maxQueueSize) {
// we're full already
return;
}
if (events.length > this.maxQueueSize - qlength) {
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['queue_stats'])),
counterType: 'docs_lost',
incrementBy: events.length,
});
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['queue_stats'])),
counterType: 'num_capacity_exceeded',
incrementBy: 1,
});
this.queue.push(...this.processEvents(events.slice(0, this.maxQueueSize - qlength)));
} else {
this.queue.push(...this.processEvents(events));
public reportEvent(
...args: Parameters<AnalyticsServiceSetup['reportEvent']>
): ReturnType<AnalyticsServiceSetup['reportEvent']> {
if (this.analyticsReportEvent) {
this.analyticsReportEvent(...args);
}
}
public async isTelemetryOptedIn() {
this.isOptedIn = await this.telemetryStart?.getIsOptedIn();
public registerEvents(registerEventType: AnalyticsServiceSetup['registerEventType']) {
registerEventType({
eventType: TELEMETRY_EBT_LIVE_QUERY_EVENT,
schema: {
action_id: { type: 'keyword', _meta: { description: '' } },
'@timestamp': { type: 'date', _meta: { description: '' } },
expiration: { type: 'date', _meta: { description: '' } },
agent_ids: { type: 'pass_through', _meta: { description: '', optional: true } },
agent_all: { type: 'boolean', _meta: { description: '', optional: true } },
agent_platforms: { type: 'pass_through', _meta: { description: '', optional: true } },
agent_policy_ids: { type: 'pass_through', _meta: { description: '', optional: true } },
agents: { type: 'long', _meta: { description: '' } },
metadata: { type: 'pass_through', _meta: { description: '', optional: true } },
queries: { type: 'pass_through', _meta: { description: '' } },
alert_ids: { type: 'pass_through', _meta: { description: '', optional: true } },
event_ids: { type: 'pass_through', _meta: { description: '', optional: true } },
case_ids: { type: 'pass_through', _meta: { description: '', optional: true } },
pack_id: { type: 'keyword', _meta: { description: '', optional: true } },
pack_name: { type: 'keyword', _meta: { description: '', optional: true } },
pack_prebuilt: { type: 'boolean', _meta: { description: '', optional: true } },
},
});
return this.isOptedIn === true;
}
private async sendIfDue() {
if (this.isSending) {
return;
}
if (this.queue.length === 0) {
return;
}
try {
this.isSending = true;
this.isOptedIn = await this.isTelemetryOptedIn();
if (!this.isOptedIn) {
this.logger.debug(`Telemetry is not opted-in.`);
this.queue = [];
this.isSending = false;
return;
}
const clusterInfo = this.receiver?.getClusterInfo();
const [telemetryUrl, licenseInfo] = await Promise.all([
this.fetchTelemetryUrl('alerts-endpoint'),
this.receiver?.fetchLicenseInfo(),
]);
this.logger.debug(`Telemetry URL: ${telemetryUrl}`);
this.logger.debug(
`cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}`
);
const toSend: TelemetryEvent[] = this.queue.slice().map((event) => ({
...event,
...(licenseInfo ? { license: this.receiver?.copyLicenseFields(licenseInfo) } : {}),
cluster_uuid: clusterInfo?.cluster_uuid,
cluster_name: clusterInfo?.cluster_name,
}));
this.queue = [];
await this.sendEvents(
toSend,
telemetryUrl,
'alerts-endpoint',
clusterInfo?.cluster_uuid,
clusterInfo?.cluster_name,
clusterInfo?.version?.number,
licenseInfo?.uid
);
} catch (err) {
this.queue = [];
}
this.isSending = false;
}
public processEvents(events: TelemetryEvent[]): TelemetryEvent[] {
return events;
}
/**
* This function sends events to the elastic telemetry channel. Caution is required
* because it does no allowlist filtering at send time. The function call site is
* responsible for ensuring sure no sensitive material is in telemetry events.
*
* @param channel the elastic telemetry channel
* @param toSend telemetry events
*/
public async sendOnDemand(channel: string, toSend: unknown[]) {
const clusterInfo = this.receiver?.getClusterInfo();
try {
const [telemetryUrl, licenseInfo] = await Promise.all([
this.fetchTelemetryUrl(channel),
this.receiver?.fetchLicenseInfo(),
]);
this.logger.debug(`Telemetry URL: ${telemetryUrl}`);
this.logger.debug(
`cluster_uuid: ${clusterInfo?.cluster_uuid} cluster_name: ${clusterInfo?.cluster_name}`
);
await this.sendEvents(
toSend,
telemetryUrl,
channel,
clusterInfo?.cluster_uuid,
clusterInfo?.cluster_name,
clusterInfo?.version?.number,
licenseInfo?.uid
);
// eslint-disable-next-line no-empty
} catch (err) {}
}
private async fetchTelemetryUrl(channel: string): Promise<string> {
const telemetryUrl = await this.telemetrySetup?.getTelemetryUrl();
if (!telemetryUrl) {
throw Error("Couldn't get telemetry URL");
}
return this.getV3UrlFromV2(telemetryUrl.toString(), channel);
}
// Forms URLs like:
// https://telemetry.elastic.co/v3/send/my-channel-name or
// https://telemetry-staging.elastic.co/v3-dev/send/my-channel-name
public getV3UrlFromV2(v2url: string, channel: string): string {
const url = new URL(v2url);
if (!url.hostname.includes('staging')) {
url.pathname = `/v3/send/${channel}`;
} else {
url.pathname = `/v3-dev/send/${channel}`;
}
return url.toString();
}
private async sendEvents(
events: unknown[],
telemetryUrl: string,
channel: string,
clusterUuid: string | undefined,
clusterName: string | undefined,
clusterVersionNumber: string | undefined,
licenseId: string | undefined
) {
const ndjson = transformDataToNdjson(events);
try {
this.logger.debug(`Sending ${events.length} telemetry events to ${channel}`);
const resp = await axios.post(telemetryUrl, ndjson, {
headers: {
'Content-Type': 'application/x-ndjson',
...(clusterUuid ? { 'X-Elastic-Cluster-ID': clusterUuid } : undefined),
...(clusterName ? { 'X-Elastic-Cluster-Name': clusterName } : undefined),
'X-Elastic-Stack-Version': clusterVersionNumber ? clusterVersionNumber : '8.0.0',
...(licenseId ? { 'X-Elastic-License-ID': licenseId } : {}),
registerEventType({
eventType: TELEMETRY_EBT_PACK_EVENT,
schema: {
name: {
type: 'keyword',
_meta: {
description: '',
},
},
timeout: 5000,
});
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
counterType: resp.status.toString(),
incrementBy: 1,
});
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
counterType: 'docs_sent',
incrementBy: events.length,
});
this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`);
} catch (err) {
this.logger.debug(`Error sending events: ${err}`);
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
counterType: 'docs_lost',
incrementBy: events.length,
});
this.telemetryUsageCounter?.incrementCounter({
counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
counterType: 'num_exceptions',
incrementBy: 1,
});
}
queries: {
type: 'pass_through',
_meta: {
description: 'Pack queries',
},
},
policies: {
type: 'short',
_meta: {
description: 'Number of agent policies assigned to the pack',
},
},
prebuilt: {
type: 'boolean',
_meta: {
description: 'Elastic prebuilt pack',
},
},
enabled: {
type: 'boolean',
_meta: {
description: 'Pack enabled',
},
},
},
});
registerEventType({
eventType: TELEMETRY_EBT_CONFIG_EVENT,
schema: {
id: {
type: 'keyword',
_meta: {
description: '',
},
},
version: {
type: 'keyword',
_meta: {
description: 'osquery_manger integration version',
},
},
enabled: {
type: 'boolean',
_meta: {
description: '',
},
},
config: {
type: 'pass_through',
_meta: {
description: 'Osquery package policy config',
},
},
},
});
registerEventType({
eventType: TELEMETRY_EBT_SAVED_QUERY_EVENT,
schema: {
id: {
type: 'keyword',
_meta: {
description: '',
},
},
query: {
type: 'text',
_meta: {
description: '',
},
},
platform: {
type: 'keyword',
_meta: {
description: '',
optional: true,
},
},
interval: {
type: 'short',
_meta: {
description: '',
optional: true,
},
},
prebuilt: {
type: 'boolean',
_meta: {
description: '',
},
},
ecs_mapping: {
type: 'pass_through',
_meta: {
description: '',
optional: true,
},
},
},
});
}
}

9
x-pack/plugins/osquery/server/lib/telemetry/task.test.ts
View file

@ -50,7 +50,7 @@ describe('test osquery telemetry task', () => {
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
});
test('telemetry task should run if opted in', async () => {
test('telemetry task should run', async () => {
const {
testLastTimestamp,
testResult,
@ -72,12 +72,6 @@ describe('test osquery telemetry task', () => {
);
});
test('telemetry task should not run if opted out', async () => {
const { mockTelemetryTaskConfig } = await testTelemetryTaskRun(false);
expect(mockTelemetryTaskConfig.runTask).not.toHaveBeenCalled();
});
async function testTelemetryTaskRun(optedIn: boolean) {
const now = new Date();
const testType = 'security:test-task';
@ -104,7 +98,6 @@ describe('test osquery telemetry task', () => {
const testResult = (await taskRunner.run()) as SuccessfulRunResult;
expect(mockTelemetryTaskConfig.getLastExecutionTime).toHaveBeenCalled();
expect(mockTelemetryEventsSender.isTelemetryOptedIn).toHaveBeenCalled();
expect(testResult).not.toBeNull();
expect(testResult).toHaveProperty('state.lastExecutionTimestamp');

9
x-pack/plugins/osquery/server/lib/telemetry/task.ts
View file

@ -31,7 +31,7 @@ export type OsqueryTelemetryTaskRunner = (
receiver: TelemetryReceiver,
sender: TelemetryEventsSender,
taskExecutionPeriod: TaskExecutionPeriod
) => Promise<number>;
) => Promise<void>;
export interface TaskExecutionPeriod {
last?: string;
@ -135,13 +135,6 @@ export class OsqueryTelemetryTask {
return 0;
}
const isOptedIn = await this.sender.isTelemetryOptedIn();
if (!isOptedIn) {
this.logger.debug(`[task ${taskId}]: telemetry is not opted-in`);
return 0;
}
this.logger.debug(`[task ${taskId}]: running task`);
return this.config.runTask(taskId, this.logger, this.receiver, this.sender, executionPeriod);

31
x-pack/plugins/osquery/server/lib/telemetry/tasks/configs.ts
View file

@ -6,52 +6,37 @@
*/
import type { Logger } from '@kbn/core/server';
import { TELEMETRY_CHANNEL_CONFIGS } from '../constants';
import { TELEMETRY_EBT_CONFIG_EVENT } from '../constants';
import { templateConfigs } from '../helpers';
import type { TelemetryEventsSender } from '../sender';
import type { TelemetryReceiver } from '../receiver';
import type { ESClusterInfo, ESLicense } from '../types';
export function createTelemetryConfigsTaskConfig() {
return {
type: 'osquery:telemetry-configs',
title: 'Osquery Configs Telemetry',
interval: '5m',
interval: '24h',
timeout: '10m',
version: '1.0.0',
version: '1.1.0',
runTask: async (
taskId: string,
logger: Logger,
receiver: TelemetryReceiver,
sender: TelemetryEventsSender
) => {
const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
receiver.fetchClusterInfo(),
receiver.fetchLicenseInfo(),
]);
const clusterInfo =
clusterInfoPromise.status === 'fulfilled'
? clusterInfoPromise.value
: ({} as ESClusterInfo);
const licenseInfo =
licenseInfoPromise.status === 'fulfilled'
? licenseInfoPromise.value
: ({} as ESLicense | undefined);
const configsResponse = await receiver.fetchConfigs();
if (!configsResponse?.total) {
logger.debug('no configs found');
return 0;
return;
}
const configsJson = templateConfigs(configsResponse?.items, clusterInfo, licenseInfo);
const configsJson = templateConfigs(configsResponse?.items);
sender.sendOnDemand(TELEMETRY_CHANNEL_CONFIGS, configsJson);
return configsResponse.total;
configsJson.forEach((config) => {
sender.reportEvent(TELEMETRY_EBT_CONFIG_EVENT, config);
});
},
};
}

29
x-pack/plugins/osquery/server/lib/telemetry/tasks/packs.ts
View file

@ -6,11 +6,10 @@
*/
import type { Logger } from '@kbn/core/server';
import { TELEMETRY_CHANNEL_PACKS } from '../constants';
import { TELEMETRY_EBT_PACK_EVENT } from '../constants';
import { templatePacks } from '../helpers';
import type { TelemetryEventsSender } from '../sender';
import type { TelemetryReceiver } from '../receiver';
import type { ESClusterInfo, ESLicense } from '../types';
export function createTelemetryPacksTaskConfig() {
return {
@ -18,40 +17,26 @@ export function createTelemetryPacksTaskConfig() {
title: 'Osquery Packs Telemetry',
interval: '24h',
timeout: '10m',
version: '1.0.0',
version: '1.1.0',
runTask: async (
taskId: string,
logger: Logger,
receiver: TelemetryReceiver,
sender: TelemetryEventsSender
) => {
const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
receiver.fetchClusterInfo(),
receiver.fetchLicenseInfo(),
]);
const clusterInfo =
clusterInfoPromise.status === 'fulfilled'
? clusterInfoPromise.value
: ({} as ESClusterInfo);
const licenseInfo =
licenseInfoPromise.status === 'fulfilled'
? licenseInfoPromise.value
: ({} as ESLicense | undefined);
const packsResponse = await receiver.fetchPacks();
if (!packsResponse?.total) {
logger.debug('no packs found');
return 0;
return;
}
const packsJson = templatePacks(packsResponse?.saved_objects, clusterInfo, licenseInfo);
const packsJson = templatePacks(packsResponse?.saved_objects);
sender.sendOnDemand(TELEMETRY_CHANNEL_PACKS, packsJson);
return packsResponse.total;
packsJson.forEach((pack) => {
sender.reportEvent(TELEMETRY_EBT_PACK_EVENT, pack);
});
},
};
}

31
x-pack/plugins/osquery/server/lib/telemetry/tasks/saved_queries.ts
View file

@ -6,11 +6,10 @@
*/
import type { Logger } from '@kbn/core/server';
import { TELEMETRY_CHANNEL_SAVED_QUERIES } from '../constants';
import { TELEMETRY_EBT_SAVED_QUERY_EVENT } from '../constants';
import { templateSavedQueries } from '../helpers';
import type { TelemetryEventsSender } from '../sender';
import type { TelemetryReceiver } from '../receiver';
import type { ESClusterInfo, ESLicense } from '../types';
export function createTelemetrySavedQueriesTaskConfig() {
return {
@ -18,44 +17,30 @@ export function createTelemetrySavedQueriesTaskConfig() {
title: 'Osquery Saved Queries Telemetry',
interval: '24h',
timeout: '10m',
version: '1.0.0',
version: '1.1.0',
runTask: async (
taskId: string,
logger: Logger,
receiver: TelemetryReceiver,
sender: TelemetryEventsSender
) => {
const [clusterInfoPromise, licenseInfoPromise] = await Promise.allSettled([
receiver.fetchClusterInfo(),
receiver.fetchLicenseInfo(),
]);
const clusterInfo =
clusterInfoPromise.status === 'fulfilled'
? clusterInfoPromise.value
: ({} as ESClusterInfo);
const licenseInfo =
licenseInfoPromise.status === 'fulfilled'
? licenseInfoPromise.value
: ({} as ESLicense | undefined);
const savedQueriesResponse = await receiver.fetchSavedQueries();
if (!savedQueriesResponse?.total) {
logger.debug('no saved queries found');
return 0;
return;
}
const prebuiltSavedQueryIds = await receiver.fetchPrebuiltSavedQueryIds();
const savedQueriesJson = templateSavedQueries(
savedQueriesResponse?.saved_objects,
clusterInfo,
licenseInfo
prebuiltSavedQueryIds
);
sender.sendOnDemand(TELEMETRY_CHANNEL_SAVED_QUERIES, savedQueriesJson);
return savedQueriesResponse.total;
savedQueriesJson.forEach((savedQuery) => {
sender.reportEvent(TELEMETRY_EBT_SAVED_QUERY_EVENT, savedQuery);
});
},
};
}

185
x-pack/plugins/osquery/server/plugin.ts
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import type {
PluginInitializerContext,
CoreSetup,
@ -13,8 +12,7 @@ import type {
Plugin,
Logger,
} from '@kbn/core/server';
import { DEFAULT_APP_CATEGORIES, SavedObjectsClient } from '@kbn/core/server';
import type { UsageCounter } from '@kbn/usage-collection-plugin/server';
import { SavedObjectsClient } from '@kbn/core/server';
import type { PackagePolicy } from '@kbn/fleet-plugin/common';
import type { DataRequestHandlerContext } from '@kbn/data-plugin/server';
import type { DataViewsService } from '@kbn/data-views-plugin/common';
@ -28,12 +26,7 @@ import { initUsageCollectors } from './usage';
import type { OsqueryAppContext } from './lib/osquery_app_context_services';
import { OsqueryAppContextService } from './lib/osquery_app_context_services';
import type { ConfigType } from './config';
import {
packSavedObjectType,
packAssetSavedObjectType,
savedQuerySavedObjectType,
} from '../common/types';
import { OSQUERY_INTEGRATION_NAME, PLUGIN_ID } from '../common';
import { OSQUERY_INTEGRATION_NAME } from '../common';
import { getPackagePolicyDeleteCallback } from './lib/fleet_integration';
import { TelemetryEventsSender } from './lib/telemetry/sender';
import { TelemetryReceiver } from './lib/telemetry/receiver';
@ -41,162 +34,7 @@ import { initializeTransformsIndices } from './create_indices/create_transforms_
import { initializeTransforms } from './create_transforms/create_transforms';
import { createDataViews } from './create_data_views';
const registerFeatures = (features: SetupPlugins['features']) => {
features.registerKibanaFeature({
id: PLUGIN_ID,
name: i18n.translate('xpack.osquery.features.osqueryFeatureName', {
defaultMessage: 'Osquery',
}),
category: DEFAULT_APP_CATEGORIES.management,
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
order: 2300,
privileges: {
all: {
api: [`${PLUGIN_ID}-read`, `${PLUGIN_ID}-write`],
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
savedObject: {
all: [],
read: [],
},
ui: ['write'],
},
read: {
api: [`${PLUGIN_ID}-read`],
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
savedObject: {
all: [],
read: [],
},
ui: ['read'],
},
},
subFeatures: [
{
name: i18n.translate('xpack.osquery.features.liveQueriesSubFeatureName', {
defaultMessage: 'Live queries',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writeLiveQueries`, `${PLUGIN_ID}-readLiveQueries`],
id: 'live_queries_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [],
read: [],
},
ui: ['writeLiveQueries', 'readLiveQueries'],
},
{
api: [`${PLUGIN_ID}-readLiveQueries`],
id: 'live_queries_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [],
},
ui: ['readLiveQueries'],
},
],
},
{
groupType: 'independent',
privileges: [
{
api: [`${PLUGIN_ID}-runSavedQueries`],
id: 'run_saved_queries',
name: i18n.translate('xpack.osquery.features.runSavedQueriesPrivilegeName', {
defaultMessage: 'Run Saved queries',
}),
includeIn: 'all',
savedObject: {
all: [],
read: [],
},
ui: ['runSavedQueries'],
},
],
},
],
},
{
name: i18n.translate('xpack.osquery.features.savedQueriesSubFeatureName', {
defaultMessage: 'Saved queries',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writeSavedQueries`, `${PLUGIN_ID}-readSavedQueries`],
id: 'saved_queries_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [savedQuerySavedObjectType],
read: [],
},
ui: ['writeSavedQueries', 'readSavedQueries'],
},
{
api: [`${PLUGIN_ID}-readSavedQueries`],
id: 'saved_queries_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [savedQuerySavedObjectType],
},
ui: ['readSavedQueries'],
},
],
},
],
},
{
name: i18n.translate('xpack.osquery.features.packsSubFeatureName', {
defaultMessage: 'Packs',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writePacks`, `${PLUGIN_ID}-readPacks`],
id: 'packs_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [packSavedObjectType, packAssetSavedObjectType],
read: [],
},
ui: ['writePacks', 'readPacks'],
},
{
api: [`${PLUGIN_ID}-readPacks`],
id: 'packs_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [packSavedObjectType],
},
ui: ['readPacks'],
},
],
},
],
},
],
});
};
import { registerFeatures } from './utils/register_features';
export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginStart> {
private readonly logger: Logger;
@ -205,8 +43,6 @@ export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginSt
private readonly telemetryReceiver: TelemetryReceiver;
private readonly telemetryEventsSender: TelemetryEventsSender;
private telemetryUsageCounter?: UsageCounter;
constructor(private readonly initializerContext: PluginInitializerContext) {
this.context = initializerContext;
this.logger = initializerContext.logger.get();
@ -238,8 +74,6 @@ export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginSt
usageCollection: plugins.usageCollection,
});
this.telemetryUsageCounter = plugins.usageCollection?.createUsageCounter(PLUGIN_ID);
core.getStartServices().then(([{ elasticsearch }, depsStart]) => {
const osquerySearchStrategy = osquerySearchStrategyProvider(
depsStart.data,
@ -250,12 +84,7 @@ export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginSt
defineRoutes(router, osqueryContext);
});
this.telemetryEventsSender.setup(
this.telemetryReceiver,
plugins.telemetry,
plugins.taskManager,
this.telemetryUsageCounter
);
this.telemetryEventsSender.setup(this.telemetryReceiver, plugins.taskManager, core.analytics);
return {};
}
@ -275,11 +104,7 @@ export class OsqueryPlugin implements Plugin<OsqueryPluginSetup, OsqueryPluginSt
this.telemetryReceiver.start(core, this.osqueryAppContextService);
this.telemetryEventsSender.start(
plugins.telemetry,
plugins.taskManager,
this.telemetryReceiver
);
this.telemetryEventsSender.start(plugins.taskManager, this.telemetryReceiver);
plugins.fleet?.fleetSetupCompleted().then(async () => {
const packageInfo = await plugins.fleet?.packageService.asInternalUser.getInstallation(

2
x-pack/plugins/osquery/server/routes/fleet_wrapper/get_agent_policies.ts
View file

@ -14,7 +14,7 @@ import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import type { IRouter } from '@kbn/core/server';
import { OSQUERY_INTEGRATION_NAME, PLUGIN_ID } from '../../../common';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
export const getAgentPoliciesRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
router.get(

2
x-pack/plugins/osquery/server/routes/fleet_wrapper/get_agent_policy.ts
View file

@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema';
import type { IRouter } from '@kbn/core/server';
import { PLUGIN_ID } from '../../../common';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
export const getAgentPolicyRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
router.get(

2
x-pack/plugins/osquery/server/routes/fleet_wrapper/get_package_policies.ts
View file

@ -10,7 +10,7 @@ import type { IRouter } from '@kbn/core/server';
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
import { PLUGIN_ID, OSQUERY_INTEGRATION_NAME } from '../../../common';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
export const getPackagePoliciesRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
router.get(

22
x-pack/plugins/osquery/server/routes/live_query/create_live_query_route.ts
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { some, flatten, map, pick, pickBy, isEmpty } from 'lodash';
import { some, flatten, map, pick, pickBy, isEmpty, omit } from 'lodash';
import uuid from 'uuid';
import moment from 'moment-timezone';
@ -18,13 +18,13 @@ import { buildRouteValidation } from '../../utils/build_validation/route_validat
import type { CreateLiveQueryRequestBodySchema } from '../../../common/schemas/routes/live_query';
import { createLiveQueryRequestBodySchema } from '../../../common/schemas/routes/live_query';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { packSavedObjectType, savedQuerySavedObjectType } from '../../../common/types';
import { ACTIONS_INDEX } from '../../../common/constants';
import { convertSOQueriesToPack } from '../pack/utils';
import type { PackSavedObjectAttributes } from '../../common/types';
import { TELEMETRY_CHANNEL_LIVE_QUERIES } from '../../lib/telemetry/constants';
import { TELEMETRY_EBT_LIVE_QUERY_EVENT } from '../../lib/telemetry/constants';
import { isSavedQueryPrebuilt } from '../saved_query/utils';
import { getInternalSavedObjectsClient } from '../utils';
export const createLiveQueryRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
router.post(
@ -141,7 +141,10 @@ export const createLiveQueryRoute = (router: IRouter, osqueryContext: OsqueryApp
query: request.body.query,
saved_query_id: savedQueryId,
saved_query_prebuilt: savedQueryId
? await isSavedQueryPrebuilt(osqueryContext, savedQueryId)
? await isSavedQueryPrebuilt(
osqueryContext.service.getPackageService()?.asInternalUser,
savedQueryId
)
: undefined,
ecs_mapping: request.body.ecs_mapping,
agents: selectedAgents,
@ -180,13 +183,10 @@ export const createLiveQueryRoute = (router: IRouter, osqueryContext: OsqueryApp
});
}
const telemetryOptIn = await osqueryContext.telemetryEventsSender.isTelemetryOptedIn();
if (telemetryOptIn) {
osqueryContext.telemetryEventsSender.sendOnDemand(TELEMETRY_CHANNEL_LIVE_QUERIES, [
osqueryAction,
]);
}
osqueryContext.telemetryEventsSender.reportEvent(TELEMETRY_EBT_LIVE_QUERY_EVENT, {
...omit(osqueryAction, ['type', 'input_type', 'user_id']),
agents: osqueryAction.agents.length,
});
return response.ok({
body: { data: osqueryAction },

2
x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts
View file

@ -20,7 +20,7 @@ import { OSQUERY_INTEGRATION_NAME } from '../../../common';
import { PLUGIN_ID } from '../../../common';
import { packSavedObjectType } from '../../../common/types';
import { convertPackQueriesToSO, convertSOQueriesToPack } from './utils';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
import type { PackSavedObjectAttributes } from '../../common/types';
export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {

2
x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts
View file

@ -21,7 +21,7 @@ import { packSavedObjectType } from '../../../common/types';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
import { PLUGIN_ID } from '../../../common';
import { convertSOQueriesToPack, convertPackQueriesToSO } from './utils';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
import type { PackSavedObjectAttributes } from '../../common/types';
export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {

5
x-pack/plugins/osquery/server/routes/saved_query/delete_saved_query_route.ts
View file

@ -27,7 +27,10 @@ export const deleteSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp
const coreContext = await context.core;
const savedObjectsClient = coreContext.savedObjects.client;
const isPrebuilt = await isSavedQueryPrebuilt(osqueryContext, request.params.id);
const isPrebuilt = await isSavedQueryPrebuilt(
osqueryContext.service.getPackageService()?.asInternalUser,
request.params.id
);
if (isPrebuilt) {
return response.conflict({ body: `Elastic prebuilt Saved query cannot be deleted.` });
}

4
x-pack/plugins/osquery/server/routes/saved_query/find_saved_query_route.ts
View file

@ -44,7 +44,9 @@ export const findSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC
sortOrder: request.query.sortOrder ?? 'desc',
});
const prebuiltSavedQueriesMap = await getInstalledSavedQueriesMap(osqueryContext);
const prebuiltSavedQueriesMap = await getInstalledSavedQueriesMap(
osqueryContext.service.getPackageService()?.asInternalUser
);
const savedObjects = savedQueries.saved_objects.map((savedObject) => {
// eslint-disable-next-line @typescript-eslint/naming-convention
const ecs_mapping = savedObject.attributes.ecs_mapping;

5
x-pack/plugins/osquery/server/routes/saved_query/read_saved_query_route.ts
View file

@ -40,7 +40,10 @@ export const readSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAppC
);
}
savedQuery.attributes.prebuilt = await isSavedQueryPrebuilt(osqueryContext, savedQuery.id);
savedQuery.attributes.prebuilt = await isSavedQueryPrebuilt(
osqueryContext.service.getPackageService()?.asInternalUser,
savedQuery.id
);
return response.ok({
body: { data: savedQuery },

5
x-pack/plugins/osquery/server/routes/saved_query/update_saved_query_route.ts
View file

@ -64,7 +64,10 @@ export const updateSavedQueryRoute = (router: IRouter, osqueryContext: OsqueryAp
ecs_mapping,
} = request.body;
const isPrebuilt = await isSavedQueryPrebuilt(osqueryContext, request.params.id);
const isPrebuilt = await isSavedQueryPrebuilt(
osqueryContext.service.getPackageService()?.asInternalUser,
request.params.id
);
if (isPrebuilt) {
return response.conflict({ body: `Elastic prebuilt Saved query cannot be updated.` });

32
x-pack/plugins/osquery/server/routes/saved_query/utils.ts
View file

@ -5,20 +5,15 @@
* 2.0.
*/
import { find, reduce } from 'lodash';
import { find, filter, map, reduce } from 'lodash';
import type { KibanaAssetReference } from '@kbn/fleet-plugin/common';
import type { PackageClient } from '@kbn/fleet-plugin/server';
import { OSQUERY_INTEGRATION_NAME } from '../../../common';
import { savedQuerySavedObjectType } from '../../../common/types';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
const getInstallation = async (osqueryContext: OsqueryAppContext) =>
await osqueryContext.service
.getPackageService()
?.asInternalUser?.getInstallation(OSQUERY_INTEGRATION_NAME);
export const getInstalledSavedQueriesMap = async (osqueryContext: OsqueryAppContext) => {
const installation = await getInstallation(osqueryContext);
export const getInstalledSavedQueriesMap = async (packageService: PackageClient | undefined) => {
const installation = await packageService?.getInstallation(OSQUERY_INTEGRATION_NAME);
if (installation) {
return reduce<KibanaAssetReference, Record<string, KibanaAssetReference>>(
@ -37,11 +32,26 @@ export const getInstalledSavedQueriesMap = async (osqueryContext: OsqueryAppCont
return {};
};
export const getPrebuiltSavedQueryIds = async (packageService: PackageClient | undefined) => {
const installation = await packageService?.getInstallation(OSQUERY_INTEGRATION_NAME);
if (installation) {
const installationSavedQueries = filter(
installation.installed_kibana,
(item) => item.type === savedQuerySavedObjectType
);
return map(installationSavedQueries, 'id');
}
return [];
};
export const isSavedQueryPrebuilt = async (
osqueryContext: OsqueryAppContext,
packageService: PackageClient | undefined,
savedQueryId: string
) => {
const installation = await getInstallation(osqueryContext);
const installation = await packageService?.getInstallation(OSQUERY_INTEGRATION_NAME);
if (installation) {
const installationSavedQueries = find(

2
x-pack/plugins/osquery/server/routes/status/create_status_route.ts
View file

@ -18,7 +18,7 @@ import { packSavedObjectType } from '../../../common/types';
import { PLUGIN_ID, OSQUERY_INTEGRATION_NAME } from '../../../common';
import type { OsqueryAppContext } from '../../lib/osquery_app_context_services';
import { convertPackQueriesToSO } from '../pack/utils';
import { getInternalSavedObjectsClient } from '../../usage/collector';
import { getInternalSavedObjectsClient } from '../utils';
export const createStatusRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
router.get(

10
x-pack/plugins/osquery/server/routes/utils.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import type { CoreSetup } from '@kbn/core/server';
import { SavedObjectsClient } from '@kbn/core/server';
import { reduce } from 'lodash';
export const convertECSMappingToArray = (ecsMapping: Record<string, object> | undefined) =>
@ -27,3 +29,11 @@ export const convertECSMappingToObject = (
},
{} as Record<string, { field?: string; value?: string }>
);
export const getInternalSavedObjectsClient = async (
getStartServices: CoreSetup['getStartServices']
) => {
const [coreStart] = await getStartServices();
return new SavedObjectsClient(coreStart.savedObjects.createInternalRepository());
};

8
x-pack/plugins/osquery/server/saved_objects.ts
View file

@ -7,8 +7,12 @@
import type { CoreSetup } from '@kbn/core/server';
import { savedQueryType, packType, packAssetType } from './lib/saved_query/saved_object_mappings';
import { usageMetricType } from './routes/usage/saved_object_mappings';
import {
savedQueryType,
packType,
packAssetType,
usageMetricType,
} from './lib/saved_query/saved_object_mappings';
export const initSavedObjects = (savedObjects: CoreSetup['savedObjects']) => {
savedObjects.registerType(usageMetricType);

173
x-pack/plugins/osquery/server/utils/register_features.ts Normal file
View file

@ -0,0 +1,173 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
import {
packSavedObjectType,
packAssetSavedObjectType,
savedQuerySavedObjectType,
} from '../../common/types';
import { PLUGIN_ID } from '../../common';
import type { SetupPlugins } from '../types';
export const registerFeatures = (features: SetupPlugins['features']) => {
features.registerKibanaFeature({
id: PLUGIN_ID,
name: i18n.translate('xpack.osquery.features.osqueryFeatureName', {
defaultMessage: 'Osquery',
}),
category: DEFAULT_APP_CATEGORIES.management,
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
order: 2300,
privileges: {
all: {
api: [`${PLUGIN_ID}-read`, `${PLUGIN_ID}-write`],
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
savedObject: {
all: [],
read: [],
},
ui: ['write'],
},
read: {
api: [`${PLUGIN_ID}-read`],
app: [PLUGIN_ID, 'kibana'],
catalogue: [PLUGIN_ID],
savedObject: {
all: [],
read: [],
},
ui: ['read'],
},
},
subFeatures: [
{
name: i18n.translate('xpack.osquery.features.liveQueriesSubFeatureName', {
defaultMessage: 'Live queries',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writeLiveQueries`, `${PLUGIN_ID}-readLiveQueries`],
id: 'live_queries_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [],
read: [],
},
ui: ['writeLiveQueries', 'readLiveQueries'],
},
{
api: [`${PLUGIN_ID}-readLiveQueries`],
id: 'live_queries_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [],
},
ui: ['readLiveQueries'],
},
],
},
{
groupType: 'independent',
privileges: [
{
api: [`${PLUGIN_ID}-runSavedQueries`],
id: 'run_saved_queries',
name: i18n.translate('xpack.osquery.features.runSavedQueriesPrivilegeName', {
defaultMessage: 'Run Saved queries',
}),
includeIn: 'all',
savedObject: {
all: [],
read: [],
},
ui: ['runSavedQueries'],
},
],
},
],
},
{
name: i18n.translate('xpack.osquery.features.savedQueriesSubFeatureName', {
defaultMessage: 'Saved queries',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writeSavedQueries`, `${PLUGIN_ID}-readSavedQueries`],
id: 'saved_queries_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [savedQuerySavedObjectType],
read: [],
},
ui: ['writeSavedQueries', 'readSavedQueries'],
},
{
api: [`${PLUGIN_ID}-readSavedQueries`],
id: 'saved_queries_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [savedQuerySavedObjectType],
},
ui: ['readSavedQueries'],
},
],
},
],
},
{
name: i18n.translate('xpack.osquery.features.packsSubFeatureName', {
defaultMessage: 'Packs',
}),
privilegeGroups: [
{
groupType: 'mutually_exclusive',
privileges: [
{
api: [`${PLUGIN_ID}-writePacks`, `${PLUGIN_ID}-readPacks`],
id: 'packs_all',
includeIn: 'all',
name: 'All',
savedObject: {
all: [packSavedObjectType, packAssetSavedObjectType],
read: [],
},
ui: ['writePacks', 'readPacks'],
},
{
api: [`${PLUGIN_ID}-readPacks`],
id: 'packs_read',
includeIn: 'read',
name: 'Read',
savedObject: {
all: [],
read: [packSavedObjectType],
},
ui: ['readPacks'],
},
],
},
],
},
],
});
};