[Security Solution] Quickstart script tooling for Detections and Response (#190634)

## Summary

Creates CLI script tooling for building data, rules, exceptions, and
lists in any (local, cloud, serverless) environment for manual testing.
The initial commits here add generated clients for accessing security
solution, exceptions, and lists APIs and a placeholder script where
those clients are set up for use. See README for more details.

Much of the code in this PR is auto-generated clients. The hand written
code is intended to be primarily in `quickstart/modules/`, where we can
add wrapper code to simplify the process for common test environment
setup. For example, `createValueListException` takes an array of items
and some metadata and automatically creates a new value list and an
exception that references that value list. `/modules/data/` contains
functions to generate documents of arbitrary size, and we can add more
functions to create various other types of documents.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main 2024-09-06 06:41:57 -07:00 committed by GitHub
parent 335b153a92
commit 3cc7029197
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 3855 additions and 13 deletions

View file

@ -19,7 +19,7 @@ import { getGeneratedFilePath } from './lib/get_generated_file_path';
import { removeGenArtifacts } from './lib/remove_gen_artifacts';
import { lint } from './openapi_linter';
import { getGenerationContext } from './parser/get_generation_context';
import type { OpenApiDocument } from './parser/openapi_types';
import type { OpenApiDocument, ParsedSource } from './parser/openapi_types';
import { initTemplateService, TemplateName } from './template_service/template_service';
export interface GeneratorConfig {
@ -54,11 +54,12 @@ export const generate = async (config: GeneratorConfig) => {
const schemaPaths = await globby([sourceFilesGlob]);
console.log(`🕵️‍♀️ Found ${schemaPaths.length} schemas, parsing`);
let parsedSources = await Promise.all(
let parsedSources: ParsedSource[] = await Promise.all(
schemaPaths.map(async (sourcePath) => {
const parsedSchema = (await SwaggerParser.parse(sourcePath)) as OpenApiDocument;
return {
sourcePath,
generatedPath: getGeneratedFilePath(sourcePath),
generationContext: getGenerationContext(parsedSchema),
};
})
@ -68,6 +69,10 @@ export const generate = async (config: GeneratorConfig) => {
({ generationContext }) =>
generationContext.operations.length > 0 || generationContext.components !== undefined
);
parsedSources.sort((a, b) => a.sourcePath.localeCompare(b.sourcePath));
parsedSources.forEach((source) =>
source.generationContext.operations.sort((a, b) => a.operationId.localeCompare(b.operationId))
);
console.log(`🧹 Cleaning up any previously generated artifacts`);
if (bundle) {
@ -92,26 +97,24 @@ export const generate = async (config: GeneratorConfig) => {
// Sort the operations by operationId so the output is deterministic
.sort((a, b) => a.operationId.localeCompare(b.operationId));
const result = TemplateService.compileTemplate(templateName, {
const result = TemplateService.compileBundleTemplate(templateName, {
operations,
components: {},
sources: parsedSources,
info: {
title,
version: 'Bundle (no version)',
},
imports: {},
circularRefs: new Set<string>(),
});
await fs.writeFile(bundle.outFile, result);
console.log(`📖 Wrote bundled artifact to ${chalk.bold(bundle.outFile)}`);
} else {
await Promise.all(
parsedSources.map(async ({ sourcePath, generationContext }) => {
parsedSources.map(async ({ generatedPath, generationContext }) => {
const result = TemplateService.compileTemplate(templateName, generationContext);
// Write the generation result to disk
await fs.writeFile(getGeneratedFilePath(sourcePath), result);
await fs.writeFile(generatedPath, result);
})
);
}

View file

@ -11,7 +11,7 @@ import { getApiOperationsList } from './lib/get_api_operations_list';
import { getComponents } from './lib/get_components';
import { getImportsMap, ImportsMap } from './lib/get_imports_map';
import { normalizeSchema } from './lib/normalize_schema';
import { NormalizedOperation, OpenApiDocument } from './openapi_types';
import { NormalizedOperation, OpenApiDocument, ParsedSource } from './openapi_types';
import { getInfo } from './lib/get_info';
import { getCircularRefs } from './lib/get_circular_refs';
@ -23,6 +23,12 @@ export interface GenerationContext {
circularRefs: Set<string>;
}
export interface BundleGenerationContext {
operations: NormalizedOperation[];
sources: ParsedSource[];
info: OpenAPIV3.InfoObject;
}
export function getGenerationContext(document: OpenApiDocument): GenerationContext {
const normalizedDocument = normalizeSchema(document);

View file

@ -88,6 +88,8 @@ export function getApiOperationsList(parsedSchema: OpenApiDocument): NormalizedO
const requestBody = operation.requestBody?.content?.['application/json']?.schema as
| NormalizedSchemaItem
| undefined;
const requestAttachment = operation.requestBody?.content?.['multipart/form-data']
?.schema as NormalizedSchemaItem | undefined;
const normalizedOperation: NormalizedOperation = {
path,
method,
@ -98,6 +100,7 @@ export function getApiOperationsList(parsedSchema: OpenApiDocument): NormalizedO
requestParams,
requestQuery,
requestBody,
requestAttachment,
response,
};

View file

@ -7,6 +7,7 @@
*/
import type { OpenAPIV3 } from 'openapi-types';
import { GenerationContext } from './get_generation_context';
interface AdditionalProperties {
/**
@ -74,5 +75,12 @@ export interface NormalizedOperation {
requestParams?: NormalizedSchemaItem;
requestQuery?: NormalizedSchemaItem;
requestBody?: NormalizedSchemaItem;
requestAttachment?: NormalizedSchemaItem;
response?: NormalizedSchemaItem;
}
export interface ParsedSource {
sourcePath: string;
generatedPath: string;
generationContext: GenerationContext;
}

View file

@ -8,7 +8,7 @@
import type Handlebars from '@kbn/handlebars';
import { HelperOptions } from 'handlebars';
import { snakeCase, camelCase } from 'lodash';
import { snakeCase, camelCase, upperCase } from 'lodash';
export function registerHelpers(handlebarsInstance: typeof Handlebars) {
handlebarsInstance.registerHelper('concat', (...args) => {
@ -17,6 +17,7 @@ export function registerHelpers(handlebarsInstance: typeof Handlebars) {
});
handlebarsInstance.registerHelper('snakeCase', snakeCase);
handlebarsInstance.registerHelper('camelCase', camelCase);
handlebarsInstance.registerHelper('upperCase', upperCase);
handlebarsInstance.registerHelper('toJSON', (value: unknown) => {
return JSON.stringify(value);
});

View file

@ -8,7 +8,7 @@
import Handlebars from 'handlebars';
import { resolve } from 'path';
import { GenerationContext } from '../parser/get_generation_context';
import { BundleGenerationContext, GenerationContext } from '../parser/get_generation_context';
import { registerHelpers } from './register_helpers';
import { registerTemplates } from './register_templates';
@ -18,6 +18,7 @@ export type TemplateName = (typeof AVAILABLE_TEMPLATES)[number];
export interface ITemplateService {
compileTemplate: (templateName: TemplateName, context: GenerationContext) => string;
compileBundleTemplate: (templateName: TemplateName, context: BundleGenerationContext) => string;
}
/**
@ -34,5 +35,8 @@ export const initTemplateService = async (): Promise<ITemplateService> => {
compileTemplate: (templateName: TemplateName, context: GenerationContext) => {
return handlebars.compile(templates[templateName])(context);
},
compileBundleTemplate: (templateName: TemplateName, context: BundleGenerationContext) => {
return handlebars.compile(templates[templateName])(context);
},
};
};

View file

@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
{{> disclaimer}}
import type { KbnClient } from '@kbn/test';
import { ToolingLog } from '@kbn/tooling-log';
import { ELASTIC_HTTP_VERSION_HEADER, X_ELASTIC_INTERNAL_ORIGIN_REQUEST } from '@kbn/core-http-common';
import { replaceParams } from '@kbn/openapi-common/shared';
import { catchAxiosErrorFormatAndThrow } from '@kbn/securitysolution-utils';
import { FtrProviderContext } from 'x-pack/test/api_integration/ftr_provider_context';
{{#each sources}}
{{#if generationContext.operations}}
import type {
{{#each generationContext.operations}}
{{operationId}}RequestQueryInput,
{{operationId}}RequestParamsInput,
{{operationId}}RequestBodyInput,
{{operationId}}Response,
{{/each}}
} from '{{generatedPath}}';
{{/if}}
{{/each}}
export interface ClientOptions {
kbnClient: KbnClient;
log: ToolingLog;
}
export class Client {
readonly kbnClient: KbnClient;
readonly log: ToolingLog;
constructor(options: ClientOptions) {
this.kbnClient = options.kbnClient;
this.log = options.log;
}
{{#each operations}}
{{#if description}}
/**
* {{{description}}}
*/
{{/if}}
async {{camelCase operationId}} ({{#if (or requestQuery requestParams requestBody requestAttachment)}}props: {{operationId}}Props{{/if}}) {
this.log.info(`${new Date().toISOString()} Calling API {{operationId}}`);
return this.kbnClient
.request{{#if response}}<{{operationId}}Response>{{/if}}({
path: {{#if requestParams}}replaceParams('{{path}}', props.params){{else}}'{{path}}'{{/if}},
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '{{version}}',
},
method: '{{upperCase method}}',
{{#if requestBody}}body: props.body,{{else if requestAttachment}}body: props.attachment,{{/if}}
{{#if requestQuery}}query: props.query,{{/if}}
})
.catch(catchAxiosErrorFormatAndThrow)
}
{{/each}}
}
{{#each operations}}
{{#if (or requestQuery requestParams requestBody requestAttachment)}}
export interface {{operationId}}Props {
{{~#if requestQuery}}query: {{operationId}}RequestQueryInput;{{/if}}
{{~#if requestParams}}params: {{operationId}}RequestParamsInput;{{/if}}
{{~#if requestBody}}body: {{operationId}}RequestBodyInput;{{/if}}
{{~#if requestAttachment}}attachment: FormData;{{/if}}
}
{{/if}}
{{/each}}

View file

@ -0,0 +1,375 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/*
* NOTICE: Do not edit this file manually.
* This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
*
* info:
* title: Exceptions API client for quickstart
* version: Bundle (no version)
*/
import type { KbnClient } from '@kbn/test';
import { ToolingLog } from '@kbn/tooling-log';
import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
import { replaceParams } from '@kbn/openapi-common/shared';
import { catchAxiosErrorFormatAndThrow } from '@kbn/securitysolution-utils';
import type {
CreateExceptionListItemRequestBodyInput,
CreateExceptionListItemResponse,
} from './create_exception_list_item/create_exception_list_item.gen';
import type {
CreateExceptionListRequestBodyInput,
CreateExceptionListResponse,
} from './create_exception_list/create_exception_list.gen';
import type {
CreateRuleExceptionListItemsRequestParamsInput,
CreateRuleExceptionListItemsRequestBodyInput,
CreateRuleExceptionListItemsResponse,
} from './create_rule_exceptions/create_rule_exceptions.gen';
import type {
CreateSharedExceptionListRequestBodyInput,
CreateSharedExceptionListResponse,
} from './create_shared_exceptions_list/create_shared_exceptions_list.gen';
import type {
DeleteExceptionListItemRequestQueryInput,
DeleteExceptionListItemResponse,
} from './delete_exception_list_item/delete_exception_list_item.gen';
import type {
DeleteExceptionListRequestQueryInput,
DeleteExceptionListResponse,
} from './delete_exception_list/delete_exception_list.gen';
import type {
DuplicateExceptionListRequestQueryInput,
DuplicateExceptionListResponse,
} from './duplicate_exception_list/duplicate_exception_list.gen';
import type { ExportExceptionListRequestQueryInput } from './export_exception_list/export_exception_list.gen';
import type {
FindExceptionListItemsRequestQueryInput,
FindExceptionListItemsResponse,
} from './find_exception_list_items/find_exception_list_items.gen';
import type {
FindExceptionListsRequestQueryInput,
FindExceptionListsResponse,
} from './find_exception_lists/find_exception_lists.gen';
import type {
ImportExceptionListRequestQueryInput,
ImportExceptionListResponse,
} from './import_exceptions/import_exceptions.gen';
import type {
ReadExceptionListItemRequestQueryInput,
ReadExceptionListItemResponse,
} from './read_exception_list_item/read_exception_list_item.gen';
import type {
ReadExceptionListSummaryRequestQueryInput,
ReadExceptionListSummaryResponse,
} from './read_exception_list_summary/read_exception_list_summary.gen';
import type {
ReadExceptionListRequestQueryInput,
ReadExceptionListResponse,
} from './read_exception_list/read_exception_list.gen';
import type {
UpdateExceptionListItemRequestBodyInput,
UpdateExceptionListItemResponse,
} from './update_exception_list_item/update_exception_list_item.gen';
import type {
UpdateExceptionListRequestBodyInput,
UpdateExceptionListResponse,
} from './update_exception_list/update_exception_list.gen';
export interface ClientOptions {
kbnClient: KbnClient;
log: ToolingLog;
}
export class Client {
readonly kbnClient: KbnClient;
readonly log: ToolingLog;
constructor(options: ClientOptions) {
this.kbnClient = options.kbnClient;
this.log = options.log;
}
async createExceptionList(props: CreateExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateExceptionList`);
return this.kbnClient
.request<CreateExceptionListResponse>({
path: '/api/exception_lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async createExceptionListItem(props: CreateExceptionListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateExceptionListItem`);
return this.kbnClient
.request<CreateExceptionListItemResponse>({
path: '/api/exception_lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async createRuleExceptionListItems(props: CreateRuleExceptionListItemsProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateRuleExceptionListItems`);
return this.kbnClient
.request<CreateRuleExceptionListItemsResponse>({
path: replaceParams('/api/detection_engine/rules/{id}/exceptions', props.params),
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async createSharedExceptionList(props: CreateSharedExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateSharedExceptionList`);
return this.kbnClient
.request<CreateSharedExceptionListResponse>({
path: '/api/exceptions/shared',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async deleteExceptionList(props: DeleteExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API DeleteExceptionList`);
return this.kbnClient
.request<DeleteExceptionListResponse>({
path: '/api/exception_lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'DELETE',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async deleteExceptionListItem(props: DeleteExceptionListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API DeleteExceptionListItem`);
return this.kbnClient
.request<DeleteExceptionListItemResponse>({
path: '/api/exception_lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'DELETE',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async duplicateExceptionList(props: DuplicateExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API DuplicateExceptionList`);
return this.kbnClient
.request<DuplicateExceptionListResponse>({
path: '/api/exception_lists/_duplicate',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
/**
* Exports an exception list and its associated items to an .ndjson file
*/
async exportExceptionList(props: ExportExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API ExportExceptionList`);
return this.kbnClient
.request({
path: '/api/exception_lists/_export',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async findExceptionListItems(props: FindExceptionListItemsProps) {
this.log.info(`${new Date().toISOString()} Calling API FindExceptionListItems`);
return this.kbnClient
.request<FindExceptionListItemsResponse>({
path: '/api/exception_lists/items/_find',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async findExceptionLists(props: FindExceptionListsProps) {
this.log.info(`${new Date().toISOString()} Calling API FindExceptionLists`);
return this.kbnClient
.request<FindExceptionListsResponse>({
path: '/api/exception_lists/_find',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
/**
* Imports an exception list and associated items
*/
async importExceptionList(props: ImportExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API ImportExceptionList`);
return this.kbnClient
.request<ImportExceptionListResponse>({
path: '/api/exception_lists/_import',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.attachment,
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readExceptionList(props: ReadExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API ReadExceptionList`);
return this.kbnClient
.request<ReadExceptionListResponse>({
path: '/api/exception_lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readExceptionListItem(props: ReadExceptionListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API ReadExceptionListItem`);
return this.kbnClient
.request<ReadExceptionListItemResponse>({
path: '/api/exception_lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readExceptionListSummary(props: ReadExceptionListSummaryProps) {
this.log.info(`${new Date().toISOString()} Calling API ReadExceptionListSummary`);
return this.kbnClient
.request<ReadExceptionListSummaryResponse>({
path: '/api/exception_lists/summary',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async updateExceptionList(props: UpdateExceptionListProps) {
this.log.info(`${new Date().toISOString()} Calling API UpdateExceptionList`);
return this.kbnClient
.request<UpdateExceptionListResponse>({
path: '/api/exception_lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PUT',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async updateExceptionListItem(props: UpdateExceptionListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API UpdateExceptionListItem`);
return this.kbnClient
.request<UpdateExceptionListItemResponse>({
path: '/api/exception_lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PUT',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
}
export interface CreateExceptionListProps {
body: CreateExceptionListRequestBodyInput;
}
export interface CreateExceptionListItemProps {
body: CreateExceptionListItemRequestBodyInput;
}
export interface CreateRuleExceptionListItemsProps {
params: CreateRuleExceptionListItemsRequestParamsInput;
body: CreateRuleExceptionListItemsRequestBodyInput;
}
export interface CreateSharedExceptionListProps {
body: CreateSharedExceptionListRequestBodyInput;
}
export interface DeleteExceptionListProps {
query: DeleteExceptionListRequestQueryInput;
}
export interface DeleteExceptionListItemProps {
query: DeleteExceptionListItemRequestQueryInput;
}
export interface DuplicateExceptionListProps {
query: DuplicateExceptionListRequestQueryInput;
}
export interface ExportExceptionListProps {
query: ExportExceptionListRequestQueryInput;
}
export interface FindExceptionListItemsProps {
query: FindExceptionListItemsRequestQueryInput;
}
export interface FindExceptionListsProps {
query: FindExceptionListsRequestQueryInput;
}
export interface ImportExceptionListProps {
query: ImportExceptionListRequestQueryInput;
attachment: FormData;
}
export interface ReadExceptionListProps {
query: ReadExceptionListRequestQueryInput;
}
export interface ReadExceptionListItemProps {
query: ReadExceptionListItemRequestQueryInput;
}
export interface ReadExceptionListSummaryProps {
query: ReadExceptionListSummaryRequestQueryInput;
}
export interface UpdateExceptionListProps {
body: UpdateExceptionListRequestBodyInput;
}
export interface UpdateExceptionListItemProps {
body: UpdateExceptionListItemRequestBodyInput;
}

View file

@ -34,4 +34,18 @@ const ROOT = resolve(__dirname, '..');
),
},
});
await generate({
title: 'Exceptions API client for quickstart',
rootDir: ROOT,
sourceGlob: './api/**/*.schema.yaml',
templateName: 'api_client_quickstart',
skipLinting: true,
bundle: {
outFile: join(
REPO_ROOT,
'packages/kbn-securitysolution-exceptions-common/api/quickstart_client.gen.ts'
),
},
});
})();

View file

@ -10,6 +10,10 @@
"@kbn/openapi-common",
"@kbn/zod-helpers",
"@kbn/securitysolution-lists-common",
"@kbn/zod"
"@kbn/test",
"@kbn/tooling-log",
"@kbn/core-http-common",
"@kbn/securitysolution-utils",
"@kbn/zod",
]
}

View file

@ -0,0 +1,370 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
/*
* NOTICE: Do not edit this file manually.
* This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
*
* info:
* title: Lists API client for quickstart
* version: Bundle (no version)
*/
import type { KbnClient } from '@kbn/test';
import { ToolingLog } from '@kbn/tooling-log';
import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
import { catchAxiosErrorFormatAndThrow } from '@kbn/securitysolution-utils';
import type { CreateListIndexResponse } from './create_list_index/create_list_index.gen';
import type {
CreateListItemRequestBodyInput,
CreateListItemResponse,
} from './create_list_item/create_list_item.gen';
import type { CreateListRequestBodyInput, CreateListResponse } from './create_list/create_list.gen';
import type { DeleteListIndexResponse } from './delete_list_index/delete_list_index.gen';
import type {
DeleteListItemRequestQueryInput,
DeleteListItemResponse,
} from './delete_list_item/delete_list_item.gen';
import type {
DeleteListRequestQueryInput,
DeleteListResponse,
} from './delete_list/delete_list.gen';
import type { ExportListItemsRequestQueryInput } from './export_list_items/export_list_items.gen';
import type {
FindListItemsRequestQueryInput,
FindListItemsResponse,
} from './find_list_items/find_list_items.gen';
import type { FindListsRequestQueryInput, FindListsResponse } from './find_lists/find_lists.gen';
import type {
ImportListItemsRequestQueryInput,
ImportListItemsResponse,
} from './import_list_items/import_list_items.gen';
import type {
PatchListItemRequestBodyInput,
PatchListItemResponse,
} from './patch_list_item/patch_list_item.gen';
import type { PatchListRequestBodyInput, PatchListResponse } from './patch_list/patch_list.gen';
import type { ReadListIndexResponse } from './read_list_index/read_list_index.gen';
import type {
ReadListItemRequestQueryInput,
ReadListItemResponse,
} from './read_list_item/read_list_item.gen';
import type { ReadListPrivilegesResponse } from './read_list_privileges/read_list_privileges.gen';
import type { ReadListRequestQueryInput, ReadListResponse } from './read_list/read_list.gen';
import type {
UpdateListItemRequestBodyInput,
UpdateListItemResponse,
} from './update_list_item/update_list_item.gen';
import type { UpdateListRequestBodyInput, UpdateListResponse } from './update_list/update_list.gen';
export interface ClientOptions {
kbnClient: KbnClient;
log: ToolingLog;
}
export class Client {
readonly kbnClient: KbnClient;
readonly log: ToolingLog;
constructor(options: ClientOptions) {
this.kbnClient = options.kbnClient;
this.log = options.log;
}
async createList(props: CreateListProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateList`);
return this.kbnClient
.request<CreateListResponse>({
path: '/api/lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async createListIndex() {
this.log.info(`${new Date().toISOString()} Calling API CreateListIndex`);
return this.kbnClient
.request<CreateListIndexResponse>({
path: '/api/lists/index',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
})
.catch(catchAxiosErrorFormatAndThrow);
}
async createListItem(props: CreateListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API CreateListItem`);
return this.kbnClient
.request<CreateListItemResponse>({
path: '/api/lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async deleteList(props: DeleteListProps) {
this.log.info(`${new Date().toISOString()} Calling API DeleteList`);
return this.kbnClient
.request<DeleteListResponse>({
path: '/api/lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'DELETE',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async deleteListIndex() {
this.log.info(`${new Date().toISOString()} Calling API DeleteListIndex`);
return this.kbnClient
.request<DeleteListIndexResponse>({
path: '/api/lists/index',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'DELETE',
})
.catch(catchAxiosErrorFormatAndThrow);
}
async deleteListItem(props: DeleteListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API DeleteListItem`);
return this.kbnClient
.request<DeleteListItemResponse>({
path: '/api/lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'DELETE',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
/**
* Exports list item values from the specified list
*/
async exportListItems(props: ExportListItemsProps) {
this.log.info(`${new Date().toISOString()} Calling API ExportListItems`);
return this.kbnClient
.request({
path: '/api/lists/items/_export',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async findListItems(props: FindListItemsProps) {
this.log.info(`${new Date().toISOString()} Calling API FindListItems`);
return this.kbnClient
.request<FindListItemsResponse>({
path: '/api/lists/items/_find',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async findLists(props: FindListsProps) {
this.log.info(`${new Date().toISOString()} Calling API FindLists`);
return this.kbnClient
.request<FindListsResponse>({
path: '/api/lists/_find',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
/**
* Imports a list of items from a `.txt` or `.csv` file. The maximum file size is 9 million bytes.
You can import items to a new or existing list.
*/
async importListItems(props: ImportListItemsProps) {
this.log.info(`${new Date().toISOString()} Calling API ImportListItems`);
return this.kbnClient
.request<ImportListItemsResponse>({
path: '/api/lists/items/_import',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'POST',
body: props.attachment,
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async patchList(props: PatchListProps) {
this.log.info(`${new Date().toISOString()} Calling API PatchList`);
return this.kbnClient
.request<PatchListResponse>({
path: '/api/lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PATCH',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async patchListItem(props: PatchListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API PatchListItem`);
return this.kbnClient
.request<PatchListItemResponse>({
path: '/api/lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PATCH',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readList(props: ReadListProps) {
this.log.info(`${new Date().toISOString()} Calling API ReadList`);
return this.kbnClient
.request<ReadListResponse>({
path: '/api/lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readListIndex() {
this.log.info(`${new Date().toISOString()} Calling API ReadListIndex`);
return this.kbnClient
.request<ReadListIndexResponse>({
path: '/api/lists/index',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readListItem(props: ReadListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API ReadListItem`);
return this.kbnClient
.request<ReadListItemResponse>({
path: '/api/lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
query: props.query,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async readListPrivileges() {
this.log.info(`${new Date().toISOString()} Calling API ReadListPrivileges`);
return this.kbnClient
.request<ReadListPrivilegesResponse>({
path: '/api/lists/privileges',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'GET',
})
.catch(catchAxiosErrorFormatAndThrow);
}
async updateList(props: UpdateListProps) {
this.log.info(`${new Date().toISOString()} Calling API UpdateList`);
return this.kbnClient
.request<UpdateListResponse>({
path: '/api/lists',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PUT',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async updateListItem(props: UpdateListItemProps) {
this.log.info(`${new Date().toISOString()} Calling API UpdateListItem`);
return this.kbnClient
.request<UpdateListItemResponse>({
path: '/api/lists/items',
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
},
method: 'PUT',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
}
export interface CreateListProps {
body: CreateListRequestBodyInput;
}
export interface CreateListItemProps {
body: CreateListItemRequestBodyInput;
}
export interface DeleteListProps {
query: DeleteListRequestQueryInput;
}
export interface DeleteListItemProps {
query: DeleteListItemRequestQueryInput;
}
export interface ExportListItemsProps {
query: ExportListItemsRequestQueryInput;
}
export interface FindListItemsProps {
query: FindListItemsRequestQueryInput;
}
export interface FindListsProps {
query: FindListsRequestQueryInput;
}
export interface ImportListItemsProps {
query: ImportListItemsRequestQueryInput;
attachment: FormData;
}
export interface PatchListProps {
body: PatchListRequestBodyInput;
}
export interface PatchListItemProps {
body: PatchListItemRequestBodyInput;
}
export interface ReadListProps {
query: ReadListRequestQueryInput;
}
export interface ReadListItemProps {
query: ReadListItemRequestQueryInput;
}
export interface UpdateListProps {
body: UpdateListRequestBodyInput;
}
export interface UpdateListItemProps {
body: UpdateListItemRequestBodyInput;
}

View file

@ -34,4 +34,18 @@ const ROOT = resolve(__dirname, '..');
),
},
});
await generate({
title: 'Lists API client for quickstart',
rootDir: ROOT,
sourceGlob: './api/**/*.schema.yaml',
templateName: 'api_client_quickstart',
skipLinting: true,
bundle: {
outFile: join(
REPO_ROOT,
'packages/kbn-securitysolution-lists-common/api/quickstart_client.gen.ts'
),
},
});
})();

View file

@ -9,6 +9,10 @@
"kbn_references": [
"@kbn/zod-helpers",
"@kbn/openapi-common",
"@kbn/test",
"@kbn/tooling-log",
"@kbn/core-http-common",
"@kbn/securitysolution-utils",
"@kbn/zod",
]
}

View file

@ -7,6 +7,7 @@
*/
export * from './src/add_remove_id_to_item';
export * from './src/axios';
export * from './src/transform_data_to_ndjson';
export * from './src/path_validations';
export * from './src/esql';

View file

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { AxiosError } from 'axios';
export class FormattedAxiosError extends Error {
public readonly request: {
method: string;
url: string;
data: unknown;
};
public readonly response: {
status: number;
statusText: string;
data: any;
};
constructor(axiosError: AxiosError) {
const method = axiosError.config?.method ?? '';
const url = axiosError.config?.url ?? '';
super(
`${axiosError.message}${
axiosError?.response?.data ? `: ${JSON.stringify(axiosError?.response?.data)}` : ''
}${url ? `\n(Request: ${method} ${url})` : ''}`
);
this.request = {
method,
url,
data: axiosError.config?.data ?? '',
};
this.response = {
status: axiosError?.response?.status ?? 0,
statusText: axiosError?.response?.statusText ?? '',
data: axiosError?.response?.data,
};
this.name = this.constructor.name;
}
toJSON() {
return {
message: this.message,
request: this.request,
response: this.response,
};
}
toString() {
return JSON.stringify(this.toJSON(), null, 2);
}
}
/**
* Used with `promise.catch()`, it will format the Axios error to a new error and will re-throw
* @param error
*/
export const catchAxiosErrorFormatAndThrow = (error: Error): never => {
if (error instanceof AxiosError) {
throw new FormattedAxiosError(error);
}
throw error;
};

View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import limit from 'p-limit';
/**
* This type is just an async function's type
*/
type RequestFactory<Output> = () => Promise<Output>;
/**
* Helper function to call a large number of async functions with limited concurrency.
* Example pattern of how to create functions to pass in:
*
* const ruleCopies = duplicateRuleParams(basicRule, 200);
* const functions = ruleCopies.map((rule) => () => detectionsClient.createRule({ body: rule }));
*
* Note that the `map` call in the example returns a *function* that calls detectionsClient.createRule, it doesn't call createRule immediately.
*
* @param functions Async functions to call with limited concurrency
* @param concurrency Maximum number of concurrent function calls
* @returns Results from all functions passed in
*/
export const concurrentlyExec = async <Output>(
requestFactories: Array<RequestFactory<Output>>,
concurrency: number = 10
) => {
const limiter = limit(concurrency);
const promises = requestFactories.map((f) => limiter(f));
return Promise.all(promises);
};

View file

@ -68,7 +68,7 @@ export interface ReqOptions {
description?: string;
path: string;
query?: Record<string, any>;
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
body?: any;
retries?: number;
headers?: Record<string, string>;

File diff suppressed because it is too large Load diff

View file

@ -12,6 +12,8 @@ const { resolve, join } = require('path');
const SECURITY_SOLUTION_ROOT = resolve(__dirname, '../..');
// This script is also run in CI: to track down the scripts that run it in CI, code search for `yarn openapi:generate` in the `.buildkite` top level directory
(async () => {
await generate({
title: 'API route schemas',
@ -30,4 +32,18 @@ const SECURITY_SOLUTION_ROOT = resolve(__dirname, '../..');
outFile: join(REPO_ROOT, 'x-pack/test/api_integration/services/security_solution_api.gen.ts'),
},
});
await generate({
title: 'API client for quickstart',
rootDir: SECURITY_SOLUTION_ROOT,
sourceGlob: './common/**/*.schema.yaml',
templateName: 'api_client_quickstart',
skipLinting: false,
bundle: {
outFile: join(
REPO_ROOT,
'x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts'
),
},
});
})();

View file

@ -0,0 +1,136 @@
# Quickstart for Developers
These tools make it fast and easy to create detection rules, exceptions, value lists, and source data for testing.
## Usage
`node x-pack/plugins/security_solution/scripts/quickstart/run.js`: Runs the script defined in `scratchpad.ts`
Options:
--username: User name to be used for auth against elasticsearch and kibana (Default: elastic).
--password: User name Password (Default: changeme)
--kibana: The url to Kibana (Default: http://127.0.0.1:5601). In most cases you'll want to set this URL to include the basepath as well.
--apikey: The API key for authentication, overrides username/password - use for serverless projects
`scratchpad.ts` already contains code to set up clients for Elasticsearch and Kibana. In addition it provides clients for Security Solution, Lists, and Exceptions APIs, built on top of the Kibana client. However, it does not create any rules/exceptions/lists/data - it's a blank slate for you to immediately begin creating the resources you want for testing. Please don't commit data-generating code to `scratchpad.ts`! Instead, when you have built a data-generating script that might be useful to others, please extract the useful components to the `quickstart/modules` folder and leave `scratchpad.ts` empty for the next developer.
### Environments
The API clients are designed to work with any delivery method - local, cloud, or serverless deployments. For deployments that do not allow username/password auth, use an API key.
## Modules
Extracting data-generating logic into reusable modules that other people will actually use is the hardest part of sharing these scripts. To that end, it's crucial that the modules are organized as neatly as possible and extremely clear about what they do. If the modules are even slightly confusing, it will be faster for people to rebuild the same logic than to figure out how the existing scripts work.
### Data
Functions to create documents with various properties. This initial implementation has a function to create a document with an arbitrary number of fields and arbitrary amount of data in each field, but should be extended with more functions to create sets of documents with specific relationships such as X total documents with Y number of unique hosts etc.
### Entity Analytics
Functions to help install fake entity analytics data. Useful for testing alert enrichment based on entity analytics.
### Exceptions
Functions to help create exceptions with various properties. For example, one helper takes an array of values and automatically creates a value list exception item from that array - internally, it creates the value list and an exception item that references the list.
### Frozen (TODO)
Functions to help create frozen tier data quickly. These functions (once implemented) will take existing data and immediately move it to frozen for test purposes.
### Lists
Functions to help interact with the Lists APIs. The initial helper function makes it easy to import a value list from an array, since the process of attaching a file to a request (as the API expects) is not that intuitive.
### Mappings
Functions to help setup mappings. Provides the ECS mapping as well as helpers to generate mappings with tons of fields.
### Rules
Functions to help create rules along with data specific to each rule (WIP). Each sample rule defined in this folder should have an associated function to generate data that triggers alerts for the rule.
## Speed
To run a number of API requests in parallel, use `concurrentlyExec` from @kbn/securitysolution-utils.
## Examples
### Create a Rule
```
// Extra imports
import { concurrentlyExec } from '@kbn/securitysolution-utils/src/client_concurrency';
import { basicRule } from './modules/rules/new_terms/basic_rule';
import { duplicateRuleParams } from './modules/rules';
// ... omitted client setup stuff
// Core logic
const ruleCopies = duplicateRuleParams(basicRule, 200);
const functions = ruleCopies.map((rule) => () => detectionsClient.createRule({ body: rule }));
const responses = await concurrentlyExec(functions);
```
### Create 200 Rules and an Exception for each one
```
// Extra imports
import { concurrentlyExec } from '@kbn/securitysolution-utils/src/client_concurrency';
import { basicRule } from './modules/rules/new_terms/basic_rule';
import { duplicateRuleParams } from './modules/rules';
import { buildCreateRuleExceptionListItemsProps } from './modules/exceptions';
// ... omitted client setup stuff
// Core logic
const ruleCopies = duplicateRuleParams(basicRule, 200);
const response = await detectionsClient.bulkCreateRules({ body: ruleCopies });
const createdRules: RuleResponse[] = response.data.filter(
(r) => r.id != null
) as RuleResponse[];
// This map looks a bit confusing, but the concept is simple: take the rules we just created and
// create a *function* per rule to create an exception for that rule. We want a function to call later instead of just
// calling the API immediately to limit the number of requests in flight (with `concurrentlyExec`)
const exceptionsFunctions = createdRules.map(
(r) => () =>
exceptionsClient.createRuleExceptionListItems(
buildCreateRuleExceptionListItemsProps({ id: r.id })
)
);
const exceptionsResponses = await concurrentlyExec(exceptionsFunctions);
```
### Run 10 Rule Preview Requests Simultaneously
```
const previewPromises = range(50).map(
(idx) => () =>
detectionsClient.rulePreview({
body: {
...getBasicRuleMetadata(),
type: 'query',
timeframeEnd: '2024-08-21T20:37:37.114Z',
invocationCount: 1,
from: 'now-6m',
interval: '5m',
index: [index],
query: '*',
},
})
);
const results = (await concurrentlyExec(previewPromises, 50)).map(
(result) => result.data.logs
);
```
## Future Work
### Interactive Mode
It may be useful to have a mode where the CLI waits for input from the user and creates resources selected from a predefined list.
### Resource Tracking/Cleanup
It may also be useful to have the tooling automatically keep track of the created resources so they can be deleted automatically when finished.

View file

@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { range } from 'lodash';
import crypto from 'crypto';
/**
* A basic function to generate random data filling out a document. Useful for forcing
* Elasticsearch to handle large amounts of data in a single doc.
* @param numFields Number of fields to generate in each document
* @param fieldSize Number of bytes to generate for each field
* @returns Object representing the new document
*/
export const buildLargeDocument = ({
numFields,
fieldSize,
}: {
numFields: number;
fieldSize: number;
}): Record<string, string> => {
const doc: Record<string, string> = {};
range(numFields).forEach((idx) => {
doc[`field_${idx}`] = crypto.randomBytes(fieldSize).toString('hex');
});
return doc;
};
export const addTimestampToDoc = ({
timestamp = new Date(),
doc,
}: {
timestamp?: Date;
doc: Record<string, unknown>;
}): Record<string, unknown> => {
doc['@timestamp'] = timestamp;
return doc;
};
export const addFieldToDoc = ({
fieldName,
fieldValue,
doc,
}: {
fieldName: string;
fieldValue: unknown;
doc: Record<string, unknown>;
}): Record<string, unknown> => {
doc[fieldName] = fieldValue;
return doc;
};

View file

@ -0,0 +1,154 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { Client } from '@elastic/elasticsearch';
import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types';
import { getRiskScoreLatestIndex } from '../../../../common/entity_analytics/risk_engine';
export const getRiskScoreIndexMappings: () => MappingTypeMapping = () => ({
dynamic: 'false',
properties: {
'@timestamp': {
type: 'date',
},
host: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
},
notes: {
type: 'keyword',
},
},
},
},
},
user: {
properties: {
name: {
type: 'keyword',
},
risk: {
properties: {
calculated_level: {
type: 'keyword',
},
calculated_score: {
type: 'float',
},
calculated_score_norm: {
type: 'float',
},
category_1_count: {
type: 'long',
},
category_1_score: {
type: 'float',
},
id_field: {
type: 'keyword',
},
id_value: {
type: 'keyword',
},
inputs: {
properties: {
category: {
type: 'keyword',
},
description: {
type: 'keyword',
},
id: {
type: 'keyword',
},
index: {
type: 'keyword',
},
risk_score: {
type: 'float',
},
timestamp: {
type: 'date',
},
},
},
notes: {
type: 'keyword',
},
},
},
},
},
},
});
export const createRiskScoreIndex = async ({ client }: { client: Client }) => {
const riskScoreIndexName = getRiskScoreLatestIndex();
await client.indices.create({
index: riskScoreIndexName,
mappings: getRiskScoreIndexMappings(),
});
};
export const addRiskScoreDoc = async ({ client }: { client: Client }) => {
const riskScoreIndexName = getRiskScoreLatestIndex();
await client.index({
index: riskScoreIndexName,
document: {
host: {
name: 'test host',
risk: { calculated_level: 'low', calculated_score: 21, calculated_score_norm: 51 },
},
},
});
};

View file

@ -0,0 +1,117 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ExceptionListItemEntry } from '@kbn/securitysolution-exceptions-common/api';
import type {
CreateRuleExceptionListItemsProps,
Client as ExceptionsClient,
} from '@kbn/securitysolution-exceptions-common/api/quickstart_client.gen';
import type { ListType } from '@kbn/securitysolution-lists-common/api';
import type { Client as ListsClient } from '@kbn/securitysolution-lists-common/api/quickstart_client.gen';
import { importListItemsWrapper } from '../lists';
export const getMatchEntry: () => ExceptionListItemEntry = () => ({
type: 'match',
field: 'host.name',
value: 'host-1',
operator: 'included',
});
export const buildCreateRuleExceptionListItemsProps: (props: {
id: string;
}) => CreateRuleExceptionListItemsProps = ({ id }) => ({
params: { id },
body: {
items: [
{
description: 'test',
type: 'simple',
name: 'test',
entries: [
{
type: 'match',
field: 'test',
value: 'test',
operator: 'included',
},
],
},
],
},
});
export const buildListExceptionListItemsProps: (props: {
id: string;
listId: string;
listType: ListType;
}) => CreateRuleExceptionListItemsProps = ({ id, listId, listType }) => ({
params: { id },
body: {
items: [
{
description: 'test',
type: 'simple',
name: 'test',
entries: [
{
type: 'list',
field: 'test',
operator: 'included',
list: {
id: listId,
type: listType,
},
},
],
},
],
},
});
export const createValueListException = async ({
listItems,
listName = 'myList',
listType = 'keyword',
exceptionListId,
exceptionsClient,
listsClient,
}: {
listItems: string[];
listName: string;
listType: ListType;
exceptionListId: string;
exceptionsClient: ExceptionsClient;
listsClient: ListsClient;
}) => {
await importListItemsWrapper({
listsClient,
listName,
listFileType: 'txt',
listType,
listItems,
});
await exceptionsClient.createExceptionListItem({
body: {
description: 'test',
list_id: exceptionListId,
type: 'simple',
name: 'test item',
entries: [
{
type: 'list',
field: 'host.name',
list: {
id: `${listName}.txt`,
type: listType,
},
operator: 'included',
},
],
},
});
};

View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ListType } from '@kbn/securitysolution-lists-common/api';
import type { Client as ListsClient } from '@kbn/securitysolution-lists-common/api/quickstart_client.gen';
/**
* Efficiently turn an array of values into a value list in Kibana. Since the value list import API expects a file to be attached,
* this function handles turning an array into a file attachment for the API and making the appropriate request structure.
*
* @param listsClient The lists client for accessing lists APIs through the CLI tool.
* @param listName Name of the list to create. Should be unique, as this will become the list ID as well.
* @param listFileType The file type suffix to use when sending the import list request. This should have no effect on the functionality of the created list,
* it's effectively just metadata.
* @param listItems Array of values to insert into the list.
* @param listType Elasticsearch field type to use for the list values.
* @returns Created list info
*/
export const importListItemsWrapper = ({
listsClient,
listName,
listFileType,
listItems,
listType,
}: {
listsClient: ListsClient;
listName: string;
listFileType: 'txt' | 'csv';
listItems: unknown[];
listType: ListType;
}) => {
const blob = new Blob([listItems.join('\r\n')], { type: 'application/json' });
const body = new FormData();
body.append('file', blob, `${listName}.${listFileType}`);
return listsClient.importListItems({ query: { type: listType }, attachment: body });
};

View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { range } from 'lodash';
import type {
IndicesIndexSettings,
MappingProperty,
MappingTypeMapping,
} from '@elastic/elasticsearch/lib/api/types';
import { mappingFromFieldMap } from '@kbn/alerting-plugin/common';
import { ecsFieldMap } from '@kbn/alerts-as-data-utils';
export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap);
export interface GenerateLargeMappingPropertiesProps {
size: number;
fieldNameGenerator?: (idx: number) => string;
fieldType?: MappingProperty['type'];
}
/**
* Generates a large number of field mappings.
* @param size Number of fields to generate
* @param fieldNameGenerator Optional function to determine how the fields should be named - defaults to `field_1`, `field_2`, etc. Dot notation can be
* used to nest fields, e.g. myField.subfield_1, myField.subfield_2, etc.
* @param fieldType The type of ES field to generate
* @returns An object ready to be inserted into the `properties` of an ES mapping object
*/
export const generateLargeMappingProperties = ({
size,
fieldNameGenerator = (idx: number) => `field_${idx}`,
fieldType = 'keyword',
}: GenerateLargeMappingPropertiesProps): Record<string, MappingProperty> => {
const properties: Record<string, MappingProperty> = {};
range(size).forEach((i) => {
properties[fieldNameGenerator(i)] = { type: fieldType } as MappingProperty; // Cast is needed here because TS can't seem to figure out this type correctly
});
return properties;
};
/**
* Simple wrapper around `generateLargeMappingProperties` to build a complete ES mapping object instead of just the core `properties`. See
* generateLargeMappingProperties for details.
* @returns A complete ES mapping object
*/
export const generateLargeMapping = (
props: GenerateLargeMappingPropertiesProps
): MappingTypeMapping => {
return { properties: generateLargeMappingProperties(props) };
};
/**
* If you're generating a large mapping (more than 1k fields), you'll need to also set this index setting or else you'll see the following error:
* Root causes:
illegal_argument_exception: Limit of total fields [1000] has been exceeded
*/
export const getSettings: ({ maxFields }: { maxFields: number }) => IndicesIndexSettings = ({
maxFields,
}: {
maxFields: number;
}) => ({
'index.mapping.total_fields.limit': maxFields,
});
/**
* Injects additional fields into a mapping. Useful for adding additional fields to the standard ECS mapping.
* @param mapping
* @param properties
* @returns A new mapping combining the original mapping and new properties.
*/
export const addPropertiesToMapping = (
mapping: MappingTypeMapping,
properties: Record<string, MappingProperty>
): MappingTypeMapping => {
return {
...mapping,
properties: {
...mapping.properties,
...properties,
},
};
};

View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { range } from 'lodash';
import type { RuleCreateProps } from '../../../../common/api/detection_engine';
export const duplicateRuleParams = (
rule: RuleCreateProps,
numCopies: number
): RuleCreateProps[] => {
return range(numCopies).map((idx) => ({ ...rule, name: `${rule.name}_${idx}` }));
};
export const getBasicRuleMetadata = () => ({
name: 'Test rule',
description: 'Test rule',
severity: 'low' as const,
risk_score: 21,
});

View file

@ -0,0 +1,64 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { Client } from '@elastic/elasticsearch';
import type { NewTermsRuleCreateProps } from '../../../../../common/api/detection_engine';
import { getEcsMapping, generateLargeMappingProperties, getSettings } from '../../mappings';
export const basicRule: NewTermsRuleCreateProps = {
type: 'new_terms',
description: 'test rule',
name: 'test rule',
risk_score: 20,
severity: 'low',
query: '*',
new_terms_fields: ['host.name'],
history_window_start: 'now-7d',
enabled: false,
index: ['test'],
};
/**
* Create test data to trigger the new terms rule above.
*/
export const createData = async (client: Client) => {
const index = 'test-index';
const ecsMapping = getEcsMapping();
await client.indices.create({
index,
mappings: {
...ecsMapping,
properties: { ...ecsMapping.properties, ...generateLargeMappingProperties({ size: 100 }) },
},
settings: getSettings({ maxFields: 5000 }),
});
const now = new Date();
const old = new Date(now);
old.setDate(old.getDate() - 3);
await client.bulk({
index,
operations: [
{ index: {} },
{
'@timestamp': now.toISOString(),
'host.name': 'host-1',
},
{ index: {} },
{
'@timestamp': old.toISOString(),
'host.name': 'host-1',
},
{ index: {} },
{
'@timestamp': now.toISOString(),
'host.name': 'host-2',
},
],
});
};

View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { QueryRuleCreateProps } from '../../../../../common/api/detection_engine';
export const buildBasicQueryRule: () => QueryRuleCreateProps = () => ({
type: 'query',
query: '*:*',
name: 'Sample Query Rule',
description: 'Sample Query Rule',
severity: 'low',
risk_score: 21,
});

View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ThreatMatchRuleCreateProps } from '../../../../../common/api/detection_engine';
export const basicThreatMatchRule: ThreatMatchRuleCreateProps = {
type: 'threat_match',
name: 'Basic threat match rule',
description: 'Basic threat match rule',
severity: 'low',
risk_score: 21,
query: '*',
index: ['test'],
threat_query: '*',
threat_index: ['threat_test'],
threat_mapping: [
{
entries: [
{
field: 'host.name',
value: 'host.name',
type: 'mapping',
},
],
},
],
};
export const generateThreatMatchRuleData = () => {};

View file

@ -0,0 +1,6 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
require('../../../../../src/setup_node_env');
require('./scratchpad').cli();

View file

@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { run } from '@kbn/dev-cli-runner';
import { Client as ListsClient } from '@kbn/securitysolution-lists-common/api/quickstart_client.gen';
import { Client as ExceptionsClient } from '@kbn/securitysolution-exceptions-common/api/quickstart_client.gen';
import { concurrentlyExec } from '@kbn/securitysolution-utils/src/client_concurrency';
import { HORIZONTAL_LINE } from '../endpoint/common/constants';
import { createEsClient, createKbnClient } from '../endpoint/common/stack_services';
import { createToolingLogger } from '../../common/endpoint/data_loaders/utils';
import { Client as DetectionsClient } from '../../common/api/quickstart_client.gen';
import { duplicateRuleParams } from './modules/rules';
import { basicRule } from './modules/rules/new_terms/basic_rule';
export const cli = () => {
run(
async (cliContext) => {
/**
* START Client setup - Generic Kibana Client, ES Client, and Detections/Lists/Exceptions specific clients
*/
createToolingLogger.setDefaultLogLevelFromCliFlags(cliContext.flags);
const log = cliContext.log;
const kbnClient = createKbnClient({
log,
url: cliContext.flags.kibana as string,
username: cliContext.flags.username as string,
password: cliContext.flags.password as string,
apiKey: cliContext.flags.apikey as string,
});
const esClient = createEsClient({
log,
url: cliContext.flags.elasticsearch as string,
username: cliContext.flags.username as string,
password: cliContext.flags.password as string,
apiKey: cliContext.flags.apikey as string,
});
const detectionsClient = new DetectionsClient({ kbnClient, log });
const listsClient = new ListsClient({ kbnClient, log });
const exceptionsClient = new ExceptionsClient({ kbnClient, log });
log.info(`${HORIZONTAL_LINE}
Environment Data Loader
${HORIZONTAL_LINE}
`);
log.info(`Loading data to: ${kbnClient.resolveUrl('')}`);
/**
* END Client setup
* START Custom data loader logic
*/
// Replace this code with whatever you want!
const ruleCopies = duplicateRuleParams(basicRule, 200);
const functions = ruleCopies.map((rule) => () => detectionsClient.createRule({ body: rule }));
await concurrentlyExec(functions);
listsClient.findLists({ query: {} });
exceptionsClient.findExceptionLists({ query: {} });
esClient.indices.exists({ index: 'test' });
/**
* END Custom data loader logic
*/
},
// Options
{
description: `Loads data into an environment for testing/development`,
flags: {
string: ['kibana', 'username', 'password', 'apikey'],
default: {
kibana: 'http://127.0.0.1:5601',
elasticsearch: 'http://127.0.0.1:9200',
username: 'elastic',
password: 'changeme',
},
allowUnexpected: false,
help: `
--username User name to be used for auth against elasticsearch and
kibana (Default: elastic).
--password User name Password (Default: changeme)
--kibana The url to Kibana (Default: http://127.0.0.1:5601)
--apikey The API key for authentication, overrides username/password
`,
},
}
);
};

View file

@ -216,6 +216,8 @@
"@kbn/esql-ast",
"@kbn/esql-validation-autocomplete",
"@kbn/config",
"@kbn/openapi-common",
"@kbn/securitysolution-lists-common",
"@kbn/cbor",
"@kbn/zod",
"@kbn/cloud-security-posture",