github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Detection Engine] move lists to data stream (#162508)

Browse source 
## Summary

- addresses https://github.com/elastic/security-team/issues/7198
- moves list/items indices to data stream
  - adds `@timestamp` mapping to indices mappings
- migrate to data stream if indices already exist(for customers < 8.11)
or create data stream(for customers 8.11+ or serverless)
- adds
[DLM](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-streams-put-lifecycle.html)
to index templates
- replaces update/delete queries with update_by_query/delete_by_query
which supported in data streams
  - fixes existing issues with update/patch APIs for lists/items
    - update/patch for lists didn't save `version` parameter in ES
- update and patch APIs for lists/items were identical, i.e. for both
routes was called the same `update` method w/o any changes

<details>

<summary>Technical detail on moving API to
(update/delete)_by_query</summary>


`update_by_query`, `delete_by_query` do not support refresh=wait_for,
[only false/true
values](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-update-by-query.html#_refreshing_shards_2).
Which might break some of the use cases on UI(when list is removed, we
refetch all lists. Deleted list will be returned for some time. [Default
refresh time is
1s](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-refresh.html)).
So, we retry refetching deleted/updated document before finishing
request, to return reindexed document

`update_by_query` does not support OCC [as update
API](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/optimistic-concurrency-control.html).
Which is supported in both
[list](https://www.elastic.co/guide/en/security/current/lists-api-update-container.html)/[list
item
](https://www.elastic.co/guide/en/security/current/lists-api-update-item.html)updates
through _version parameter.
_version is base64 encoded "_seq_no", "_primary_term" props used for OCC

So, to keep it without breaking changes: implemented check for version
conflict within update method
</details>

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Vitalii Dmyterko 2023-08-23 19:42:57 +01:00 • committed by GitHub
parent 154ca404d0
commit 505d8265c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
104 changed files with 2607 additions and 767 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx
View file

@ -211,6 +211,7 @@ describe('AutocompleteFieldListsComponent', () => {
await waitFor(() => {
expect(mockOnChange).toHaveBeenCalledWith({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: 'some user',

1
packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts
View file

@ -42,6 +42,7 @@ export const NAME = 'some name';
// TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715
// import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
export const getListResponseMock = (): ListSchema => ({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: USER,

6
packages/kbn-securitysolution-es-utils/index.ts
View file

@ -8,8 +8,10 @@
export * from './src/bad_request_error';
export * from './src/create_boostrap_index';
export * from './src/create_data_stream';
export * from './src/decode_version';
export * from './src/delete_all_index';
export * from './src/delete_data_stream';
export * from './src/delete_index_template';
export * from './src/delete_policy';
export * from './src/delete_template';
@ -18,11 +20,15 @@ export * from './src/get_bootstrap_index_exists';
export * from './src/get_index_aliases';
export * from './src/get_index_count';
export * from './src/get_index_exists';
export * from './src/get_data_stream_exists';
export * from './src/get_index_template_exists';
export * from './src/get_policy_exists';
export * from './src/get_template_exists';
export * from './src/migrate_to_data_stream';
export * from './src/read_index';
export * from './src/read_privileges';
export * from './src/put_mappings';
export * from './src/remove_policy_from_index';
export * from './src/set_index_template';
export * from './src/set_policy';
export * from './src/set_template';

23
packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
/**
* creates data stream
* @param esClient
* @param name
*/
export const createDataStream = async (
esClient: ElasticsearchClient,
name: string
): Promise<unknown> => {
return esClient.indices.createDataStream({
name,
});
};

28
packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
/**
* deletes data stream
* @param esClient
* @param name
*/
export const deleteDataStream = async (
esClient: ElasticsearchClient,
name: string
): Promise<boolean> => {
return (
await esClient.indices.deleteDataStream(
{
name,
},
{ meta: true }
)
).body.acknowledged;
};

38
packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
/**
* checks if data stream exists
* @param esClient
* @param name
*/
export const getDataStreamExists = async (
esClient: ElasticsearchClient,
name: string
): Promise<boolean> => {
try {
const body = await esClient.indices.getDataStream({ name, expand_wildcards: 'all' });
return body.data_streams.length > 0;
} catch (err) {
if (err.body != null && err.body.status === 404) {
return false;
} else if (
// if index already created, _data_stream/${name} request will produce the following error
// data stream does not exist at this point, so we can return false
err?.body?.error?.reason?.includes(
`The provided expression [${name}] matches an alias, specify the corresponding concrete indices instead.`
)
) {
return false;
} else {
throw err.body ? err.body : err;
}
}
};

23
packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
/**
* migrate to data stream
* @param esClient
* @param name
*/
export const migrateToDataStream = async (
esClient: ElasticsearchClient,
name: string
): Promise<unknown> => {
return esClient.indices.migrateToDataStream({
name,
});
};

26
packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import type { ElasticsearchClient } from '../elasticsearch_client';
/**
* update mappings of index
* @param esClient
* @param index
* @param mappings
*/
export const putMappings = async (
esClient: ElasticsearchClient,
index: string,
mappings: Record<string, MappingProperty>
): Promise<unknown> => {
return await esClient.indices.putMapping({
index,
properties: mappings,
});
};

16
packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
export const removePolicyFromIndex = async (
esClient: ElasticsearchClient,
index: string
): Promise<unknown> => {
return (await esClient.ilm.removePolicy({ index }, { meta: true })).body;
};

1
packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts
View file

@ -56,6 +56,7 @@ export * from './sort_field';
export * from './sort_order';
export * from './tags';
export * from './tie_breaker_id';
export * from './timestamp';
export * from './total';
export * from './type';
export * from './underscore_version';

3
packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts
View file

@ -12,3 +12,6 @@ export const meta = t.object;
export type Meta = t.TypeOf<typeof meta>;
export const metaOrUndefined = t.union([meta, t.undefined]);
export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
export const nullableMetaOrUndefined = t.union([metaOrUndefined, t.null]);
export type NullableMetaOrUndefined = t.TypeOf<typeof nullableMetaOrUndefined>;

13
packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import * as t from 'io-ts';
import { IsoDateString } from '@kbn/securitysolution-io-ts-types';
export const timestamp = IsoDateString;
export const timestampOrUndefined = t.union([IsoDateString, t.undefined]);

1
packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts
View file

@ -21,6 +21,7 @@ import {
export const getListItemResponseMock = (): ListItemSchema => ({
_version: undefined,
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
deserializer: undefined,

2
packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts
View file

@ -11,6 +11,7 @@ import * as t from 'io-ts';
import { _versionOrUndefined } from '../../common/underscore_version';
import { deserializerOrUndefined } from '../../common/deserializer';
import { metaOrUndefined } from '../../common/meta';
import { timestampOrUndefined } from '../../common/timestamp';
import { serializerOrUndefined } from '../../common/serializer';
import { created_at } from '../../common/created_at';
import { created_by } from '../../common/created_by';
@ -25,6 +26,7 @@ import { value } from '../../common/value';
export const listItemSchema = t.exact(
t.type({
_version: _versionOrUndefined,
'@timestamp': timestampOrUndefined,
created_at,
created_by,
deserializer: deserializerOrUndefined,

1
packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts
View file

@ -23,6 +23,7 @@ import {
export const getListResponseMock = (): ListSchema => ({
_version: undefined,
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
description: DESCRIPTION,

2
packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts
View file

@ -13,6 +13,7 @@ import { deserializerOrUndefined } from '../../common/deserializer';
import { metaOrUndefined } from '../../common/meta';
import { serializerOrUndefined } from '../../common/serializer';
import { created_at } from '../../common/created_at';
import { timestampOrUndefined } from '../../common/timestamp';
import { created_by } from '../../common/created_by';
import { description } from '../../common/description';
import { id } from '../../common/id';
@ -26,6 +27,7 @@ import { updated_by } from '../../common/updated_by';
export const listSchema = t.exact(
t.type({
_version: _versionOrUndefined,
'@timestamp': timestampOrUndefined,
created_at,
created_by,
description,

6
packages/kbn-securitysolution-list-api/src/api/index.ts
View file

@ -37,7 +37,7 @@ import {
import {
ENDPOINT_LIST_URL,
EXCEPTION_FILTER,
INTERNAL_EXCEPTION_FILTER,
EXCEPTION_LIST_ITEM_URL,
EXCEPTION_LIST_URL,
} from '@kbn/securitysolution-list-constants';
@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({
http,
signal,
}: GetExceptionFilterFromExceptionListIdsProps): Promise<ExceptionFilterResponse> =>
http.fetch(EXCEPTION_FILTER, {
http.fetch(INTERNAL_EXCEPTION_FILTER, {
method: 'POST',
body: JSON.stringify({
exception_list_ids: exceptionListIds,
@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({
chunkSize,
signal,
}: GetExceptionFilterFromExceptionsProps): Promise<ExceptionFilterResponse> =>
http.fetch(EXCEPTION_FILTER, {
http.fetch(INTERNAL_EXCEPTION_FILTER, {
method: 'POST',
body: JSON.stringify({
exceptions,

4
packages/kbn-securitysolution-list-api/src/list_api/index.ts
View file

@ -37,7 +37,7 @@ import {
LIST_ITEM_URL,
LIST_PRIVILEGES_URL,
LIST_URL,
FIND_LISTS_BY_SIZE,
INTERNAL_FIND_LISTS_BY_SIZE,
} from '@kbn/securitysolution-list-constants';
import { toError, toPromise } from '../fp_utils';
@ -115,7 +115,7 @@ const findListsBySize = async ({
per_page,
signal,
}: ApiParams & FindListSchemaEncoded): Promise<FoundListsBySizeSchema> => {
return http.fetch(`${FIND_LISTS_BY_SIZE}`, {
return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, {
method: 'GET',
query: {
cursor,

1
packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts
View file

@ -23,6 +23,7 @@ import {
} from '../constants.mock';
export const getListResponseMock = (): ListSchema => ({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: USER,

4
packages/kbn-securitysolution-list-constants/index.ts
View file

@ -20,8 +20,8 @@ export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`;
* Internal value list routes
*/
export const INTERNAL_LIST_URL = '/internal/lists';
export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
export const INTERNAL_FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
export const INTERNAL_EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
/**
* Exception list routes

1
packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts
View file

@ -23,6 +23,7 @@ import {
} from '../constants.mock';
export const getListResponseMock = (): ListSchema => ({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: USER,

1
x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts
View file

@ -20,6 +20,7 @@ import {
} from '../../constants.mock';
export const getListItemResponseMock = (): ListItemSchema => ({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: USER,

1
x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts
View file

@ -22,6 +22,7 @@ import {
} from '../../constants.mock';
export const getListResponseMock = (): ListSchema => ({
'@timestamp': DATE_NOW,
_version: undefined,
created_at: DATE_NOW,
created_by: USER,

102
x-pack/plugins/lists/server/routes/create_list_index_route.ts
View file

@ -1,102 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { createListIndexResponse } from '../../common/api';
import type { ListsPluginRouter } from '../types';
import { buildSiemResponse } from './utils';
import { getListClient } from '.';
export const createListIndexRoute = (router: ListsPluginRouter): void => {
router.post(
{
options: {
tags: ['access:lists-all'],
},
path: LIST_INDEX,
validate: false,
},
async (context, _, response) => {
const siemResponse = buildSiemResponse(response);
try {
const lists = await getListClient(context);
const listIndexExists = await lists.getListIndexExists();
const listItemIndexExists = await lists.getListItemIndexExists();
const policyExists = await lists.getListPolicyExists();
const policyListItemExists = await lists.getListItemPolicyExists();
if (!policyExists) {
await lists.setListPolicy();
}
if (!policyListItemExists) {
await lists.setListItemPolicy();
}
const templateExists = await lists.getListTemplateExists();
const templateListItemsExists = await lists.getListItemTemplateExists();
const legacyTemplateExists = await lists.getLegacyListTemplateExists();
const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists();
if (!templateExists) {
await lists.setListTemplate();
}
if (!templateListItemsExists) {
await lists.setListItemTemplate();
}
try {
// Check if the old legacy lists and items template exists and remove it
if (legacyTemplateExists) {
await lists.deleteLegacyListTemplate();
}
if (legacyTemplateListItemsExists) {
await lists.deleteLegacyListItemTemplate();
}
} catch (err) {
if (err.statusCode !== 404) {
throw err;
}
}
if (listIndexExists && listItemIndexExists) {
return siemResponse.error({
body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`,
statusCode: 409,
});
} else {
if (!listIndexExists) {
await lists.createListBootStrapIndex();
}
if (!listItemIndexExists) {
await lists.createListItemBootStrapIndex();
}
const [validated, errors] = validate({ acknowledged: true }, createListIndexResponse);
if (errors != null) {
return siemResponse.error({ body: errors, statusCode: 500 });
} else {
return response.ok({ body: validated ?? {} });
}
}
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
);
};

112
x-pack/plugins/lists/server/routes/delete_list_index_route.ts
View file

@ -1,112 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { deleteListIndexResponse } from '../../common/api';
import { buildSiemResponse } from './utils';
import { getListClient } from '.';
/**
* Deletes all of the indexes, template, ilm policies, and aliases. You can check
* this by looking at each of these settings from ES after a deletion:
*
* GET /_template/.lists-default
* GET /.lists-default-000001/
* GET /_ilm/policy/.lists-default
* GET /_alias/.lists-default
*
* GET /_template/.items-default
* GET /.items-default-000001/
* GET /_ilm/policy/.items-default
* GET /_alias/.items-default
*
* And ensuring they're all gone
*/
export const deleteListIndexRoute = (router: ListsPluginRouter): void => {
router.delete(
{
options: {
tags: ['access:lists-all'],
},
path: LIST_INDEX,
validate: false,
},
async (context, _, response) => {
const siemResponse = buildSiemResponse(response);
try {
const lists = await getListClient(context);
const listIndexExists = await lists.getListIndexExists();
const listItemIndexExists = await lists.getListItemIndexExists();
if (!listIndexExists && !listItemIndexExists) {
return siemResponse.error({
body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`,
statusCode: 404,
});
} else {
if (listIndexExists) {
await lists.deleteListIndex();
}
if (listItemIndexExists) {
await lists.deleteListItemIndex();
}
const listsPolicyExists = await lists.getListPolicyExists();
const listItemPolicyExists = await lists.getListItemPolicyExists();
if (listsPolicyExists) {
await lists.deleteListPolicy();
}
if (listItemPolicyExists) {
await lists.deleteListItemPolicy();
}
const listsTemplateExists = await lists.getListTemplateExists();
const listItemTemplateExists = await lists.getListItemTemplateExists();
if (listsTemplateExists) {
await lists.deleteListTemplate();
}
if (listItemTemplateExists) {
await lists.deleteListItemTemplate();
}
// check if legacy template exists
const legacyTemplateExists = await lists.getLegacyListTemplateExists();
const legacyItemTemplateExists = await lists.getLegacyListItemTemplateExists();
if (legacyTemplateExists) {
await lists.deleteLegacyListTemplate();
}
if (legacyItemTemplateExists) {
await lists.deleteLegacyListItemTemplate();
}
const [validated, errors] = validate({ acknowledged: true }, deleteListIndexResponse);
if (errors != null) {
return siemResponse.error({ body: errors, statusCode: 500 });
} else {
return response.ok({ body: validated ?? {} });
}
}
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
);
};

40
x-pack/plugins/lists/server/routes/index.ts
View file

@ -9,43 +9,43 @@ export * from './create_endpoint_list_item_route';
export * from './create_endpoint_list_route';
export * from './create_exception_list_item_route';
export * from './create_exception_list_route';
export * from './create_list_index_route';
export * from './create_list_item_route';
export * from './create_list_route';
export * from './list_index/create_list_index_route';
export * from './list_item/create_list_item_route';
export * from './list/create_list_route';
export * from './delete_endpoint_list_item_route';
export * from './delete_exception_list_route';
export * from './delete_exception_list_item_route';
export * from './delete_list_index_route';
export * from './delete_list_item_route';
export * from './delete_list_route';
export * from './list_index/delete_list_index_route';
export * from './list_item/delete_list_item_route';
export * from './list/delete_list_route';
export * from './duplicate_exception_list_route';
export * from './export_exception_list_route';
export * from './export_list_item_route';
export * from './list_index/export_list_item_route';
export * from './find_endpoint_list_item_route';
export * from './find_exception_list_item_route';
export * from './find_exception_list_route';
export * from './find_list_item_route';
export * from './find_list_route';
export * from './find_lists_by_size_route';
export * from './get_exception_filter_route';
export * from './list_item/find_list_item_route';
export * from './list_index/find_list_route';
export * from './internal/find_lists_by_size_route';
export * from './internal/create_exception_filter_route';
export * from './import_exceptions_route';
export * from './import_list_item_route';
export * from './list/import_list_item_route';
export * from './init_routes';
export * from './patch_list_item_route';
export * from './patch_list_route';
export * from './list_item/patch_list_item_route';
export * from './list/patch_list_route';
export * from './read_endpoint_list_item_route';
export * from './read_exception_list_item_route';
export * from './read_exception_list_route';
export * from './read_list_index_route';
export * from './read_list_item_route';
export * from './read_list_route';
export * from './read_privileges_route';
export * from './list_index/read_list_index_route';
export * from './list_item/read_list_item_route';
export * from './list/read_list_route';
export * from './list_privileges/read_list_privileges_route';
export * from './summary_exception_list_route';
export * from './update_endpoint_list_item_route';
export * from './update_exception_list_item_route';
export * from './update_exception_list_route';
export * from './update_list_item_route';
export * from './update_list_route';
export * from './list_item/update_list_item_route';
export * from './list/update_list_route';
export * from './utils';
// internal

13
x-pack/plugins/lists/server/routes/get_exception_filter_route.ts → x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts
View file

@ -11,13 +11,12 @@ import {
ExceptionListItemSchema,
FoundExceptionListItemSchema,
} from '@kbn/securitysolution-io-ts-list-types';
import { EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants';
import { INTERNAL_EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants';
import { buildExceptionFilter } from '../services/exception_lists/build_exception_filter';
import { ListsPluginRouter } from '../types';
import { getExceptionFilterRequest } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { buildExceptionFilter } from '../../services/exception_lists/build_exception_filter';
import { ListsPluginRouter } from '../../types';
import { getExceptionFilterRequest } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
export const getExceptionFilterRoute = (router: ListsPluginRouter): void => {
router.post(
@ -25,7 +24,7 @@ export const getExceptionFilterRoute = (router: ListsPluginRouter): void => {
options: {
tags: ['access:securitySolution'],
},
path: `${EXCEPTION_FILTER}`,
path: INTERNAL_EXCEPTION_FILTER,
validate: {
body: buildRouteValidation(getExceptionFilterRequest),
},

13
x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts → x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts
View file

@ -8,17 +8,16 @@
import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import {
FIND_LISTS_BY_SIZE,
INTERNAL_FIND_LISTS_BY_SIZE,
MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE,
MAXIMUM_SMALL_VALUE_LIST_SIZE,
} from '@kbn/securitysolution-list-constants';
import { chunk } from 'lodash';
import type { ListsPluginRouter } from '../types';
import { decodeCursor } from '../services/utils';
import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from './utils';
import type { ListsPluginRouter } from '../../types';
import { decodeCursor } from '../../services/utils';
import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils';
export const findListsBySizeRoute = (router: ListsPluginRouter): void => {
router.get(
@ -26,7 +25,7 @@ export const findListsBySizeRoute = (router: ListsPluginRouter): void => {
options: {
tags: ['access:lists-read'],
},
path: `${FIND_LISTS_BY_SIZE}`,
path: INTERNAL_FIND_LISTS_BY_SIZE,
validate: {
query: buildRouteValidation(findListsBySizeRequestQuery),
},

26
x-pack/plugins/lists/server/routes/create_list_route.ts → x-pack/plugins/lists/server/routes/list/create_list_route.ts
View file

@ -9,12 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { CreateListRequestDecoded, createListRequest, createListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import {
CreateListRequestDecoded,
createListRequest,
createListResponse,
} from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const createListRoute = (router: ListsPluginRouter): void => {
router.post(
@ -35,13 +37,19 @@ export const createListRoute = (router: ListsPluginRouter): void => {
const { name, description, deserializer, id, serializer, type, meta, version } =
request.body;
const lists = await getListClient(context);
const listExists = await lists.getListIndexExists();
if (!listExists) {
const dataStreamExists = await lists.getListDataStreamExists();
const indexExists = await lists.getListIndexExists();
if (!dataStreamExists && !indexExists) {
return siemResponse.error({
body: `To create a list, the index must exist first. Index "${lists.getListIndex()}" does not exist`,
body: `To create a list, the data stream must exist first. Data stream "${lists.getListName()}" does not exist`,
statusCode: 400,
});
} else {
// needs to be migrated to data stream
if (!dataStreamExists && indexExists) {
await lists.migrateListIndexToDataStream();
}
if (id != null) {
const list = await lists.getList({ id });
if (list != null) {

14
x-pack/plugins/lists/server/routes/delete_list_route.ts → x-pack/plugins/lists/server/routes/list/delete_list_route.ts
View file

@ -18,14 +18,12 @@ import {
import { getSavedObjectType } from '@kbn/securitysolution-list-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import type { ExceptionListClient } from '../services/exception_lists/exception_list_client';
import { escapeQuotes } from '../services/utils/escape_query';
import { deleteListRequestQuery, deleteListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getExceptionListClient, getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import type { ExceptionListClient } from '../../services/exception_lists/exception_list_client';
import { escapeQuotes } from '../../services/utils/escape_query';
import { deleteListRequestQuery, deleteListResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getExceptionListClient, getListClient } from '..';
export const deleteListRoute = (router: ListsPluginRouter): void => {
router.delete(

46
x-pack/plugins/lists/server/routes/import_list_item_route.ts → x-pack/plugins/lists/server/routes/list/import_list_item_route.ts
View file

@ -10,14 +10,12 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { ConfigType } from '../config';
import { importListItemRequestQuery, importListItemResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { createStreamFromBuffer } from './utils/create_stream_from_buffer';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { ConfigType } from '../../config';
import { importListItemRequestQuery, importListItemResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { createStreamFromBuffer } from '../utils/create_stream_from_buffer';
import { getListClient } from '..';
export const importListItemRoute = (router: ListsPluginRouter, config: ConfigType): void => {
router.post(
@ -45,13 +43,33 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp
const stream = createStreamFromBuffer(request.body);
const { deserializer, list_id: listId, serializer, type } = request.query;
const lists = await getListClient(context);
const listExists = await lists.getListIndexExists();
if (!listExists) {
return siemResponse.error({
body: `To import a list item, the index must exist first. Index "${lists.getListIndex()}" does not exist`,
statusCode: 400,
});
const listDataExists = await lists.getListDataStreamExists();
if (!listDataExists) {
const listIndexExists = await lists.getListIndexExists();
if (!listIndexExists) {
return siemResponse.error({
body: `To import a list item, the data steam must exist first. Data stream "${lists.getListName()}" does not exist`,
statusCode: 400,
});
}
// otherwise migration is needed
await lists.migrateListIndexToDataStream();
}
const listItemDataExists = await lists.getListItemDataStreamExists();
if (!listItemDataExists) {
const listItemIndexExists = await lists.getListItemIndexExists();
if (!listItemIndexExists) {
return siemResponse.error({
body: `To import a list item, the data steam must exist first. Data stream "${lists.getListItemName()}" does not exist`,
statusCode: 400,
});
}
// otherwise migration is needed
await lists.migrateListItemIndexToDataStream();
}
if (listId != null) {
const list = await lists.getList({ id: listId });
if (list == null) {

22
x-pack/plugins/lists/server/routes/patch_list_route.ts → x-pack/plugins/lists/server/routes/list/patch_list_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { patchListRequest, patchListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { patchListRequest, patchListResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const patchListRoute = (router: ListsPluginRouter): void => {
router.patch(
@ -32,7 +30,17 @@ export const patchListRoute = (router: ListsPluginRouter): void => {
try {
const { name, description, id, meta, _version, version } = request.body;
const lists = await getListClient(context);
const list = await lists.updateList({ _version, description, id, meta, name, version });
const dataStreamExists = await lists.getListDataStreamExists();
// needs to be migrated to data stream if index exists
if (!dataStreamExists) {
const indexExists = await lists.getListIndexExists();
if (indexExists) {
await lists.migrateListIndexToDataStream();
}
}
const list = await lists.patchList({ _version, description, id, meta, name, version });
if (list == null) {
return siemResponse.error({
body: `list id: "${id}" not found`,

10
x-pack/plugins/lists/server/routes/read_list_route.ts → x-pack/plugins/lists/server/routes/list/read_list_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { readListRequestQuery, readListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { readListRequestQuery, readListResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const readListRoute = (router: ListsPluginRouter): void => {
router.get(

20
x-pack/plugins/lists/server/routes/update_list_route.ts → x-pack/plugins/lists/server/routes/list/update_list_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { updateListRequest, updateListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { updateListRequest, updateListResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const updateListRoute = (router: ListsPluginRouter): void => {
router.put(
@ -32,6 +30,16 @@ export const updateListRoute = (router: ListsPluginRouter): void => {
try {
const { name, description, id, meta, _version, version } = request.body;
const lists = await getListClient(context);
const dataStreamExists = await lists.getListDataStreamExists();
// needs to be migrated to data stream if index exists
if (!dataStreamExists) {
const indexExists = await lists.getListIndexExists();
if (indexExists) {
await lists.migrateListIndexToDataStream();
}
}
const list = await lists.updateList({ _version, description, id, meta, name, version });
if (list == null) {
return siemResponse.error({

84
x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts Normal file
View file

@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { createListIndexResponse } from '../../../common/api';
import type { ListsPluginRouter } from '../../types';
import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils';
import { getListClient } from '..';
export const createListIndexRoute = (router: ListsPluginRouter): void => {
router.post(
{
options: {
tags: ['access:lists-all'],
},
path: LIST_INDEX,
validate: false,
},
async (context, _, response) => {
const siemResponse = buildSiemResponse(response);
try {
const lists = await getListClient(context);
const listDataStreamExists = await lists.getListDataStreamExists();
const listItemDataStreamExists = await lists.getListItemDataStreamExists();
const templateListExists = await lists.getListTemplateExists();
const templateListItemsExists = await lists.getListItemTemplateExists();
if (!templateListExists || !listDataStreamExists) {
await lists.setListTemplate();
}
if (!templateListItemsExists || !listItemDataStreamExists) {
await lists.setListItemTemplate();
}
await removeLegacyTemplatesIfExist(lists);
if (listDataStreamExists && listItemDataStreamExists) {
return siemResponse.error({
body: `data stream: "${lists.getListName()}" and "${lists.getListItemName()}" already exists`,
statusCode: 409,
});
}
if (!listDataStreamExists) {
const listIndexExists = await lists.getListIndexExists();
await (listIndexExists
? lists.migrateListIndexToDataStream()
: lists.createListDataStream());
}
if (!listItemDataStreamExists) {
const listItemIndexExists = await lists.getListItemIndexExists();
await (listItemIndexExists
? lists.migrateListItemIndexToDataStream()
: lists.createListItemDataStream());
}
const [validated, errors] = validate({ acknowledged: true }, createListIndexResponse);
if (errors != null) {
return siemResponse.error({ body: errors, statusCode: 500 });
} else {
return response.ok({ body: validated ?? {} });
}
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
);
};

147
x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts Normal file
View file

@ -0,0 +1,147 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { ListClient } from '../../services/lists/list_client';
import type { ListsPluginRouter } from '../../types';
import { deleteListIndexResponse } from '../../../common/api';
import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils';
import { getListClient } from '..';
/**
* Deletes all of the indexes, template, ilm policies, and aliases. You can check
* this by looking at each of these settings from ES after a deletion:
*
* GET /_template/.lists-default
* GET /.lists-default-000001/
* GET /_ilm/policy/.lists-default
* GET /_alias/.lists-default
*
* GET /_template/.items-default
* GET /.items-default-000001/
* GET /_ilm/policy/.items-default
* GET /_alias/.items-default
*
* And ensuring they're all gone
*/
export const deleteListIndexRoute = (router: ListsPluginRouter): void => {
router.delete(
{
options: {
tags: ['access:lists-all'],
},
path: LIST_INDEX,
validate: false,
},
async (context, _, response) => {
const siemResponse = buildSiemResponse(response);
try {
const lists = await getListClient(context);
const listIndexExists = await lists.getListIndexExists();
const listItemIndexExists = await lists.getListItemIndexExists();
const listDataStreamExists = await lists.getListDataStreamExists();
const listItemDataStreamExists = await lists.getListItemDataStreamExists();
// return early if no data stream or indices exist
if (
!listDataStreamExists &&
!listItemDataStreamExists &&
!listIndexExists &&
!listItemIndexExists
) {
return siemResponse.error({
body: `index and data stream: "${lists.getListName()}" and "${lists.getListItemName()}" does not exist`,
statusCode: 404,
});
}
// ensure data streams deleted if exist
await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists);
// ensure indices deleted if exist and were not migrated
await deleteIndices(lists, listIndexExists, listItemIndexExists);
await deleteIndexTemplates(lists);
await removeLegacyTemplatesIfExist(lists);
const [validated, errors] = validate({ acknowledged: true }, deleteListIndexResponse);
if (errors != null) {
return siemResponse.error({ body: errors, statusCode: 500 });
} else {
return response.ok({ body: validated ?? {} });
}
} catch (err) {
const error = transformError(err);
return siemResponse.error({
body: error.message,
statusCode: error.statusCode,
});
}
}
);
};
/**
* Delete list/item indices
*/
const deleteIndices = async (
lists: ListClient,
listIndexExists: boolean,
listItemIndexExists: boolean
): Promise<void> => {
if (listIndexExists) {
await lists.deleteListIndex();
}
if (listItemIndexExists) {
await lists.deleteListItemIndex();
}
const listsPolicyExists = await lists.getListPolicyExists();
const listItemPolicyExists = await lists.getListItemPolicyExists();
if (listsPolicyExists) {
await lists.deleteListPolicy();
}
if (listItemPolicyExists) {
await lists.deleteListItemPolicy();
}
};
/**
* Delete list/item data streams
*/
const deleteDataStreams = async (
lists: ListClient,
listDataStreamExists: boolean,
listItemDataStreamExists: boolean
): Promise<void> => {
if (listDataStreamExists) {
await lists.deleteListDataStream();
}
if (listItemDataStreamExists) {
await lists.deleteListItemDataStream();
}
};
/**
* Delete list/item index templates
*/
const deleteIndexTemplates = async (lists: ListClient): Promise<void> => {
const listsTemplateExists = await lists.getListTemplateExists();
const listItemTemplateExists = await lists.getListItemTemplateExists();
if (listsTemplateExists) {
await lists.deleteListTemplate();
}
if (listItemTemplateExists) {
await lists.deleteListItemTemplate();
}
};

10
x-pack/plugins/lists/server/routes/export_list_item_route.ts → x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts
View file

@ -10,12 +10,10 @@ import { Stream } from 'stream';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { exportListItemRequestQuery } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { exportListItemRequestQuery } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const exportListItemRoute = (router: ListsPluginRouter): void => {
router.post(

9
x-pack/plugins/lists/server/routes/find_list_route.ts → x-pack/plugins/lists/server/routes/list_index/find_list_route.ts
View file

@ -9,11 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { decodeCursor } from '../services/utils';
import { findListRequestQuery, findListResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from './utils';
import type { ListsPluginRouter } from '../../types';
import { decodeCursor } from '../../services/utils';
import { findListRequestQuery, findListResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils';
export const findListRoute = (router: ListsPluginRouter): void => {
router.get(

28
x-pack/plugins/lists/server/routes/read_list_index_route.ts → x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { readListIndexResponse } from '../../common/api';
import { buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { readListIndexResponse } from '../../../common/api';
import { buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const readListIndexRoute = (router: ListsPluginRouter): void => {
router.get(
@ -30,12 +28,12 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => {
try {
const lists = await getListClient(context);
const listIndexExists = await lists.getListIndexExists();
const listItemIndexExists = await lists.getListItemIndexExists();
const listDataStreamExists = await lists.getListDataStreamExists();
const listItemDataStreamExists = await lists.getListItemDataStreamExists();
if (listIndexExists || listItemIndexExists) {
if (listDataStreamExists && listItemDataStreamExists) {
const [validated, errors] = validate(
{ list_index: listIndexExists, list_item_index: listItemIndexExists },
{ list_index: listDataStreamExists, list_item_index: listItemDataStreamExists },
readListIndexResponse
);
if (errors != null) {
@ -43,19 +41,19 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => {
} else {
return response.ok({ body: validated ?? {} });
}
} else if (!listIndexExists && listItemIndexExists) {
} else if (!listDataStreamExists && listItemDataStreamExists) {
return siemResponse.error({
body: `index ${lists.getListIndex()} does not exist`,
body: `data stream ${lists.getListName()} does not exist`,
statusCode: 404,
});
} else if (!listItemIndexExists && listIndexExists) {
} else if (!listItemDataStreamExists && listDataStreamExists) {
return siemResponse.error({
body: `index ${lists.getListItemIndex()} does not exist`,
body: `data stream ${lists.getListItemName()} does not exist`,
statusCode: 404,
});
} else {
return siemResponse.error({
body: `index ${lists.getListIndex()} and index ${lists.getListItemIndex()} does not exist`,
body: `data stream ${lists.getListName()} and data stream ${lists.getListItemName()} does not exist`,
statusCode: 404,
});
}

10
x-pack/plugins/lists/server/routes/create_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import { createListItemRequest, createListItemResponse } from '../../common/api';
import type { ListsPluginRouter } from '../types';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import { createListItemRequest, createListItemResponse } from '../../../common/api';
import type { ListsPluginRouter } from '../../types';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const createListItemRoute = (router: ListsPluginRouter): void => {
router.post(

10
x-pack/plugins/lists/server/routes/delete_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts
View file

@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import type { ListsPluginRouter } from '../../types';
import {
deleteListItemArrayResponse,
deleteListItemRequestQuery,
deleteListItemResponse,
} from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
} from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const deleteListItemRoute = (router: ListsPluginRouter): void => {
router.delete(

9
x-pack/plugins/lists/server/routes/find_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts
View file

@ -9,15 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { decodeCursor } from '../services/utils';
import type { ListsPluginRouter } from '../../types';
import { decodeCursor } from '../../services/utils';
import {
FindListItemRequestQueryDecoded,
findListItemRequestQuery,
findListItemResponse,
} from '../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from './utils';
} from '../../../common/api';
import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils';
export const findListItemRoute = (router: ListsPluginRouter): void => {
router.get(

22
x-pack/plugins/lists/server/routes/patch_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { patchListItemRequest, patchListItemResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { patchListItemRequest, patchListItemResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const patchListItemRoute = (router: ListsPluginRouter): void => {
router.patch(
@ -32,7 +30,17 @@ export const patchListItemRoute = (router: ListsPluginRouter): void => {
try {
const { value, id, meta, _version } = request.body;
const lists = await getListClient(context);
const listItem = await lists.updateListItem({
const dataStreamExists = await lists.getListItemDataStreamExists();
// needs to be migrated to data stream if index exists
if (!dataStreamExists) {
const indexExists = await lists.getListItemIndexExists();
if (indexExists) {
await lists.migrateListItemIndexToDataStream();
}
}
const listItem = await lists.patchListItem({
_version,
id,
meta,

10
x-pack/plugins/lists/server/routes/read_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts
View file

@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import type { ListsPluginRouter } from '../../types';
import {
readListItemArrayResponse,
readListItemRequestQuery,
readListItemResponse,
} from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
} from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const readListItemRoute = (router: ListsPluginRouter): void => {
router.get(

20
x-pack/plugins/lists/server/routes/update_list_item_route.ts → x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts
View file

@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils';
import { transformError } from '@kbn/securitysolution-es-utils';
import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { updateListItemRequest, updateListItemResponse } from '../../common/api';
import { buildRouteValidation, buildSiemResponse } from './utils';
import { getListClient } from '.';
import type { ListsPluginRouter } from '../../types';
import { updateListItemRequest, updateListItemResponse } from '../../../common/api';
import { buildRouteValidation, buildSiemResponse } from '../utils';
import { getListClient } from '..';
export const updateListItemRoute = (router: ListsPluginRouter): void => {
router.put(
@ -32,6 +30,16 @@ export const updateListItemRoute = (router: ListsPluginRouter): void => {
try {
const { value, id, meta, _version } = request.body;
const lists = await getListClient(context);
const dataStreamExists = await lists.getListItemDataStreamExists();
// needs to be migrated to data stream if index exists
if (!dataStreamExists) {
const indexExists = await lists.getListItemIndexExists();
if (indexExists) {
await lists.migrateListItemIndexToDataStream();
}
}
const listItem = await lists.updateListItem({
_version,
id,

0
x-pack/plugins/lists/server/routes/read_privileges_route.mock.ts → x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.mock.ts
View file

9
x-pack/plugins/lists/server/routes/read_privileges_route.ts → x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts
View file

@ -9,9 +9,8 @@ import { readPrivileges, transformError } from '@kbn/securitysolution-es-utils';
import { merge } from 'lodash/fp';
import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants';
import type { ListsPluginRouter } from '../types';
import { buildSiemResponse, getListClient } from './utils';
import type { ListsPluginRouter } from '../../types';
import { buildSiemResponse, getListClient } from '../utils';
export const readPrivilegesRoute = (router: ListsPluginRouter): void => {
router.get(
@ -27,8 +26,8 @@ export const readPrivilegesRoute = (router: ListsPluginRouter): void => {
try {
const esClient = (await context.core).elasticsearch.client.asCurrentUser;
const lists = await getListClient(context);
const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListIndex());
const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemIndex());
const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListName());
const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemName());
const privileges = merge(
{
listItems: clusterPrivilegesListItems,

1
x-pack/plugins/lists/server/routes/utils/index.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
export * from './remove_templates_if_exist';
export * from './get_error_message_exception_list_item';
export * from './get_error_message_exception_list';
export * from './get_list_client';

26
x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ListClient } from '../../services/lists/list_client';
export const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise<void> => {
const legacyTemplateExists = await lists.getLegacyListTemplateExists();
const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists();
try {
// Check if the old legacy lists and items template exists and remove it
if (legacyTemplateExists) {
await lists.deleteLegacyListTemplate();
}
if (legacyTemplateListItemsExists) {
await lists.deleteLegacyListItemTemplate();
}
} catch (err) {
if (err.statusCode !== 404) {
throw err;
}
}
};

1
x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts
View file

@ -10,6 +10,7 @@ import { DATE_NOW, LIST_ID, META, TIE_BREAKER, USER, VALUE } from '../../../comm
import { IndexEsListItemSchema } from './index_es_list_item_schema';
export const getIndexESListItemMock = (ip = VALUE): IndexEsListItemSchema => ({
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
deserializer: undefined,

2
x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts
View file

@ -14,6 +14,7 @@ import {
metaOrUndefined,
serializerOrUndefined,
tie_breaker_id,
timestamp,
updated_at,
updated_by,
} from '@kbn/securitysolution-io-ts-list-types';
@ -23,6 +24,7 @@ import { esDataTypeUnion } from '../common/schemas';
export const indexEsListItemSchema = t.intersection([
t.exact(
t.type({
'@timestamp': timestamp,
created_at,
created_by,
deserializer: deserializerOrUndefined,

1
x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts
View file

@ -20,6 +20,7 @@ import {
import { IndexEsListSchema } from './index_es_list_schema';
export const getIndexESListMock = (): IndexEsListSchema => ({
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
description: DESCRIPTION,

2
x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts
View file

@ -16,6 +16,7 @@ import {
name,
serializerOrUndefined,
tie_breaker_id,
timestamp,
type,
updated_at,
updated_by,
@ -24,6 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types';
export const indexEsListSchema = t.exact(
t.type({
'@timestamp': timestamp,
created_at,
created_by,
description,

2
x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts
View file

@ -6,6 +6,7 @@
*/
import * as t from 'io-ts';
import { version } from '@kbn/securitysolution-io-ts-types';
import {
descriptionOrUndefined,
metaOrUndefined,
@ -21,6 +22,7 @@ export const updateEsListSchema = t.exact(
name: nameOrUndefined,
updated_at,
updated_by,
version,
})
);

1
x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts
View file

@ -22,6 +22,7 @@ import { getShardMock } from '../common/get_shard.mock';
import { SearchEsListItemSchema } from './search_es_list_item_schema';
export const getSearchEsListItemsAsAllUndefinedMock = (): SearchEsListItemSchema => ({
'@timestamp': DATE_NOW,
binary: undefined,
boolean: undefined,
byte: undefined,

6
x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts
View file

@ -11,9 +11,10 @@ import {
created_by,
deserializerOrUndefined,
list_id,
metaOrUndefined,
nullableMetaOrUndefined,
serializerOrUndefined,
tie_breaker_id,
timestampOrUndefined,
updated_at,
updated_by,
} from '@kbn/securitysolution-io-ts-list-types';
@ -46,6 +47,7 @@ import {
export const searchEsListItemSchema = t.exact(
t.type({
'@timestamp': timestampOrUndefined,
binary: binaryOrUndefined,
boolean: booleanOrUndefined,
byte: byteOrUndefined,
@ -70,7 +72,7 @@ export const searchEsListItemSchema = t.exact(
list_id,
long: longOrUndefined,
long_range: longRangeOrUndefined,
meta: metaOrUndefined,
meta: nullableMetaOrUndefined,
serializer: serializerOrUndefined,
shape: shapeOrUndefined,
short: shortOrUndefined,

1
x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts
View file

@ -25,6 +25,7 @@ import { getShardMock } from '../common/get_shard.mock';
import { SearchEsListSchema } from './search_es_list_schema';
export const getSearchEsListMock = (): SearchEsListSchema => ({
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
description: DESCRIPTION,

6
x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts
View file

@ -12,10 +12,11 @@ import {
description,
deserializerOrUndefined,
immutable,
metaOrUndefined,
name,
nullableMetaOrUndefined,
serializerOrUndefined,
tie_breaker_id,
timestampOrUndefined,
type,
updated_at,
updated_by,
@ -24,12 +25,13 @@ import { version } from '@kbn/securitysolution-io-ts-types';
export const searchEsListSchema = t.exact(
t.type({
'@timestamp': timestampOrUndefined,
created_at,
created_by,
description,
deserializer: deserializerOrUndefined,
immutable,
meta: metaOrUndefined,
meta: nullableMetaOrUndefined,
name,
serializer: serializerOrUndefined,
tie_breaker_id,

8
x-pack/plugins/lists/server/services/items/create_list_item.test.ts
View file

@ -14,7 +14,7 @@ import { getIndexESListItemMock } from '../../schemas/elastic_query/index_es_lis
import { CreateListItemOptions, createListItem } from './create_list_item';
import { getCreateListItemOptionsMock } from './create_list_item.mock';
describe('crete_list_item', () => {
describe('create_list_item', () => {
beforeEach(() => {
jest.clearAllMocks();
});
@ -26,7 +26,7 @@ describe('crete_list_item', () => {
test('it returns a list item as expected with the id changed out for the elastic id', async () => {
const options = getCreateListItemOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.index.mockResponse(
esClient.create.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
@ -46,14 +46,14 @@ describe('crete_list_item', () => {
index: LIST_ITEM_INDEX,
refresh: 'wait_for',
};
expect(options.esClient.index).toBeCalledWith(expected);
expect(options.esClient.create).toBeCalledWith(expected);
});
test('It returns an auto-generated id if id is sent in undefined', async () => {
const options = getCreateListItemOptionsMock();
options.id = undefined;
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.index.mockResponse(
esClient.create.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);

5
x-pack/plugins/lists/server/services/items/create_list_item.ts
View file

@ -52,6 +52,7 @@ export const createListItem = async ({
const createdAt = dateNow ?? new Date().toISOString();
const tieBreakerId = tieBreaker ?? uuidv4();
const baseBody = {
'@timestamp': createdAt,
created_at: createdAt,
created_by: user,
deserializer,
@ -68,9 +69,9 @@ export const createListItem = async ({
...baseBody,
...elasticQuery,
};
const response = await esClient.index({
const response = await esClient.create({
body,
id,
id: id ?? uuidv4(),
index: listItemIndex,
refresh: 'wait_for',
});

1
x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts
View file

@ -56,6 +56,7 @@ describe('crete_list_item_bulk', () => {
body: [
{ create: { _index: LIST_ITEM_INDEX } },
{
'@timestamp': '2020-04-20T15:25:31.830Z',
created_at: '2020-04-20T15:25:31.830Z',
created_by: 'some user',
deserializer: undefined,

1
x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts
View file

@ -60,6 +60,7 @@ export const createListItemsBulk = async ({
});
if (elasticQuery != null) {
const elasticBody: IndexEsListItemSchema = {
'@timestamp': createdAt,
created_at: createdAt,
created_by: user,
deserializer,

14
x-pack/plugins/lists/server/services/items/delete_list_item.test.ts
View file

@ -40,16 +40,20 @@ describe('delete_list_item', () => {
expect(deletedListItem).toEqual(listItem);
});
test('Delete calls "delete" if a list item is returned from "getListItem"', async () => {
test('Delete calls "deleteByQuery" if a list item is returned from "getListItem"', async () => {
const listItem = getListItemResponseMock();
(getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem);
const options = getDeleteListItemOptionsMock();
await deleteListItem(options);
const deleteQuery = {
id: LIST_ITEM_ID,
const deleteByQuery = {
index: LIST_ITEM_INDEX,
refresh: 'wait_for',
query: {
ids: {
values: [LIST_ITEM_ID],
},
},
refresh: false,
};
expect(options.esClient.delete).toBeCalledWith(deleteQuery);
expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery);
});
});

10
x-pack/plugins/lists/server/services/items/delete_list_item.ts
View file

@ -25,10 +25,14 @@ export const deleteListItem = async ({
if (listItem == null) {
return null;
} else {
await esClient.delete({
id,
await esClient.deleteByQuery({
index: listItemIndex,
refresh: 'wait_for',
query: {
ids: {
values: [id],
},
},
refresh: false,
});
}
return listItem;

1
x-pack/plugins/lists/server/services/items/find_list_item.test.ts
View file

@ -31,6 +31,7 @@ describe('find_list_item', () => {
{
_id: 'some-list-item-id',
_source: {
'@timestamp': '2020-04-20T15:25:31.830Z',
_version: 'undefined',
created_at: '2020-04-20T15:25:31.830Z',
created_by: 'some user',

1
x-pack/plugins/lists/server/services/items/get_list_item.test.ts
View file

@ -50,6 +50,7 @@ describe('get_list_item', () => {
test('it returns null if all the values underneath the source type is undefined', async () => {
const data = getSearchListItemMock();
data.hits.hits[0]._source = {
'@timestamp': DATE_NOW,
binary: undefined,
boolean: undefined,
byte: undefined,

1
x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts
View file

@ -62,6 +62,7 @@ describe('get_list_item_by_values', () => {
expect(listItem).toEqual([
{
'@timestamp': DATE_NOW,
created_at: DATE_NOW,
created_by: USER,
id: LIST_ITEM_ID,

5
x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts
View file

@ -23,11 +23,12 @@ describe('get_list_item_template', () => {
test('it returns a list template with the string filled in', async () => {
const template = getListItemTemplate('some_index');
expect(template).toEqual({
index_patterns: ['some_index-*'],
data_stream: {},
index_patterns: ['some_index'],
template: {
lifecycle: {},
mappings: { listMappings: {} },
settings: {
index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } },
mapping: {
total_fields: {
limit: 10000,

10
x-pack/plugins/lists/server/services/items/get_list_item_template.ts
View file

@ -9,16 +9,12 @@ import listsItemsMappings from './list_item_mappings.json';
export const getListItemTemplate = (index: string): Record<string, unknown> => {
const template = {
index_patterns: [`${index}-*`],
data_stream: {},
index_patterns: [index],
template: {
lifecycle: {},
mappings: listsItemsMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,

3
x-pack/plugins/lists/server/services/items/list_item_mappings.json
View file

@ -1,6 +1,9 @@
{
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"tie_breaker_id": {
"type": "keyword"
},

1
x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts
View file

@ -74,6 +74,7 @@ describe('search_list_item_by_values', () => {
{
items: [
{
'@timestamp': '2020-04-20T15:25:31.830Z',
_version: undefined,
created_at: '2020-04-20T15:25:31.830Z',
created_by: 'some user',

39
x-pack/plugins/lists/server/services/items/update_list_item.test.ts
View file

@ -4,7 +4,6 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks';
import type { ListItemSchema } from '@kbn/securitysolution-io-ts-list-types';
@ -14,6 +13,12 @@ import { updateListItem } from './update_list_item';
import { getListItem } from './get_list_item';
import { getUpdateListItemOptionsMock } from './update_list_item.mock';
jest.mock('../utils/check_version_conflict', () => ({
checkVersionConflict: jest.fn(),
}));
jest.mock('../utils/wait_until_document_indexed', () => ({
waitUntilDocumentIndexed: jest.fn(),
}));
jest.mock('./get_list_item', () => ({
getListItem: jest.fn(),
}));
@ -27,25 +32,22 @@ describe('update_list_item', () => {
jest.clearAllMocks();
});
test('it returns a list item as expected with the id changed out for the elastic id when there is a list item to update', async () => {
test('it returns a list item when updated', async () => {
const listItem = getListItemResponseMock();
(getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem);
const options = getUpdateListItemOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.update.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
const updatedList = await updateListItem({ ...options, esClient });
const expected: ListItemSchema = { ...getListItemResponseMock(), id: 'elastic-id-123' };
expect(updatedList).toEqual(expected);
esClient.updateByQuery.mockResponse({ updated: 1 });
const updatedListItem = await updateListItem({ ...options, esClient });
const expected: ListItemSchema = getListItemResponseMock();
expect(updatedListItem).toEqual(expected);
});
test('it returns null when there is not a list item to update', async () => {
(getListItem as unknown as jest.Mock).mockResolvedValueOnce(null);
const options = getUpdateListItemOptionsMock();
const updatedList = await updateListItem(options);
expect(updatedList).toEqual(null);
const updatedListItem = await updateListItem(options);
expect(updatedListItem).toEqual(null);
});
test('it returns null when the serializer and type such as ip_range returns nothing', async () => {
@ -57,7 +59,18 @@ describe('update_list_item', () => {
};
(getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem);
const options = getUpdateListItemOptionsMock();
const updatedList = await updateListItem(options);
expect(updatedList).toEqual(null);
const updatedListItem = await updateListItem(options);
expect(updatedListItem).toEqual(null);
});
test('throw error if no list item was updated', async () => {
const listItem = getListItemResponseMock();
(getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem);
const options = getUpdateListItemOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.updateByQuery.mockResponse({ updated: 0 });
await expect(updateListItem({ ...options, esClient })).rejects.toThrow(
'No list item has been updated'
);
});
});

80
x-pack/plugins/lists/server/services/items/update_list_item.ts
View file

@ -12,10 +12,12 @@ import type {
MetaOrUndefined,
_VersionOrUndefined,
} from '@kbn/securitysolution-io-ts-list-types';
import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils';
import { transformListItemToElasticQuery } from '../utils';
import { UpdateEsListItemSchema } from '../../schemas/elastic_query';
import {
checkVersionConflict,
transformListItemToElasticQuery,
waitUntilDocumentIndexed,
} from '../utils';
import { getListItem } from './get_list_item';
@ -28,6 +30,7 @@ export interface UpdateListItemOptions {
user: string;
meta: MetaOrUndefined;
dateNow?: string;
isPatch?: boolean;
}
export const updateListItem = async ({
@ -39,6 +42,7 @@ export const updateListItem = async ({
user,
meta,
dateNow,
isPatch = false,
}: UpdateListItemOptions): Promise<ListItemSchema | null> => {
const updatedAt = dateNow ?? new Date().toISOString();
const listItem = await getListItem({ esClient, id, listItemIndex });
@ -53,30 +57,76 @@ export const updateListItem = async ({
if (elasticQuery == null) {
return null;
} else {
const doc: UpdateEsListItemSchema = {
checkVersionConflict(_version, listItem._version);
const keyValues = Object.entries(elasticQuery).map(([key, keyValue]) => ({
key,
value: keyValue,
}));
const params = {
// when assigning undefined in painless, it will remove property and wil set it to null
// for patch we don't want to remove unspecified value in payload
assignEmpty: !isPatch,
keyValues,
meta,
updated_at: updatedAt,
updated_by: user,
...elasticQuery,
};
const response = await esClient.update({
...decodeVersion(_version),
body: {
doc,
},
id: listItem.id,
const response = await esClient.updateByQuery({
conflicts: 'proceed',
index: listItemIndex,
refresh: 'wait_for',
query: {
ids: {
values: [id],
},
},
refresh: false,
script: {
lang: 'painless',
params,
source: `
for (int i; i < params.keyValues.size(); i++) {
def entry = params.keyValues[i];
ctx._source[entry.key] = entry.value;
}
if (params.assignEmpty == true || params.containsKey('meta')) {
ctx._source.meta = params.meta;
}
ctx._source.updated_at = params.updated_at;
ctx._source.updated_by = params.updated_by;
// needed for list items that were created before migration to data streams
if (ctx._source.containsKey('@timestamp') == false) {
ctx._source['@timestamp'] = ctx._source.created_at;
}
`,
},
});
let updatedOCCVersion: string | undefined;
if (response.updated) {
const checkIfListUpdated = async (): Promise<void> => {
const updatedListItem = await getListItem({ esClient, id, listItemIndex });
if (updatedListItem?._version === listItem._version) {
throw Error('List item has not been re-indexed in time');
}
updatedOCCVersion = updatedListItem?._version;
};
await waitUntilDocumentIndexed(checkIfListUpdated);
} else {
throw Error('No list item has been updated');
}
return {
_version: encodeHitVersion(response),
'@timestamp': listItem['@timestamp'],
_version: updatedOCCVersion,
created_at: listItem.created_at,
created_by: listItem.created_by,
deserializer: listItem.deserializer,
id: response._id,
id,
list_id: listItem.list_id,
meta: meta ?? listItem.meta,
meta: isPatch ? meta ?? listItem.meta : meta,
serializer: listItem.serializer,
tie_breaker_id: listItem.tie_breaker_id,
type: listItem.type,

100
x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts
View file

@ -54,60 +54,68 @@ export const importListItemsToStream = ({
meta,
version,
}: ImportListItemsToStreamOptions): Promise<ListSchema | null> => {
return new Promise<ListSchema | null>((resolve) => {
return new Promise<ListSchema | null>((resolve, reject) => {
const readBuffer = new BufferLines({ bufferSize: config.importBufferSize, input: stream });
let fileName: string | undefined;
let list: ListSchema | null = null;
readBuffer.on('fileName', async (fileNameEmitted: string) => {
readBuffer.pause();
fileName = decodeURIComponent(fileNameEmitted);
if (listId == null) {
list = await createListIfItDoesNotExist({
description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', {
defaultMessage: 'File uploaded from file system of {fileName}',
values: { fileName },
}),
deserializer,
esClient,
id: fileName,
immutable: false,
listIndex,
meta,
name: fileName,
serializer,
type,
user,
version,
});
try {
readBuffer.pause();
fileName = decodeURIComponent(fileNameEmitted);
if (listId == null) {
list = await createListIfItDoesNotExist({
description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', {
defaultMessage: 'File uploaded from file system of {fileName}',
values: { fileName },
}),
deserializer,
esClient,
id: fileName,
immutable: false,
listIndex,
meta,
name: fileName,
serializer,
type,
user,
version,
});
}
readBuffer.resume();
} catch (err) {
reject(err);
}
readBuffer.resume();
});
readBuffer.on('lines', async (lines: string[]) => {
if (listId != null) {
await writeBufferToItems({
buffer: lines,
deserializer,
esClient,
listId,
listItemIndex,
meta,
serializer,
type,
user,
});
} else if (fileName != null) {
await writeBufferToItems({
buffer: lines,
deserializer,
esClient,
listId: fileName,
listItemIndex,
meta,
serializer,
type,
user,
});
try {
if (listId != null) {
await writeBufferToItems({
buffer: lines,
deserializer,
esClient,
listId,
listItemIndex,
meta,
serializer,
type,
user,
});
} else if (fileName != null) {
await writeBufferToItems({
buffer: lines,
deserializer,
esClient,
listId: fileName,
listItemIndex,
meta,
serializer,
type,
user,
});
}
} catch (err) {
reject(err);
}
});

10
x-pack/plugins/lists/server/services/lists/create_list.test.ts
View file

@ -15,7 +15,7 @@ import { getIndexESListMock } from '../../schemas/elastic_query/index_es_list_sc
import { CreateListOptions, createList } from './create_list';
import { getCreateListOptionsMock } from './create_list.mock';
describe('crete_list', () => {
describe('create_list', () => {
beforeEach(() => {
jest.clearAllMocks();
});
@ -27,7 +27,7 @@ describe('crete_list', () => {
test('it returns a list as expected with the id changed out for the elastic id', async () => {
const options = getCreateListOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.index.mockResponse(
esClient.create.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
@ -43,7 +43,7 @@ describe('crete_list', () => {
serializer: '(?<value>)',
};
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.index.mockResponse(
esClient.create.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
@ -67,14 +67,14 @@ describe('crete_list', () => {
index: LIST_INDEX,
refresh: 'wait_for',
};
expect(options.esClient.index).toBeCalledWith(expected);
expect(options.esClient.create).toBeCalledWith(expected);
});
test('It returns an auto-generated id if id is sent in undefined', async () => {
const options = getCreateListOptionsMock();
options.id = undefined;
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.index.mockResponse(
esClient.create.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);

7
x-pack/plugins/lists/server/services/lists/create_list.ts
View file

@ -58,6 +58,7 @@ export const createList = async ({
}: CreateListOptions): Promise<ListSchema> => {
const createdAt = dateNow ?? new Date().toISOString();
const body: IndexEsListSchema = {
'@timestamp': createdAt,
created_at: createdAt,
created_by: user,
description,
@ -72,12 +73,14 @@ export const createList = async ({
updated_by: user,
version,
};
const response = await esClient.index({
const response = await esClient.create({
body,
id,
id: id ?? uuidv4(),
index: listIndex,
refresh: 'wait_for',
});
return {
_version: encodeHitVersion(response),
id: response._id,

38
x-pack/plugins/lists/server/services/lists/delete_list.test.ts
View file

@ -12,6 +12,10 @@ import { getList } from './get_list';
import { deleteList } from './delete_list';
import { getDeleteListOptionsMock } from './delete_list.mock';
jest.mock('../utils', () => ({
waitUntilDocumentIndexed: jest.fn(),
}));
jest.mock('./get_list', () => ({
getList: jest.fn(),
}));
@ -36,34 +40,45 @@ describe('delete_list', () => {
const list = getListResponseMock();
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getDeleteListOptionsMock();
options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 });
const deletedList = await deleteList(options);
expect(deletedList).toEqual(list);
});
test('Delete calls "deleteByQuery" and "delete" if a list is returned from getList', async () => {
test('Delete calls "deleteByQuery" for list items if a list is returned from getList', async () => {
const list = getListResponseMock();
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getDeleteListOptionsMock();
options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 });
await deleteList(options);
const deleteByQuery = {
body: { query: { term: { list_id: LIST_ID } } },
conflicts: 'proceed',
index: LIST_ITEM_INDEX,
refresh: false,
};
expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery);
expect(options.esClient.deleteByQuery).toHaveBeenNthCalledWith(1, deleteByQuery);
});
test('Delete calls "delete" second if a list is returned from getList', async () => {
test('Delete calls "deleteByQuery" for list if a list is returned from getList', async () => {
const list = getListResponseMock();
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getDeleteListOptionsMock();
options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 });
await deleteList(options);
const deleteQuery = {
id: LIST_ID,
const deleteByQuery = {
body: {
query: {
ids: {
values: [LIST_ID],
},
},
},
conflicts: 'proceed',
index: LIST_INDEX,
refresh: 'wait_for',
refresh: false,
};
expect(options.esClient.delete).toHaveBeenNthCalledWith(1, deleteQuery);
expect(options.esClient.deleteByQuery).toHaveBeenCalledWith(deleteByQuery);
});
test('Delete does not call data client if the list returns null', async () => {
@ -72,4 +87,13 @@ describe('delete_list', () => {
await deleteList(options);
expect(options.esClient.delete).not.toHaveBeenCalled();
});
test('throw error if no list was deleted', async () => {
const list = getListResponseMock();
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getDeleteListOptionsMock();
options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 0 });
await expect(deleteList(options)).rejects.toThrow('No list has been deleted');
});
});

30
x-pack/plugins/lists/server/services/lists/delete_list.ts
View file

@ -8,6 +8,8 @@
import { ElasticsearchClient } from '@kbn/core/server';
import type { Id, ListSchema } from '@kbn/securitysolution-io-ts-list-types';
import { waitUntilDocumentIndexed } from '../utils';
import { getList } from './get_list';
export interface DeleteListOptions {
@ -35,15 +37,37 @@ export const deleteList = async ({
},
},
},
conflicts: 'proceed',
index: listItemIndex,
refresh: false,
});
await esClient.delete({
id,
const response = await esClient.deleteByQuery({
body: {
query: {
ids: {
values: [id],
},
},
},
conflicts: 'proceed',
index: listIndex,
refresh: 'wait_for',
refresh: false,
});
if (response.deleted) {
const checkIfListDeleted = async (): Promise<void> => {
const deletedList = await getList({ esClient, id, listIndex });
if (deletedList !== null) {
throw Error('List has not been re-indexed in time');
}
};
await waitUntilDocumentIndexed(checkIfListDeleted);
} else {
throw Error('No list has been deleted');
}
return list;
}
};

5
x-pack/plugins/lists/server/services/lists/get_list_template.test.ts
View file

@ -24,11 +24,12 @@ describe('get_list_template', () => {
test('it returns a list template with the string filled in', async () => {
const template = getListTemplate('some_index');
expect(template).toEqual({
index_patterns: ['some_index-*'],
data_stream: {},
index_patterns: ['some_index'],
template: {
lifecycle: {},
mappings: { dynamic: 'strict', properties: {} },
settings: {
index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } },
mapping: { total_fields: { limit: 10000 } },
},
},

10
x-pack/plugins/lists/server/services/lists/get_list_template.ts
View file

@ -8,16 +8,12 @@
import listMappings from './list_mappings.json';
export const getListTemplate = (index: string): Record<string, unknown> => ({
index_patterns: [`${index}-*`],
data_stream: {},
index_patterns: [index],
template: {
lifecycle: {},
mappings: listMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,

4
x-pack/plugins/lists/server/services/lists/list_client.mock.ts
View file

@ -24,8 +24,8 @@ import {
import { ListClient } from './list_client';
export class ListClientMock extends ListClient {
public getListIndex = jest.fn().mockReturnValue(LIST_INDEX);
public getListItemIndex = jest.fn().mockReturnValue(LIST_ITEM_INDEX);
public getListName = jest.fn().mockReturnValue(LIST_INDEX);
public getListItemName = jest.fn().mockReturnValue(LIST_ITEM_INDEX);
public getList = jest.fn().mockResolvedValue(getListResponseMock());
public createList = jest.fn().mockResolvedValue(getListResponseMock());
public createListIfItDoesNotExist = jest.fn().mockResolvedValue(getListResponseMock());

4
x-pack/plugins/lists/server/services/lists/list_client.test.ts
View file

@ -14,12 +14,12 @@ describe('list_client', () => {
describe('Mock client checks (not exhaustive tests against it)', () => {
test('it returns the get list index as expected', () => {
const mock = getListClientMock();
expect(mock.getListIndex()).toEqual(LIST_INDEX);
expect(mock.getListName()).toEqual(LIST_INDEX);
});
test('it returns the get list item index as expected', () => {
const mock = getListClientMock();
expect(mock.getListItemIndex()).toEqual(LIST_ITEM_INDEX);
expect(mock.getListItemName()).toEqual(LIST_ITEM_INDEX);
});
test('it returns a mock list item', async () => {

378
x-pack/plugins/lists/server/services/lists/list_client.ts
View file

@ -4,18 +4,24 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
import type { ElasticsearchClient } from '@kbn/core/server';
import {
createBootstrapIndex,
createDataStream,
deleteAllIndex,
deleteDataStream,
deleteIndexTemplate,
deletePolicy,
deleteTemplate,
getBootstrapIndexExists,
getDataStreamExists,
getIndexTemplateExists,
getPolicyExists,
getTemplateExists,
migrateToDataStream,
putMappings,
removePolicyFromIndex,
setIndexTemplate,
setPolicy,
} from '@kbn/securitysolution-es-utils';
@ -47,8 +53,10 @@ import {
updateListItem,
} from '../items';
import listsItemsPolicy from '../items/list_item_policy.json';
import listItemMappings from '../items/list_item_mappings.json';
import listPolicy from './list_policy.json';
import listMappings from './list_mappings.json';
import type {
ConstructorOptions,
CreateListIfItDoesNotExistOptions,
@ -115,10 +123,10 @@ export class ListClient {
}
/**
* Returns the list index name
* @returns The list index name
* Returns the list data stream or index name
* @returns The list data stream/index name
*/
public getListIndex = (): string => {
public getListName = (): string => {
const {
spaceId,
config: { listIndex: listsIndexName },
@ -127,10 +135,10 @@ export class ListClient {
};
/**
* Returns the list item index name
* @returns The list item index name
* Returns the list item data stream or index name
* @returns The list item data stream/index name
*/
public getListItemIndex = (): string => {
public getListItemName = (): string => {
const {
spaceId,
config: { listItemIndex: listsItemsIndexName },
@ -146,8 +154,8 @@ export class ListClient {
*/
public getList = async ({ id }: GetListOptions): Promise<ListSchema | null> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getList({ esClient, id, listIndex });
const listName = this.getListName();
return getList({ esClient, id, listIndex: listName });
};
/**
@ -178,14 +186,14 @@ export class ListClient {
version,
}: CreateListOptions): Promise<ListSchema> => {
const { esClient, user } = this;
const listIndex = this.getListIndex();
const listName = this.getListName();
return createList({
description,
deserializer,
esClient,
id,
immutable,
listIndex,
listIndex: listName,
meta,
name,
serializer,
@ -225,14 +233,14 @@ export class ListClient {
version,
}: CreateListIfItDoesNotExistOptions): Promise<ListSchema> => {
const { esClient, user } = this;
const listIndex = this.getListIndex();
const listName = this.getListName();
return createListIfItDoesNotExist({
description,
deserializer,
esClient,
id,
immutable,
listIndex,
listIndex: listName,
meta,
name,
serializer,
@ -248,8 +256,18 @@ export class ListClient {
*/
public getListIndexExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getBootstrapIndexExists(esClient, listIndex);
const listName = this.getListName();
return getBootstrapIndexExists(esClient, listName);
};
/**
* True if the list data stream exists, otherwise false
* @returns True if the list data stream exists, otherwise false
*/
public getListDataStreamExists = async (): Promise<boolean> => {
const { esClient } = this;
const listName = this.getListName();
return getDataStreamExists(esClient, listName);
};
/**
@ -258,28 +276,104 @@ export class ListClient {
*/
public getListItemIndexExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getBootstrapIndexExists(esClient, listItemIndex);
const listItemName = this.getListItemName();
return getBootstrapIndexExists(esClient, listItemName);
};
/**
* True if the list item data stream exists, otherwise false
* @returns True if the list item data stream exists, otherwise false
*/
public getListItemDataStreamExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemName = this.getListItemName();
return getDataStreamExists(esClient, listItemName);
};
/**
* Creates the list boot strap index for ILM policies.
* @returns The contents of the bootstrap response from Elasticsearch
* @deprecated after moving to data streams there should not be need to use it
*/
public createListBootStrapIndex = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return createBootstrapIndex(esClient, listIndex);
const listName = this.getListName();
return createBootstrapIndex(esClient, listName);
};
/**
* Creates list data stream
* @returns The contents of the create data stream from Elasticsearch
*/
public createListDataStream = async (): Promise<unknown> => {
const { esClient } = this;
const listName = this.getListName();
return createDataStream(esClient, listName);
};
/**
* update list index mappings with @timestamp and migrates it to data stream
* @returns
*/
public migrateListIndexToDataStream = async (): Promise<void> => {
const { esClient } = this;
const listName = this.getListName();
// update list index template
await this.setListTemplate();
// first need to update mapping of existing index to add @timestamp
await putMappings(
esClient,
listName,
listMappings.properties as Record<string, MappingProperty>
);
await migrateToDataStream(esClient, listName);
await removePolicyFromIndex(esClient, listName);
if (await this.getListPolicyExists()) {
await this.deleteListPolicy();
}
};
/**
* update list items index mappings with @timestamp and migrates it to data stream
* @returns
*/
public migrateListItemIndexToDataStream = async (): Promise<void> => {
const { esClient } = this;
const listItemName = this.getListItemName();
// update list items index template
await this.setListItemTemplate();
// first need to update mapping of existing index to add @timestamp
await putMappings(
esClient,
listItemName,
listItemMappings.properties as Record<string, MappingProperty>
);
await migrateToDataStream(esClient, listItemName);
await removePolicyFromIndex(esClient, listItemName);
if (await this.getListItemPolicyExists()) {
await this.deleteListItemPolicy();
}
};
/**
* Creates the list item boot strap index for ILM policies.
* @returns The contents of the bootstrap response from Elasticsearch
* @deprecated after moving to data streams there should not be need to use it
*/
public createListItemBootStrapIndex = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return createBootstrapIndex(esClient, listItemIndex);
const listItemName = this.getListItemName();
return createBootstrapIndex(esClient, listItemName);
};
/**
* Creates list item data stream
* @returns The contents of the create data stream from Elasticsearch
*/
public createListItemDataStream = async (): Promise<unknown> => {
const { esClient } = this;
const listItemName = this.getListItemName();
return createDataStream(esClient, listItemName);
};
/**
@ -288,8 +382,8 @@ export class ListClient {
*/
public getListPolicyExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getPolicyExists(esClient, listIndex);
const listName = this.getListName();
return getPolicyExists(esClient, listName);
};
/**
@ -298,7 +392,7 @@ export class ListClient {
*/
public getListItemPolicyExists = async (): Promise<boolean> => {
const { esClient } = this;
const listsItemIndex = this.getListItemIndex();
const listsItemIndex = this.getListItemName();
return getPolicyExists(esClient, listsItemIndex);
};
@ -308,8 +402,8 @@ export class ListClient {
*/
public getListTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getIndexTemplateExists(esClient, listIndex);
const listName = this.getListName();
return getIndexTemplateExists(esClient, listName);
};
/**
@ -318,8 +412,8 @@ export class ListClient {
*/
public getListItemTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getIndexTemplateExists(esClient, listItemIndex);
const listItemName = this.getListItemName();
return getIndexTemplateExists(esClient, listItemName);
};
/**
@ -328,8 +422,8 @@ export class ListClient {
*/
public getLegacyListTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getTemplateExists(esClient, listIndex);
const listName = this.getListName();
return getTemplateExists(esClient, listName);
};
/**
@ -338,8 +432,8 @@ export class ListClient {
*/
public getLegacyListItemTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getTemplateExists(esClient, listItemIndex);
const listItemName = this.getListItemName();
return getTemplateExists(esClient, listItemName);
};
/**
@ -347,8 +441,8 @@ export class ListClient {
* @returns The contents of the list template for ILM.
*/
public getListTemplate = (): Record<string, unknown> => {
const listIndex = this.getListIndex();
return getListTemplate(listIndex);
const listName = this.getListName();
return getListTemplate(listName);
};
/**
@ -356,8 +450,8 @@ export class ListClient {
* @returns The contents of the list item template for ILM.
*/
public getListItemTemplate = (): Record<string, unknown> => {
const listItemIndex = this.getListItemIndex();
return getListItemTemplate(listItemIndex);
const listItemName = this.getListItemName();
return getListItemTemplate(listItemName);
};
/**
@ -367,8 +461,8 @@ export class ListClient {
public setListTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const template = this.getListTemplate();
const listIndex = this.getListIndex();
return setIndexTemplate(esClient, listIndex, template);
const listName = this.getListName();
return setIndexTemplate(esClient, listName, template);
};
/**
@ -378,28 +472,30 @@ export class ListClient {
public setListItemTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const template = this.getListItemTemplate();
const listItemIndex = this.getListItemIndex();
return setIndexTemplate(esClient, listItemIndex, template);
const listItemName = this.getListItemName();
return setIndexTemplate(esClient, listItemName, template);
};
/**
* Sets the list policy
* @returns The contents of the list policy set
* @deprecated after moving to data streams there should not be need to use it
*/
public setListPolicy = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return setPolicy(esClient, listIndex, listPolicy);
const listName = this.getListName();
return setPolicy(esClient, listName, listPolicy);
};
/**
* Sets the list item policy
* @returns The contents of the list policy set
* @deprecated after moving to data streams there should not be need to use it
*/
public setListItemPolicy = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return setPolicy(esClient, listItemIndex, listsItemsPolicy);
const listItemName = this.getListItemName();
return setPolicy(esClient, listItemName, listsItemsPolicy);
};
/**
@ -408,8 +504,8 @@ export class ListClient {
*/
public deleteListIndex = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deleteAllIndex(esClient, `${listIndex}-*`);
const listName = this.getListName();
return deleteAllIndex(esClient, `${listName}-*`);
};
/**
@ -418,8 +514,28 @@ export class ListClient {
*/
public deleteListItemIndex = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteAllIndex(esClient, `${listItemIndex}-*`);
const listItemName = this.getListItemName();
return deleteAllIndex(esClient, `${listItemName}-*`);
};
/**
* Deletes the list data stream
* @returns True if the list index was deleted, otherwise false
*/
public deleteListDataStream = async (): Promise<boolean> => {
const { esClient } = this;
const listName = this.getListName();
return deleteDataStream(esClient, listName);
};
/**
* Deletes the list item data stream
* @returns True if the list index was deleted, otherwise false
*/
public deleteListItemDataStream = async (): Promise<boolean> => {
const { esClient } = this;
const listItemName = this.getListItemName();
return deleteDataStream(esClient, listItemName);
};
/**
@ -428,8 +544,8 @@ export class ListClient {
*/
public deleteListPolicy = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deletePolicy(esClient, listIndex);
const listName = this.getListName();
return deletePolicy(esClient, listName);
};
/**
@ -438,8 +554,8 @@ export class ListClient {
*/
public deleteListItemPolicy = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deletePolicy(esClient, listItemIndex);
const listItemName = this.getListItemName();
return deletePolicy(esClient, listItemName);
};
/**
@ -448,8 +564,8 @@ export class ListClient {
*/
public deleteListTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deleteIndexTemplate(esClient, listIndex);
const listName = this.getListName();
return deleteIndexTemplate(esClient, listName);
};
/**
@ -458,8 +574,8 @@ export class ListClient {
*/
public deleteListItemTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteIndexTemplate(esClient, listItemIndex);
const listItemName = this.getListItemName();
return deleteIndexTemplate(esClient, listItemName);
};
/**
@ -468,8 +584,8 @@ export class ListClient {
*/
public deleteLegacyListTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deleteTemplate(esClient, listIndex);
const listName = this.getListName();
return deleteTemplate(esClient, listName);
};
/**
@ -478,8 +594,8 @@ export class ListClient {
*/
public deleteLegacyListItemTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteTemplate(esClient, listItemIndex);
const listItemName = this.getListItemName();
return deleteTemplate(esClient, listItemName);
};
/**
@ -488,8 +604,8 @@ export class ListClient {
*/
public deleteListItem = async ({ id }: DeleteListItemOptions): Promise<ListItemSchema | null> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteListItem({ esClient, id, listItemIndex });
const listItemName = this.getListItemName();
return deleteListItem({ esClient, id, listItemIndex: listItemName });
};
/**
@ -506,11 +622,11 @@ export class ListClient {
type,
}: DeleteListItemByValueOptions): Promise<ListItemArraySchema> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return deleteListItemByValue({
esClient,
listId,
listItemIndex,
listItemIndex: listItemName,
type,
value,
});
@ -524,13 +640,13 @@ export class ListClient {
*/
public deleteList = async ({ id }: DeleteListOptions): Promise<ListSchema | null> => {
const { esClient } = this;
const listIndex = this.getListIndex();
const listItemIndex = this.getListItemIndex();
const listName = this.getListName();
const listItemName = this.getListItemName();
return deleteList({
esClient,
id,
listIndex,
listItemIndex,
listIndex: listName,
listItemIndex: listItemName,
});
};
@ -547,11 +663,11 @@ export class ListClient {
stream,
}: ExportListItemsToStreamOptions): void => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
exportListItemsToStream({
esClient,
listId,
listItemIndex,
listItemIndex: listItemName,
stream,
stringToAppend,
});
@ -580,15 +696,15 @@ export class ListClient {
version,
}: ImportListItemsToStreamOptions): Promise<ListSchema | null> => {
const { esClient, user, config } = this;
const listItemIndex = this.getListItemIndex();
const listIndex = this.getListIndex();
const listItemName = this.getListItemName();
const listName = this.getListName();
return importListItemsToStream({
config,
deserializer,
esClient,
listId,
listIndex,
listItemIndex,
listIndex: listName,
listItemIndex: listItemName,
meta,
serializer,
stream,
@ -612,11 +728,11 @@ export class ListClient {
type,
}: GetListItemByValueOptions): Promise<ListItemArraySchema> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return getListItemByValue({
esClient,
listId,
listItemIndex,
listItemIndex: listItemName,
type,
value,
});
@ -646,13 +762,13 @@ export class ListClient {
meta,
}: CreateListItemOptions): Promise<ListItemSchema | null> => {
const { esClient, user } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return createListItem({
deserializer,
esClient,
id,
listId,
listItemIndex,
listItemIndex: listItemName,
meta,
serializer,
type,
@ -678,12 +794,43 @@ export class ListClient {
meta,
}: UpdateListItemOptions): Promise<ListItemSchema | null> => {
const { esClient, user } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return updateListItem({
_version,
esClient,
id,
listItemIndex,
isPatch: false,
listItemIndex: listItemName,
meta,
user,
value,
});
};
/**
* Patches a list item's value given the id of the list item.
* See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html}
* for more information around optimistic concurrency control.
* @param options
* @param options._version This is the version, useful for optimistic concurrency control.
* @param options.id id of the list to replace the list item with.
* @param options.value The value of the list item to replace.
* @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values.
*/
public patchListItem = async ({
_version,
id,
value,
meta,
}: UpdateListItemOptions): Promise<ListItemSchema | null> => {
const { esClient, user } = this;
const listItemName = this.getListItemName();
return updateListItem({
_version,
esClient,
id,
isPatch: true,
listItemIndex: listItemName,
meta,
user,
value,
@ -711,13 +858,48 @@ export class ListClient {
version,
}: UpdateListOptions): Promise<ListSchema | null> => {
const { esClient, user } = this;
const listIndex = this.getListIndex();
const listName = this.getListName();
return updateList({
_version,
description,
esClient,
id,
listIndex,
isPatch: false,
listIndex: listName,
meta,
name,
user,
version,
});
};
/**
* Patches a list container's value given the id of the list.
* @param options
* @param options._version This is the version, useful for optimistic concurrency control.
* @param options.id id of the list to replace the list container data with.
* @param options.name The new name, or "undefined" if this should not be updated.
* @param options.description The new description, or "undefined" if this should not be updated.
* @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values.
* @param options.version Updates the version of the list.
*/
public patchList = async ({
_version,
id,
name,
description,
meta,
version,
}: UpdateListOptions): Promise<ListSchema | null> => {
const { esClient, user } = this;
const listName = this.getListName();
return updateList({
_version,
description,
esClient,
id,
isPatch: true,
listIndex: listName,
meta,
name,
user,
@ -733,11 +915,11 @@ export class ListClient {
*/
public getListItem = async ({ id }: GetListItemOptions): Promise<ListItemSchema | null> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return getListItem({
esClient,
id,
listItemIndex,
listItemIndex: listItemName,
});
};
@ -755,11 +937,11 @@ export class ListClient {
value,
}: GetListItemsByValueOptions): Promise<ListItemArraySchema> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return getListItemByValues({
esClient,
listId,
listItemIndex,
listItemIndex: listItemName,
type,
value,
});
@ -779,11 +961,11 @@ export class ListClient {
value,
}: SearchListItemByValuesOptions): Promise<SearchListItemArraySchema> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
const listItemName = this.getListItemName();
return searchListItemByValues({
esClient,
listId,
listItemIndex,
listItemIndex: listItemName,
type,
value,
});
@ -813,12 +995,12 @@ export class ListClient {
runtimeMappings,
}: FindListOptions): Promise<FoundListSchema> => {
const { esClient } = this;
const listIndex = this.getListIndex();
const listName = this.getListName();
return findList({
currentIndexPosition,
esClient,
filter,
listIndex,
listIndex: listName,
page,
perPage,
runtimeMappings,
@ -855,15 +1037,15 @@ export class ListClient {
searchAfter,
}: FindListItemOptions): Promise<FoundListItemSchema | null> => {
const { esClient } = this;
const listIndex = this.getListIndex();
const listItemIndex = this.getListItemIndex();
const listName = this.getListName();
const listItemName = this.getListItemName();
return findListItem({
currentIndexPosition,
esClient,
filter,
listId,
listIndex,
listItemIndex,
listIndex: listName,
listItemIndex: listItemName,
page,
perPage,
runtimeMappings,
@ -880,14 +1062,14 @@ export class ListClient {
sortOrder,
}: FindAllListItemsOptions): Promise<FoundAllListItemsSchema | null> => {
const { esClient } = this;
const listIndex = this.getListIndex();
const listItemIndex = this.getListItemIndex();
const listName = this.getListName();
const listItemName = this.getListItemName();
return findAllListItems({
esClient,
filter,
listId,
listIndex,
listItemIndex,
listIndex: listName,
listItemIndex: listItemName,
sortField,
sortOrder,
});

3
x-pack/plugins/lists/server/services/lists/list_mappings.json
View file

@ -1,6 +1,9 @@
{
"dynamic": "strict",
"properties": {
"@timestamp": {
"type": "date"
},
"name": {
"type": "keyword"
},

39
x-pack/plugins/lists/server/services/lists/update_list.test.ts
View file

@ -14,6 +14,11 @@ import { updateList } from './update_list';
import { getList } from './get_list';
import { getUpdateListOptionsMock } from './update_list.mock';
jest.mock('../utils', () => ({
checkVersionConflict: jest.fn(),
waitUntilDocumentIndexed: jest.fn(),
}));
jest.mock('./get_list', () => ({
getList: jest.fn(),
}));
@ -27,20 +32,6 @@ describe('update_list', () => {
jest.clearAllMocks();
});
test('it returns a list as expected with the id changed out for the elastic id when there is a list to update', async () => {
const list = getListResponseMock();
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getUpdateListOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.update.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
const updatedList = await updateList({ ...options, esClient });
const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' };
expect(updatedList).toEqual(expected);
});
test('it returns a list with serializer and deserializer', async () => {
const list: ListSchema = {
...getListResponseMock(),
@ -50,15 +41,12 @@ describe('update_list', () => {
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getUpdateListOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.update.mockResponse(
// @ts-expect-error not full response interface
{ _id: 'elastic-id-123' }
);
esClient.updateByQuery.mockResolvedValue({ updated: 1 });
const updatedList = await updateList({ ...options, esClient });
const expected: ListSchema = {
...getListResponseMock(),
deserializer: '{{value}}',
id: 'elastic-id-123',
id: list.id,
serializer: '(?<value>)',
};
expect(updatedList).toEqual(expected);
@ -70,4 +58,17 @@ describe('update_list', () => {
const updatedList = await updateList(options);
expect(updatedList).toEqual(null);
});
test('throw error if no list was updated', async () => {
const list: ListSchema = {
...getListResponseMock(),
deserializer: '{{value}}',
serializer: '(?<value>)',
};
(getList as unknown as jest.Mock).mockResolvedValueOnce(list);
const options = getUpdateListOptionsMock();
const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser;
esClient.updateByQuery.mockResolvedValue({ updated: 0 });
await expect(updateList({ ...options, esClient })).rejects.toThrow('No list has been updated');
});
});

75
x-pack/plugins/lists/server/services/lists/update_list.ts
View file

@ -15,9 +15,9 @@ import type {
_VersionOrUndefined,
} from '@kbn/securitysolution-io-ts-list-types';
import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types';
import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils';
import { UpdateEsListSchema } from '../../schemas/elastic_query';
import { checkVersionConflict, waitUntilDocumentIndexed } from '../utils';
import { getList } from '.';
@ -32,6 +32,7 @@ export interface UpdateListOptions {
meta: MetaOrUndefined;
dateNow?: string;
version: VersionOrUndefined;
isPatch?: boolean;
}
export const updateList = async ({
@ -45,36 +46,90 @@ export const updateList = async ({
meta,
dateNow,
version,
isPatch = false,
}: UpdateListOptions): Promise<ListSchema | null> => {
const updatedAt = dateNow ?? new Date().toISOString();
const list = await getList({ esClient, id, listIndex });
if (list == null) {
return null;
} else {
checkVersionConflict(_version, list._version);
const calculatedVersion = version == null ? list.version + 1 : version;
const doc: UpdateEsListSchema = {
const params: UpdateEsListSchema = {
description,
meta,
name,
updated_at: updatedAt,
updated_by: user,
version: calculatedVersion,
};
const response = await esClient.update({
...decodeVersion(_version),
body: { doc },
id,
const response = await esClient.updateByQuery({
conflicts: 'proceed',
index: listIndex,
refresh: 'wait_for',
query: {
ids: {
values: [id],
},
},
refresh: false,
script: {
lang: 'painless',
params: {
...params,
// when assigning undefined in painless, it will remove property and wil set it to null
// for patch we don't want to remove unspecified value in payload
assignEmpty: !isPatch,
},
source: `
if (params.assignEmpty == true || params.containsKey('description')) {
ctx._source.description = params.description;
}
if (params.assignEmpty == true || params.containsKey('meta')) {
ctx._source.meta = params.meta;
}
if (params.assignEmpty == true || params.containsKey('name')) {
ctx._source.name = params.name;
}
if (params.assignEmpty == true || params.containsKey('version')) {
ctx._source.version = params.version;
}
ctx._source.updated_at = params.updated_at;
ctx._source.updated_by = params.updated_by;
// needed for list that were created before migration to data streams
if (ctx._source.containsKey('@timestamp') == false) {
ctx._source['@timestamp'] = ctx._source.created_at;
}
`,
},
});
let updatedOCCVersion: string | undefined;
if (response.updated) {
const checkIfListUpdated = async (): Promise<void> => {
const updatedList = await getList({ esClient, id, listIndex });
if (updatedList?._version === list._version) {
throw Error('Document has not been re-indexed in time');
}
updatedOCCVersion = updatedList?._version;
};
await waitUntilDocumentIndexed(checkIfListUpdated);
} else {
throw Error('No list has been updated');
}
return {
_version: encodeHitVersion(response),
'@timestamp': list['@timestamp'],
_version: updatedOCCVersion,
created_at: list.created_at,
created_by: list.created_by,
description: description ?? list.description,
deserializer: list.deserializer,
id: response._id,
id,
immutable: list.immutable,
meta,
meta: isPatch ? meta ?? list.meta : meta,
name: name ?? list.name,
serializer: list.serializer,
tie_breaker_id: list.tie_breaker_id,

26
x-pack/plugins/lists/server/services/utils/check_version_conflict.ts Normal file
View file

@ -0,0 +1,26 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Boom from '@hapi/boom';
import { decodeVersion } from '@kbn/securitysolution-es-utils';
/**
* checks if encoded OCC update _version matches actual version of list/item
* @param updateVersion - version in payload
* @param existingVersion - version in exiting list/item
*/
export const checkVersionConflict = (
updateVersion: string | undefined,
existingVersion: string | undefined
): void => {
if (updateVersion && existingVersion && updateVersion !== existingVersion) {
throw Boom.conflict(
`Conflict: versions mismatch. Provided versions:${JSON.stringify(
decodeVersion(updateVersion)
)} does not match ${JSON.stringify(decodeVersion(existingVersion))}`
);
}
};

2
x-pack/plugins/lists/server/services/utils/find_source_value.ts
View file

@ -23,7 +23,7 @@ export const findSourceValue = (
const foundEntry = Object.entries(listItem).find(
([key, value]) => types.includes(key) && value != null
);
if (foundEntry != null) {
if (foundEntry != null && foundEntry[1] !== null) {
const [foundType, value] = foundEntry;
switch (foundType) {
case 'shape':

2
x-pack/plugins/lists/server/services/utils/index.ts
View file

@ -8,6 +8,7 @@
export * from './calculate_scroll_math';
export * from './encode_decode_cursor';
export * from './escape_query';
export * from './check_version_conflict';
export * from './find_source_type';
export * from './find_source_value';
export * from './get_query_filter_from_type_value';
@ -21,3 +22,4 @@ export * from './transform_elastic_named_search_to_list_item';
export * from './transform_elastic_to_list_item';
export * from './transform_elastic_to_list';
export * from './transform_list_item_to_elastic_query';
export * from './wait_until_document_indexed';

3
x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts
View file

@ -24,6 +24,9 @@ export const transformElasticToList = ({
_version: encodeHitVersion(hit),
id: hit._id,
...hit._source,
// meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null
// return it as undefined to keep it consistent with payload
meta: hit._source?.meta ?? undefined,
};
});
};

5
x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts
View file

@ -56,13 +56,16 @@ export const transformElasticHitsToListItem = ({
throw new ErrorWithStatusCode(`Was expected ${type} to not be null/undefined`, 400);
} else {
return {
'@timestamp': _source?.['@timestamp'],
_version: encodeHitVersion(hit),
created_at,
created_by,
deserializer,
id: _id,
list_id,
meta,
// meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null
// return it as undefined to keep it consistent with payload
meta: meta ?? undefined,
serializer,
tie_breaker_id,
type,

25
x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import pRetry from 'p-retry';
// index.refresh_interval
// https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings
const DEFAULT_INDEX_REFRESH_TIME = 1000;
/**
* retries until list/list item has been re-indexed
* After migration to data stream and using update_by_query, delete_by_query which do support only refresh=true/false,
* this utility needed response back when updates/delete applied
* @param fn execution function to retry
*/
export const waitUntilDocumentIndexed = async (fn: () => Promise<void>): Promise<void> => {
await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME));
await pRetry(fn, {
minTimeout: DEFAULT_INDEX_REFRESH_TIME,
retries: 5,
});
};

2
x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts
View file

@ -64,7 +64,7 @@ export const getThreatList = async ({
runtime_mappings: runtimeMappings,
sort: getSortForThreatList({
index,
listItemIndex: listClient.getListItemIndex(),
listItemIndex: listClient.getListItemName(),
}),
},
track_total_hits: false,

16
x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts
View file

@ -28,14 +28,14 @@ export default ({ getService }: FtrProviderContext) => {
await deleteListsIndex(supertest, log);
});
it('should create lists indices', async () => {
it('should create lists data streams', async () => {
const { body: fetchedIndices } = await supertest
.get(LIST_INDEX)
.set('kbn-xsrf', 'true')
.expect(404);
expect(fetchedIndices).to.eql({
message: 'index .lists-default and index .items-default does not exist',
message: 'data stream .lists-default and data stream .items-default does not exist',
status_code: 404,
});
@ -46,14 +46,15 @@ export default ({ getService }: FtrProviderContext) => {
expect(body).to.eql({ list_index: true, list_item_index: true });
});
it('should update lists indices if old legacy templates exists', async () => {
it('should migrate lists indices to data streams and remove old legacy templates', async () => {
// create legacy indices
await createLegacyListsIndices(es);
const { body: listsIndex } = await supertest
await supertest
.get(LIST_INDEX)
.set('kbn-xsrf', 'true')
.expect(200);
// data stream does not exist
.expect(404);
// confirm that legacy templates are in use
const legacyListsTemplateExists = await getTemplateExists(es, '.lists-default');
@ -65,10 +66,9 @@ export default ({ getService }: FtrProviderContext) => {
expect(legacyItemsTemplateExists).to.equal(true);
expect(nonLegacyListsTemplateExists).to.equal(false);
expect(nonLegacyItemsTemplateExists).to.equal(false);
expect(listsIndex).to.eql({ list_index: true, list_item_index: true });
// Expected 409 as index exists already, but now the templates should have been updated
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(409);
// migrates old indices to data streams
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200);
const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200);

6
x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts
View file

@ -7,7 +7,7 @@
import expect from '@kbn/expect';
import { LIST_URL, FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants';
import { LIST_URL, INTERNAL_FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants';
import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock';
import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock';
import { FtrProviderContext } from '../../common/ftr_provider_context';
@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should return an empty found body correctly if no lists are loaded', async () => {
const { body } = await supertest
.get(`${FIND_LISTS_BY_SIZE}`)
.get(`${INTERNAL_FIND_LISTS_BY_SIZE}`)
.set('kbn-xsrf', 'true')
.send()
.expect(200);
@ -63,7 +63,7 @@ export default ({ getService }: FtrProviderContext): void => {
// query the single list from _find_by_size
const { body } = await supertest
.get(`${FIND_LISTS_BY_SIZE}`)
.get(`${INTERNAL_FIND_LISTS_BY_SIZE}`)
.set('kbn-xsrf', 'true')
.send()
.expect(200);

6
x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts
View file

@ -8,7 +8,7 @@
import expect from '@kbn/expect';
import {
EXCEPTION_FILTER,
INTERNAL_EXCEPTION_FILTER,
EXCEPTION_LIST_URL,
EXCEPTION_LIST_ITEM_URL,
} from '@kbn/securitysolution-list-constants';
@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext): void => {
it('should return an exception filter if correctly passed exception items', async () => {
const { body } = await supertest
.post(`${EXCEPTION_FILTER}`)
.post(`${INTERNAL_EXCEPTION_FILTER}`)
.set('kbn-xsrf', 'true')
.send(getExceptionFilterFromExceptionItemsSchemaMock())
.expect(200);
@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext): void => {
})
.expect(200);
const { body } = await supertest
.post(`${EXCEPTION_FILTER}`)
.post(`${INTERNAL_EXCEPTION_FILTER}`)
.set('kbn-xsrf', 'true')
.send(getExceptionFilterFromExceptionIdsSchemaMock())
.expect(200);

34
x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts
View file

@ -19,12 +19,14 @@ import {
removeListServerGeneratedProperties,
removeListItemServerGeneratedProperties,
waitFor,
createListsIndices,
} from '../../utils';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext): void => {
const supertest = getService('supertest');
const log = getService('log');
const es = getService('es');
describe('import_list_items', () => {
describe('importing list items without an index', () => {
@ -39,7 +41,7 @@ export default ({ getService }: FtrProviderContext): void => {
expect(body).to.eql({
status_code: 400,
message:
'To import a list item, the index must exist first. Index ".lists-default" does not exist',
'To import a list item, the data steam must exist first. Data stream ".lists-default" does not exist',
});
});
});
@ -110,6 +112,36 @@ export default ({ getService }: FtrProviderContext): void => {
};
expect(bodyToCompare).to.eql(outputtedList);
});
describe('legacy index (before migration to data streams)', () => {
beforeEach(async () => {
await deleteListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should import list to legacy index and migrate it', async () => {
// create legacy indices
await createListsIndices(es);
const { body } = await supertest
.post(`${LIST_ITEM_URL}/_import?type=ip`)
.set('kbn-xsrf', 'true')
.attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.txt')
.expect('Content-Type', 'application/json; charset=utf-8')
.expect(200);
const bodyToCompare = removeListServerGeneratedProperties(body);
const outputtedList: Partial<ListSchema> = {
...getListResponseMockWithoutAutoGeneratedValues(),
name: 'list_items.txt',
description: 'File uploaded from file system of list_items.txt',
};
expect(bodyToCompare).to.eql(outputtedList);
});
});
});
});
};

2
x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts
View file

@ -13,6 +13,8 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
loadTestFile(require.resolve('./create_lists'));
loadTestFile(require.resolve('./create_lists_index'));
loadTestFile(require.resolve('./create_list_items'));
loadTestFile(require.resolve('./patch_lists'));
loadTestFile(require.resolve('./patch_list_items'));
loadTestFile(require.resolve('./read_lists'));
loadTestFile(require.resolve('./read_list_items'));
loadTestFile(require.resolve('./update_lists'));

281
x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts Normal file
View file

@ -0,0 +1,281 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import type {
PatchListItemSchema,
CreateListItemSchema,
ListItemSchema,
} from '@kbn/securitysolution-io-ts-list-types';
import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock';
import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock';
import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock';
import { getUpdateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_item_schema.mock';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import {
createListsIndex,
deleteListsIndex,
removeListItemServerGeneratedProperties,
createListsIndices,
createListBypassingChecks,
createListItemBypassingChecks,
} from '../../utils';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const log = getService('log');
const retry = getService('retry');
const es = getService('es');
describe('patch_list_items', () => {
describe('patch list items', () => {
beforeEach(async () => {
await createListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should patch a single list item property of value using an id', async () => {
const listItemId = getCreateMinimalListItemSchemaMock().id as string;
// create a simple list
await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// create a simple list item
await supertest
.post(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListItemSchemaMock())
.expect(200);
// patch a simple list item's value
const patchListItemPayload: PatchListItemSchema = {
id: listItemId,
value: '192.168.0.2',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchListItemPayload);
const outputListItem: Partial<ListItemSchema> = {
...getListItemResponseMockWithoutAutoGeneratedValues(),
value: '192.168.0.2',
};
const bodyToCompare = removeListItemServerGeneratedProperties(body);
expect(bodyToCompare).to.eql(outputListItem);
await retry.waitFor('updates should be persistent', async () => {
const { body: listItemBody } = await supertest
.get(LIST_ITEM_URL)
.query({ id: getCreateMinimalListItemSchemaMock().id })
.set('kbn-xsrf', 'true');
expect(removeListItemServerGeneratedProperties(listItemBody)).to.eql(outputListItem);
return true;
});
});
it('should patch a single list item of value using an auto-generated id of both list and list item', async () => {
const { id, ...listNoId } = getCreateMinimalListSchemaMock();
// create a simple list with no id which will use an auto-generated id
const { body: createListBody } = await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(listNoId)
.expect(200);
// create a simple list item also with an auto-generated id using the list's auto-generated id
const listItem: CreateListItemSchema = {
...getCreateMinimalListItemSchemaMock(),
list_id: createListBody.id,
};
const { body: createListItemBody } = await supertest
.post(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(listItem)
.expect(200);
// patch a simple list item's value
const patchListItemPayload: PatchListItemSchema = {
id: createListItemBody.id,
value: '192.168.0.2',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchListItemPayload)
.expect(200);
const outputListItem: Partial<ListItemSchema> = {
...getListItemResponseMockWithoutAutoGeneratedValues(),
value: '192.168.0.2',
};
const bodyToCompare = {
...removeListItemServerGeneratedProperties(body),
list_id: outputListItem.list_id,
};
expect(bodyToCompare).to.eql(outputListItem);
await retry.waitFor('updates should be persistent', async () => {
const { body: listItemBody } = await supertest
.get(LIST_ITEM_URL)
.query({ id: createListItemBody.id })
.set('kbn-xsrf', 'true');
const listItemBodyToCompare = {
...removeListItemServerGeneratedProperties(listItemBody),
list_id: outputListItem.list_id,
};
expect(listItemBodyToCompare).to.eql(outputListItem);
return true;
});
});
it('should not remove unspecified in patch payload meta property', async () => {
const listItemId = getCreateMinimalListItemSchemaMock().id as string;
// create a simple list
await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// create a simple list item
await supertest
.post(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send({ ...getCreateMinimalListItemSchemaMock(), meta: { test: true } })
.expect(200);
// patch a simple list item's value
const patchListItemPayload: PatchListItemSchema = {
id: listItemId,
value: '192.168.0.2',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchListItemPayload);
expect(body.meta).to.eql({ test: true });
await retry.waitFor('updates should be persistent', async () => {
const { body: listItemBody } = await supertest
.get(LIST_ITEM_URL)
.query({ id: getCreateMinimalListItemSchemaMock().id })
.set('kbn-xsrf', 'true');
expect(listItemBody.meta).to.eql({ test: true });
return true;
});
});
it('should give a 404 if it is given a fake id', async () => {
// create a simple list
await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// create a simple list item
await supertest
.post(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListItemSchemaMock())
.expect(200);
// patch a simple list item's value
const patchListItemPayload: PatchListItemSchema = {
...getUpdateMinimalListItemSchemaMock(),
id: 'some-other-id',
value: '192.168.0.2',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchListItemPayload)
.expect(404);
expect(body).to.eql({
status_code: 404,
message: 'list item id: "some-other-id" not found',
});
});
describe('legacy list items index (list created before migration to data stream)', () => {
beforeEach(async () => {
await deleteListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should patch list item that was created in legacy index and migrated through LIST_INDEX request', async () => {
const listId = 'random-list';
const listItemId = 'random-list-item';
// create legacy indices
await createListsIndices(es);
// create a simple list
await createListBypassingChecks({ es, id: listId });
await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' });
// migrates old indices to data streams
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true');
const patchPayload: PatchListItemSchema = {
id: listItemId,
value: 'new one',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchPayload)
.expect(200);
expect(body.value).to.be('new one');
});
it('should patch list item that was created in legacy index and not yet migrated', async () => {
const listId = 'random-list';
const listItemId = 'random-list-item';
// create legacy indices
await createListsIndices(es);
// create a simple list
await createListBypassingChecks({ es, id: listId });
await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' });
const patchPayload: PatchListItemSchema = {
id: listItemId,
value: 'new one',
};
const { body } = await supertest
.patch(LIST_ITEM_URL)
.set('kbn-xsrf', 'true')
.send(patchPayload)
.expect(200);
expect(body.value).to.be('new one');
});
});
});
});
};

271
x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts Normal file
View file

@ -0,0 +1,271 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import type { PatchListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types';
import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock';
import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock';
import { getUpdateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_schema.mock';
import {
createListsIndex,
deleteListsIndex,
removeListServerGeneratedProperties,
createListsIndices,
createListBypassingChecks,
} from '../../utils';
import { FtrProviderContext } from '../../common/ftr_provider_context';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const log = getService('log');
const retry = getService('retry');
const es = getService('es');
describe('patch_lists', () => {
describe('patch lists', () => {
beforeEach(async () => {
await createListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should patch a single list property of name using an id', async () => {
const listId = getCreateMinimalListSchemaMock().id as string;
// create a simple list
await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// patch a simple list's name
const patchedListPayload: PatchListSchema = {
id: listId,
name: 'some other name',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchedListPayload)
.expect(200);
const outputList: Partial<ListSchema> = {
...getListResponseMockWithoutAutoGeneratedValues(),
name: 'some other name',
version: 2,
};
const bodyToCompare = removeListServerGeneratedProperties(body);
expect(bodyToCompare).to.eql(outputList);
await retry.waitFor('patches should be persistent', async () => {
const { body: list } = await supertest
.get(LIST_URL)
.query({ id: listId })
.set('kbn-xsrf', 'true')
.expect(200);
expect(list.name).to.be('some other name');
return true;
});
});
it('should patch a single list property of name using an auto-generated id', async () => {
const { id, ...listNoId } = getCreateMinimalListSchemaMock();
// create a simple list with no id which will use an auto-generated id
const { body: createListBody } = await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(listNoId)
.expect(200);
// patch a simple list's name
const patchedListPayload: PatchListSchema = {
id: createListBody.id,
name: 'some other name',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchedListPayload)
.expect(200);
const outputList: Partial<ListSchema> = {
...getListResponseMockWithoutAutoGeneratedValues(),
name: 'some other name',
version: 2,
};
const bodyToCompare = removeListServerGeneratedProperties(body);
expect(bodyToCompare).to.eql(outputList);
await retry.waitFor('patches should be persistent', async () => {
const { body: list } = await supertest
.get(LIST_URL)
.query({ id: createListBody.id })
.set('kbn-xsrf', 'true');
expect(list.name).to.be('some other name');
return true;
});
});
it('should not remove unspecified fields in payload', async () => {
const listId = getCreateMinimalListSchemaMock().id as string;
// create a simple list
await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// patch a simple list's name
const patchedListPayload: PatchListSchema = {
id: listId,
name: 'some other name',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchedListPayload)
.expect(200);
const outputList: Partial<ListSchema> = {
...getListResponseMockWithoutAutoGeneratedValues(),
name: 'some other name',
version: 2,
};
const bodyToCompare = removeListServerGeneratedProperties(body);
expect(bodyToCompare).to.eql(outputList);
await retry.waitFor('patches should be persistent', async () => {
const { body: list } = await supertest
.get(LIST_URL)
.query({ id: getUpdateMinimalListSchemaMock().id })
.set('kbn-xsrf', 'true');
const persistentBodyToCompare = removeListServerGeneratedProperties(list);
expect(persistentBodyToCompare).to.eql(outputList);
return true;
});
});
it('should change the version of a list when it patches a property', async () => {
// create a simple list
const { body: createdList } = await supertest
.post(LIST_URL)
.set('kbn-xsrf', 'true')
.send(getCreateMinimalListSchemaMock())
.expect(200);
// patch a simple list property of name and description
const patchPayload: PatchListSchema = {
id: createdList.id,
name: 'some other name',
description: 'some other description',
};
const { body: patchedList } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchPayload);
expect(createdList.version).to.be(1);
expect(patchedList.version).to.be(2);
await retry.waitFor('patches should be persistent', async () => {
const { body: list } = await supertest
.get(LIST_URL)
.query({ id: patchedList.id })
.set('kbn-xsrf', 'true');
expect(list.version).to.be(2);
return true;
});
});
it('should give a 404 if it is given a fake id', async () => {
const simpleList: PatchListSchema = {
...getUpdateMinimalListSchemaMock(),
id: '5096dec6-b6b9-4d8d-8f93-6c2602079d9d',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(simpleList)
.expect(404);
expect(body).to.eql({
status_code: 404,
message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found',
});
});
describe('legacy list index (list created before migration to data stream)', () => {
beforeEach(async () => {
await deleteListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => {
const listId = 'random-list';
// create legacy indices
await createListsIndices(es);
// create a simple list
await createListBypassingChecks({ es, id: listId });
// migrates old indices to data streams
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true');
// patch a simple list's name
const patchPayload: PatchListSchema = {
id: listId,
name: 'some other name',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchPayload)
.expect(200);
expect(body.name).to.be('some other name');
});
it('should update list container that was created in legacy index and not yet migrated', async () => {
const listId = 'random-list';
// create legacy indices
await createListsIndices(es);
// create a simple list
await createListBypassingChecks({ es, id: listId });
// patch a simple list's name
const patchPayload: PatchListSchema = {
id: listId,
name: 'some other name',
};
const { body } = await supertest
.patch(LIST_URL)
.set('kbn-xsrf', 'true')
.send(patchPayload)
.expect(200);
expect(body.name).to.be('some other name');
});
});
});
});
};

Some files were not shown because too many files have changed in this diff Show more