mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[Security Solution][Detection Engine] move lists to data stream (#162508)
## Summary - addresses https://github.com/elastic/security-team/issues/7198 - moves list/items indices to data stream - adds `@timestamp` mapping to indices mappings - migrate to data stream if indices already exist(for customers < 8.11) or create data stream(for customers 8.11+ or serverless) - adds [DLM](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-streams-put-lifecycle.html) to index templates - replaces update/delete queries with update_by_query/delete_by_query which supported in data streams - fixes existing issues with update/patch APIs for lists/items - update/patch for lists didn't save `version` parameter in ES - update and patch APIs for lists/items were identical, i.e. for both routes was called the same `update` method w/o any changes <details> <summary>Technical detail on moving API to (update/delete)_by_query</summary> `update_by_query`, `delete_by_query` do not support refresh=wait_for, [only false/true values](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-update-by-query.html#_refreshing_shards_2). Which might break some of the use cases on UI(when list is removed, we refetch all lists. Deleted list will be returned for some time. [Default refresh time is 1s](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-refresh.html)). So, we retry refetching deleted/updated document before finishing request, to return reindexed document `update_by_query` does not support OCC [as update API](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/optimistic-concurrency-control.html). Which is supported in both [list](https://www.elastic.co/guide/en/security/current/lists-api-update-container.html)/[list item ](https://www.elastic.co/guide/en/security/current/lists-api-update-item.html)updates through _version parameter. _version is base64 encoded "_seq_no", "_primary_term" props used for OCC So, to keep it without breaking changes: implemented check for version conflict within update method </details> ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
parent
154ca404d0
commit
505d8265c8
104 changed files with 2607 additions and 767 deletions
|
@ -37,7 +37,7 @@ import {
|
|||
|
||||
import {
|
||||
ENDPOINT_LIST_URL,
|
||||
EXCEPTION_FILTER,
|
||||
INTERNAL_EXCEPTION_FILTER,
|
||||
EXCEPTION_LIST_ITEM_URL,
|
||||
EXCEPTION_LIST_URL,
|
||||
} from '@kbn/securitysolution-list-constants';
|
||||
|
@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({
|
|||
http,
|
||||
signal,
|
||||
}: GetExceptionFilterFromExceptionListIdsProps): Promise<ExceptionFilterResponse> =>
|
||||
http.fetch(EXCEPTION_FILTER, {
|
||||
http.fetch(INTERNAL_EXCEPTION_FILTER, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
exception_list_ids: exceptionListIds,
|
||||
|
@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({
|
|||
chunkSize,
|
||||
signal,
|
||||
}: GetExceptionFilterFromExceptionsProps): Promise<ExceptionFilterResponse> =>
|
||||
http.fetch(EXCEPTION_FILTER, {
|
||||
http.fetch(INTERNAL_EXCEPTION_FILTER, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
exceptions,
|
||||
|
|
|
@ -37,7 +37,7 @@ import {
|
|||
LIST_ITEM_URL,
|
||||
LIST_PRIVILEGES_URL,
|
||||
LIST_URL,
|
||||
FIND_LISTS_BY_SIZE,
|
||||
INTERNAL_FIND_LISTS_BY_SIZE,
|
||||
} from '@kbn/securitysolution-list-constants';
|
||||
import { toError, toPromise } from '../fp_utils';
|
||||
|
||||
|
@ -115,7 +115,7 @@ const findListsBySize = async ({
|
|||
per_page,
|
||||
signal,
|
||||
}: ApiParams & FindListSchemaEncoded): Promise<FoundListsBySizeSchema> => {
|
||||
return http.fetch(`${FIND_LISTS_BY_SIZE}`, {
|
||||
return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, {
|
||||
method: 'GET',
|
||||
query: {
|
||||
cursor,
|
||||
|
|
|
@ -23,6 +23,7 @@ import {
|
|||
} from '../constants.mock';
|
||||
|
||||
export const getListResponseMock = (): ListSchema => ({
|
||||
'@timestamp': DATE_NOW,
|
||||
_version: undefined,
|
||||
created_at: DATE_NOW,
|
||||
created_by: USER,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue