mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[Security Solution][Detection Engine] fixes lists/items API when @timestamp field is number (#210440)
## Summary - addresses https://github.com/elastic/security-team/issues/11831 **To Reproduce** 1. Create Security lists/items in 7.17 by uploading value list https://www.elastic.co/guide/en/security/current/value-lists-exceptions.html 2. Upgrade to 8.18 3. Visit detection engine page to ensure .lists-{SPACE} and .items-{SPACE} data streams have been created. Would be enough to lookup value lists in lists UI https://www.elastic.co/guide/en/security/current/value-lists-exceptions.html#edit-value-lists 4. Go to Kibana Upgrade assistant 5. Reindex .lists-{SPACE} and .items-{SPACE} data streams 6. After reindex lists are not retrievable with error `"data.0.@timestamp: Expected string, received number" ` through `/lists/_find` API **After fix** `@timestamp` of number type will be converted to ISO string **To test** use 8.18 mirror of this branch: https://github.com/elastic/kibana/pull/210439 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Vitalii Dmyterko
GitHub
parent
262969d15d
commit
fc5adc02fe
10 changed files with 104 additions and 9 deletions
|
|
@ -10,3 +10,8 @@ import { IsoDateString } from '@kbn/securitysolution-io-ts-types';
|
|||
|
||||
export const timestamp = IsoDateString;
|
||||
export const timestampOrUndefined = t.union([IsoDateString, t.undefined]);
|
||||
|
||||
/**
|
||||
* timestamp field type as it can be returned form ES: string, number or undefined
|
||||
*/
|
||||
export const timestampFromEsResponse = t.union([IsoDateString, t.number, t.undefined]);
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import {
|
|||
nullableMetaOrUndefined,
|
||||
serializerOrUndefined,
|
||||
tie_breaker_id,
|
||||
timestampOrUndefined,
|
||||
timestampFromEsResponse,
|
||||
updated_at,
|
||||
updated_by,
|
||||
} from '@kbn/securitysolution-io-ts-list-types';
|
||||
|
|
@ -47,7 +47,7 @@ import {
|
|||
|
||||
export const searchEsListItemSchema = t.exact(
|
||||
t.type({
|
||||
'@timestamp': timestampOrUndefined,
|
||||
'@timestamp': timestampFromEsResponse,
|
||||
binary: binaryOrUndefined,
|
||||
boolean: booleanOrUndefined,
|
||||
byte: byteOrUndefined,
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ export const getSearchEsListMock = (): SearchEsListSchema => ({
|
|||
version: VERSION,
|
||||
});
|
||||
|
||||
export const getSearchListMock = (): estypes.SearchResponse<SearchEsListSchema> => ({
|
||||
export const getSearchListMock = (
|
||||
source?: SearchEsListSchema
|
||||
): estypes.SearchResponse<SearchEsListSchema> => ({
|
||||
_scroll_id: '123',
|
||||
_shards: getShardMock(),
|
||||
hits: {
|
||||
|
|
@ -50,7 +52,7 @@ export const getSearchListMock = (): estypes.SearchResponse<SearchEsListSchema>
|
|||
_id: LIST_ID,
|
||||
_index: LIST_INDEX,
|
||||
_score: 0,
|
||||
_source: getSearchEsListMock(),
|
||||
_source: source || getSearchEsListMock(),
|
||||
},
|
||||
],
|
||||
max_score: 0,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import {
|
|||
nullableMetaOrUndefined,
|
||||
serializerOrUndefined,
|
||||
tie_breaker_id,
|
||||
timestampOrUndefined,
|
||||
timestampFromEsResponse,
|
||||
type,
|
||||
updated_at,
|
||||
updated_by,
|
||||
|
|
@ -25,7 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types';
|
|||
|
||||
export const searchEsListSchema = t.exact(
|
||||
t.type({
|
||||
'@timestamp': timestampOrUndefined,
|
||||
'@timestamp': timestampFromEsResponse,
|
||||
created_at,
|
||||
created_by,
|
||||
description,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/**
|
||||
* if date field is number, converts it to ISO string
|
||||
*/
|
||||
export const convertDateNumberToString = (
|
||||
dateValue: string | number | undefined
|
||||
): string | undefined => {
|
||||
if (typeof dateValue === 'number') {
|
||||
return new Date(dateValue).toISOString();
|
||||
}
|
||||
return dateValue;
|
||||
};
|
||||
|
|
@ -79,7 +79,7 @@ export const deserializeValue = ({
|
|||
deserializer: DeserializerOrUndefined;
|
||||
defaultValueDeserializer: string;
|
||||
defaultDeserializer: string;
|
||||
value: string | object | undefined;
|
||||
value: string | object | number | undefined;
|
||||
}): string | null => {
|
||||
if (esDataTypeRange.is(value)) {
|
||||
const template =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
getSearchEsListMock,
|
||||
getSearchListMock,
|
||||
} from '../../schemas/elastic_response/search_es_list_schema.mock';
|
||||
|
||||
import { transformElasticToList } from './transform_elastic_to_list';
|
||||
describe('transformElasticToList', () => {
|
||||
test('does not change timestamp in string format', () => {
|
||||
const response = getSearchListMock({
|
||||
...getSearchEsListMock(),
|
||||
'@timestamp': '2020-04-20T15:25:31.830Z',
|
||||
});
|
||||
|
||||
const result = transformElasticToList({
|
||||
response,
|
||||
});
|
||||
|
||||
expect(result[0]['@timestamp']).toBe('2020-04-20T15:25:31.830Z');
|
||||
});
|
||||
test('converts timestamp from number format to ISO string', () => {
|
||||
const response = getSearchListMock({ ...getSearchEsListMock(), '@timestamp': 0 });
|
||||
|
||||
const result = transformElasticToList({
|
||||
response,
|
||||
});
|
||||
|
||||
expect(result[0]['@timestamp']).toBe('1970-01-01T00:00:00.000Z');
|
||||
});
|
||||
});
|
||||
|
|
@ -11,6 +11,8 @@ import { encodeHitVersion } from '@kbn/securitysolution-es-utils';
|
|||
|
||||
import { SearchEsListSchema } from '../../schemas/elastic_response';
|
||||
|
||||
import { convertDateNumberToString } from './convert_date_number_to_string';
|
||||
|
||||
export interface TransformElasticToListOptions {
|
||||
response: estypes.SearchResponse<SearchEsListSchema>;
|
||||
}
|
||||
|
|
@ -24,6 +26,7 @@ export const transformElasticToList = ({
|
|||
_version: encodeHitVersion(hit),
|
||||
id: hit._id,
|
||||
...hit._source,
|
||||
'@timestamp': convertDateNumberToString(hit._source?.['@timestamp']),
|
||||
// meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null
|
||||
// return it as undefined to keep it consistent with payload
|
||||
meta: hit._source?.meta ?? undefined,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@
|
|||
import type { ListItemArraySchema } from '@kbn/securitysolution-io-ts-list-types';
|
||||
|
||||
import { getListItemResponseMock } from '../../../common/schemas/response/list_item_schema.mock';
|
||||
import { getSearchListItemMock } from '../../schemas/elastic_response/search_es_list_item_schema.mock';
|
||||
import {
|
||||
getSearchEsListItemMock,
|
||||
getSearchListItemMock,
|
||||
} from '../../schemas/elastic_response/search_es_list_item_schema.mock';
|
||||
|
||||
import {
|
||||
transformElasticHitsToListItem,
|
||||
|
|
@ -84,5 +87,32 @@ describe('transform_elastic_to_list_item', () => {
|
|||
const expected: ListItemArraySchema = [listItemResponse];
|
||||
expect(queryFilter).toEqual(expected);
|
||||
});
|
||||
|
||||
test('converts timestamp from number format to ISO string', () => {
|
||||
const hits = [{ _index: 'test', _source: { ...getSearchEsListItemMock(), '@timestamp': 0 } }];
|
||||
|
||||
const result = transformElasticHitsToListItem({
|
||||
hits,
|
||||
type: 'keyword',
|
||||
});
|
||||
|
||||
expect(result[0]['@timestamp']).toBe('1970-01-01T00:00:00.000Z');
|
||||
});
|
||||
|
||||
test('converts negative from number timestamp to ISO string', () => {
|
||||
const hits = [
|
||||
{
|
||||
_index: 'test',
|
||||
_source: { ...getSearchEsListItemMock(), '@timestamp': -63549289600000 },
|
||||
},
|
||||
];
|
||||
|
||||
const result = transformElasticHitsToListItem({
|
||||
hits,
|
||||
type: 'keyword',
|
||||
});
|
||||
|
||||
expect(result[0]['@timestamp']).toBe('-000044-03-15T19:33:20.000Z');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import { ErrorWithStatusCode } from '../../error_with_status_code';
|
|||
import { SearchEsListItemSchema } from '../../schemas/elastic_response';
|
||||
|
||||
import { findSourceValue } from './find_source_value';
|
||||
import { convertDateNumberToString } from './convert_date_number_to_string';
|
||||
|
||||
export interface TransformElasticToListItemOptions {
|
||||
response: estypes.SearchResponse<SearchEsListItemSchema>;
|
||||
|
|
@ -56,7 +57,7 @@ export const transformElasticHitsToListItem = ({
|
|||
throw new ErrorWithStatusCode(`Was expected ${type} to not be null/undefined`, 400);
|
||||
} else {
|
||||
return {
|
||||
'@timestamp': _source?.['@timestamp'],
|
||||
'@timestamp': convertDateNumberToString(_source?.['@timestamp']),
|
||||
_version: encodeHitVersion(hit),
|
||||
created_at,
|
||||
created_by,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue