From 9f6eb0a0cb7408a614335773240e3b360c9eceee Mon Sep 17 00:00:00 2001 From: Ying Mao Date: Thu, 26 Jun 2025 15:40:46 -0400 Subject: [PATCH] [Response Ops][Reporting] Fixing timestamp override for scheduled CSV reports (#224757) ## Summary PDF, PNG and ES|QL CSV reports all use a relative date range based on `now` so when we generate recurring exports, we override `now` with a `forceNow` parameter. Non ES|QL CSV reports use a `SearchSource` with a fixed time range, even when a relative time range is set in Discover. This PR updates the CSV search source report generation to override the fixed time range for recurring scheduled exports. ## To Verify - create a dataview (trying creating one using a field other than `@timestamp` as the time field) - populate the dataview with some data - schedule a CSV export and verify that the eventual CSV report has data in the correct time range - may be faster to schedule via the API to get a report generated faster. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Elastic Machine --- .../kbn-generate-csv/src/generate_csv.ts | 21 +- .../src/lib/override_time_range.test.ts | 609 ++++++++++++++++++ .../src/lib/override_time_range.ts | 110 ++++ .../src/lib/search_cursor_pit.ts | 2 + .../private/kbn-generate-csv/tsconfig.json | 1 + .../private/kbn-generate-csv/types.ts | 1 + 6 files changed, 743 insertions(+), 1 deletion(-) create mode 100644 src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts create mode 100644 src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts diff --git a/src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts b/src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts index 5ed92df84c58..4f5c5f4f6a69 100644 --- a/src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts +++ b/src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts @@ -9,7 +9,7 @@ import moment from 'moment'; import type { Writable } from 'stream'; - +import type { Filter } from '@kbn/es-query'; import { errors as esErrors, estypes } from '@elastic/elasticsearch'; import type { IScopedClusterClient, IUiSettingsClient, Logger } from '@kbn/core/server'; import type { ISearchClient } from '@kbn/search-types'; @@ -34,6 +34,7 @@ import type { ReportingConfigType } from '@kbn/reporting-server'; import { TaskErrorSource, createTaskRunError } from '@kbn/task-manager-plugin/server'; import { CONTENT_TYPE_CSV } from '../constants'; import type { JobParamsCSV } from '../types'; +import { overrideTimeRange } from './lib/override_time_range'; import { getExportSettings, type CsvExportSettings } from './lib/get_export_settings'; import { i18nTexts } from './lib/i18n_texts'; import { MaxSizeStringBuilder } from './lib/max_size_string_builder'; @@ -277,6 +278,24 @@ export class CsvGenerator { throw new Error(`The search must have a reference to an index pattern!`); } + if (this.job.forceNow) { + this.logger.debug(`Overriding time range filter using forceNow: ${this.job.forceNow}`); + + const currentFilters = searchSource.getField('filter') as Filter[] | Filter | undefined; + this.logger.debug(() => `Current filters: ${JSON.stringify(currentFilters)}`); + const updatedFilters = overrideTimeRange({ + currentFilters, + forceNow: this.job.forceNow, + logger: this.logger, + }); + this.logger.debug(() => `Updated filters: ${JSON.stringify(updatedFilters)}`); + + if (updatedFilters) { + searchSource.removeField('filter'); // remove existing filters + searchSource.setField('filter', updatedFilters); + } + } + const { maxSizeBytes, bom, escapeFormulaValues, timezone } = settings; const indexPatternTitle = index.getIndexPattern(); const builder = new MaxSizeStringBuilder(this.stream, byteSizeValueToNumber(maxSizeBytes), bom); diff --git a/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts b/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts new file mode 100644 index 000000000000..577cac6f7aa5 --- /dev/null +++ b/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts @@ -0,0 +1,609 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { overrideTimeRange } from './override_time_range'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; + +const mockLogger = loggingSystemMock.createLogger(); + +describe('overrideTimeRange', () => { + it('should return modified time range filter', () => { + const filter = { + meta: { + field: '@timestamp', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }; + + const updated = overrideTimeRange({ + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toEqual([ + { + meta: { + field: '@timestamp', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2025-06-18T19:30:00.000Z', + lte: '2025-06-18T19:55:00.000Z', + }, + }, + }, + }, + ]); + }); + + it('should return modified time range in filter array', () => { + const filter = [ + { + meta: { + field: '@timestamp', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toEqual([ + { + meta: { + field: '@timestamp', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2025-06-18T19:30:00.000Z', + lte: '2025-06-18T19:55:00.000Z', + }, + }, + }, + }, + ]); + }); + + it('should return modified time range in the filter array when timestamp field is not @timestamp', () => { + const filter = [ + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toEqual([ + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-06-18T19:30:00.000Z', + lte: '2025-06-18T19:55:00.000Z', + }, + }, + }, + }, + ]); + }); + + it('should maintain the same filter order', () => { + const filter = [ + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'event.action', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'event.action', + negate: false, + params: ['a', 'b', 'c'], + type: 'phrases', + value: ['a', 'b', 'c'], + }, + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'event.action': 'a', + }, + }, + { + match_phrase: { + 'event.action': 'b', + }, + }, + { + match_phrase: { + 'event.action': 'c', + }, + }, + ], + }, + }, + }, + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'another.range.field', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'another.range.field', + negate: false, + params: { + gte: '0', + lt: '10', + }, + type: 'range', + value: { + gte: '0', + lt: '10', + }, + }, + query: { + range: { + 'another.range.field': { + gte: '0', + lt: '10', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + // @ts-expect-error + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toEqual([ + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'event.action', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'event.action', + negate: false, + params: ['a', 'b', 'c'], + type: 'phrases', + value: ['a', 'b', 'c'], + }, + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'event.action': 'a', + }, + }, + { + match_phrase: { + 'event.action': 'b', + }, + }, + { + match_phrase: { + 'event.action': 'c', + }, + }, + ], + }, + }, + }, + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-06-18T19:30:00.000Z', + lte: '2025-06-18T19:55:00.000Z', + }, + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'another.range.field', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'another.range.field', + negate: false, + params: { + gte: '0', + lt: '10', + }, + type: 'range', + value: { + gte: '0', + lt: '10', + }, + }, + query: { + range: { + 'another.range.field': { + gte: '0', + lt: '10', + }, + }, + }, + }, + ]); + }); + + it('should return modified time range in the filter array range filters are present', () => { + const filter = [ + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'event.action', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'event.action', + negate: false, + params: ['a', 'b', 'c'], + type: 'phrases', + value: ['a', 'b', 'c'], + }, + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'event.action': 'a', + }, + }, + { + match_phrase: { + 'event.action': 'b', + }, + }, + { + match_phrase: { + 'event.action': 'c', + }, + }, + ], + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'another.range.field', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'another.range.field', + negate: false, + params: { + gte: '0', + lt: '10', + }, + type: 'range', + value: { + gte: '0', + lt: '10', + }, + }, + query: { + range: { + 'another.range.field': { + gte: '0', + lt: '10', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + // @ts-expect-error + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toEqual([ + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'event.start': { + format: 'strict_date_optional_time', + gte: '2025-06-18T19:30:00.000Z', + lte: '2025-06-18T19:55:00.000Z', + }, + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'event.action', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'event.action', + negate: false, + params: ['a', 'b', 'c'], + type: 'phrases', + value: ['a', 'b', 'c'], + }, + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'event.action': 'a', + }, + }, + { + match_phrase: { + 'event.action': 'b', + }, + }, + { + match_phrase: { + 'event.action': 'c', + }, + }, + ], + }, + }, + }, + { + $state: { + store: 'appState', + }, + meta: { + alias: null, + disabled: false, + field: 'another.range.field', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + key: 'another.range.field', + negate: false, + params: { + gte: '0', + lt: '10', + }, + type: 'range', + value: { + gte: '0', + lt: '10', + }, + }, + query: { + range: { + 'another.range.field': { + gte: '0', + lt: '10', + }, + }, + }, + }, + ]); + }); + + it('should return undefined if unexpected time filter found', () => { + const filter = [ + { + meta: { + field: 'event.start', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + 'another.field': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toBeUndefined(); + }); + + it('should return undefined if no meta field found', () => { + const filter = [ + { + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2025-01-01T19:38:24.286Z', + lte: '2025-01-01T20:03:24.286Z', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + // @ts-expect-error missing meta field + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + }); + expect(updated).toBeUndefined(); + }); + + it('should return undefined if invalid time', () => { + const filter = [ + { + meta: { + field: '@timestamp', + index: '0bde9920-4ade-4c19-8043-368aa37f1dae', + params: {}, + }, + query: { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: 'foo', + lte: 'bar', + }, + }, + }, + }, + ]; + + const updated = overrideTimeRange({ + currentFilters: filter, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toBeUndefined(); + }); + + it('should return undefined for undefined filters', () => { + const updated = overrideTimeRange({ + currentFilters: undefined, + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toBeUndefined(); + }); + + it('should return undefined for empty filters', () => { + const updated = overrideTimeRange({ + currentFilters: [], + forceNow: '2025-06-18T19:55:00.000Z', + logger: mockLogger, + }); + expect(updated).toBeUndefined(); + }); +}); diff --git a/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts b/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts new file mode 100644 index 000000000000..0749685daa52 --- /dev/null +++ b/src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Filter } from '@kbn/es-query'; +import { set } from '@kbn/safer-lodash-set'; +import type { Logger } from '@kbn/core/server'; +import { cloneDeep, get, has, isArray } from 'lodash'; + +const getTimeFieldAccessorString = (metaField: string): string => `query.range['${metaField}']`; +const getTimeFields = (filter: Filter) => { + const metaField = get(filter, 'meta.field'); + if (metaField) { + const timeFieldAccessorString = getTimeFieldAccessorString(metaField); + const timeFormat = get(filter, `${timeFieldAccessorString}.format`); + const timeGte = get(filter, `${timeFieldAccessorString}.gte`); + const timeLte = get(filter, `${timeFieldAccessorString}.lte`); + + return { metaField, timeFormat, timeGte, timeLte }; + } + + return {}; +}; + +const isValidDateTime = (dateString: string): boolean => { + const date = Date.parse(dateString); + return !isNaN(date) && date > 0; +}; + +interface OverrideTimeRangeOpts { + currentFilters: Filter[] | Filter | undefined; + forceNow: string; + logger: Logger; +} +export const overrideTimeRange = ({ + currentFilters, + forceNow, + logger, +}: OverrideTimeRangeOpts): Filter[] | undefined => { + if (!currentFilters) { + return; + } + + const filters = isArray(currentFilters) ? currentFilters : [currentFilters]; + if (filters.length === 0) { + return; + } + + // Looking for filters with this format which indicate a time range: + // { + // "meta": { + // "field": , + // "index": , + // "params": {} + // }, + // "query": { + // "range": { + // : { + // "format": "strict_date_optional_time", + // "gte": "2025-06-18T18:29:53.537Z", + // "lte": "2025-06-18T18:54:53.537Z" + // } + // } + // } + // } + const timeFilterIndex = filters.findIndex((filter) => { + if (has(filter, '$state')) { + return false; + } + + const { + timeFormat: maybeTimeFieldFormat, + timeGte: maybeTimeFieldGte, + timeLte: maybeTimeFieldLte, + } = getTimeFields(filter); + + if (maybeTimeFieldFormat && maybeTimeFieldGte && maybeTimeFieldLte) { + return isValidDateTime(maybeTimeFieldGte) && isValidDateTime(maybeTimeFieldLte); + } + return false; + }); + + if (timeFilterIndex >= 0) { + try { + const timeFilter = cloneDeep(filters[timeFilterIndex]); + const { metaField, timeGte, timeLte } = getTimeFields(timeFilter); + if (metaField) { + const timeGteMs = Date.parse(timeGte); + const timeLteMs = Date.parse(timeLte); + const timeDiffMs = timeLteMs - timeGteMs; + const newLte = Date.parse(forceNow); + const newGte = newLte - timeDiffMs; + + const timeFieldAccessorString = getTimeFieldAccessorString(metaField); + set(timeFilter, `${timeFieldAccessorString}.gte`, new Date(newGte).toISOString()); + set(timeFilter, `${timeFieldAccessorString}.lte`, forceNow); + + filters.splice(timeFilterIndex, 1, timeFilter); + return filters; + } + } catch (error) { + logger.warn(`Error calculating updated time range: ${error.message}`); + } + } +}; diff --git a/src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts b/src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts index fc0a78f2de83..edd1a029b36d 100644 --- a/src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts +++ b/src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts @@ -124,6 +124,8 @@ export class SearchCursorPit extends SearchCursor { throw new Error('Could not retrieve the search body!'); } + this.logger.debug(() => `Executing search with body: ${JSON.stringify(searchBody)}`); + const response = await this.searchWithPit(searchBody); if (!response) { diff --git a/src/platform/packages/private/kbn-generate-csv/tsconfig.json b/src/platform/packages/private/kbn-generate-csv/tsconfig.json index e781e93336a6..996e0ab3d40b 100644 --- a/src/platform/packages/private/kbn-generate-csv/tsconfig.json +++ b/src/platform/packages/private/kbn-generate-csv/tsconfig.json @@ -32,5 +32,6 @@ "@kbn/search-types", "@kbn/task-manager-plugin", "@kbn/esql-utils", + "@kbn/safer-lodash-set", ] } diff --git a/src/platform/packages/private/kbn-generate-csv/types.ts b/src/platform/packages/private/kbn-generate-csv/types.ts index 0497f7c153c2..a7f78726be34 100644 --- a/src/platform/packages/private/kbn-generate-csv/types.ts +++ b/src/platform/packages/private/kbn-generate-csv/types.ts @@ -17,5 +17,6 @@ export interface JobParamsCSV { browserTimezone?: string; searchSource: SerializedSearchSourceFields; columns?: string[]; + forceNow?: string; pagingStrategy?: CsvPagingStrategy; }