github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 10:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Response Ops][Reporting] Fixing timestamp override for scheduled CSV reports (#224757)

Browse source 
## Summary

PDF, PNG and ES|QL CSV reports all use a relative date range based on
`now` so when we generate recurring exports, we override `now` with a
`forceNow` parameter. Non ES|QL CSV reports use a `SearchSource` with a
fixed time range, even when a relative time range is set in Discover.

This PR updates the CSV search source report generation to override the
fixed time range for recurring scheduled exports.

## To Verify

- create a dataview (trying creating one using a field other than
`@timestamp` as the time field)
- populate the dataview with some data
- schedule a CSV export and verify that the eventual CSV report has data
in the correct time range
- may be faster to schedule via the API to get a report generated
faster.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Ying Mao 2025-06-26 15:40:46 -04:00 • committed by GitHub
parent d38801034a
commit 9f6eb0a0cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 743 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

21
src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts
View file

@ -9,7 +9,7 @@
import moment from 'moment';
import type { Writable } from 'stream';
import type { Filter } from '@kbn/es-query';
import { errors as esErrors, estypes } from '@elastic/elasticsearch';
import type { IScopedClusterClient, IUiSettingsClient, Logger } from '@kbn/core/server';
import type { ISearchClient } from '@kbn/search-types';
@ -34,6 +34,7 @@ import type { ReportingConfigType } from '@kbn/reporting-server';
import { TaskErrorSource, createTaskRunError } from '@kbn/task-manager-plugin/server';
import { CONTENT_TYPE_CSV } from '../constants';
import type { JobParamsCSV } from '../types';
import { overrideTimeRange } from './lib/override_time_range';
import { getExportSettings, type CsvExportSettings } from './lib/get_export_settings';
import { i18nTexts } from './lib/i18n_texts';
import { MaxSizeStringBuilder } from './lib/max_size_string_builder';
@ -277,6 +278,24 @@ export class CsvGenerator {
throw new Error(`The search must have a reference to an index pattern!`);
}
if (this.job.forceNow) {
this.logger.debug(`Overriding time range filter using forceNow: ${this.job.forceNow}`);
const currentFilters = searchSource.getField('filter') as Filter[] | Filter | undefined;
this.logger.debug(() => `Current filters: ${JSON.stringify(currentFilters)}`);
const updatedFilters = overrideTimeRange({
currentFilters,
forceNow: this.job.forceNow,
logger: this.logger,
});
this.logger.debug(() => `Updated filters: ${JSON.stringify(updatedFilters)}`);
if (updatedFilters) {
searchSource.removeField('filter'); // remove existing filters
searchSource.setField('filter', updatedFilters);
}
}
const { maxSizeBytes, bom, escapeFormulaValues, timezone } = settings;
const indexPatternTitle = index.getIndexPattern();
const builder = new MaxSizeStringBuilder(this.stream, byteSizeValueToNumber(maxSizeBytes), bom);

609
src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts Normal file
View file

@ -0,0 +1,609 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { overrideTimeRange } from './override_time_range';
import { loggingSystemMock } from '@kbn/core/server/mocks';
const mockLogger = loggingSystemMock.createLogger();
describe('overrideTimeRange', () => {
it('should return modified time range filter', () => {
const filter = {
meta: {
field: '@timestamp',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
};
const updated = overrideTimeRange({
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toEqual([
{
meta: {
field: '@timestamp',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2025-06-18T19:30:00.000Z',
lte: '2025-06-18T19:55:00.000Z',
},
},
},
},
]);
});
it('should return modified time range in filter array', () => {
const filter = [
{
meta: {
field: '@timestamp',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
];
const updated = overrideTimeRange({
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toEqual([
{
meta: {
field: '@timestamp',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2025-06-18T19:30:00.000Z',
lte: '2025-06-18T19:55:00.000Z',
},
},
},
},
]);
});
it('should return modified time range in the filter array when timestamp field is not @timestamp', () => {
const filter = [
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
];
const updated = overrideTimeRange({
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toEqual([
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-06-18T19:30:00.000Z',
lte: '2025-06-18T19:55:00.000Z',
},
},
},
},
]);
});
it('should maintain the same filter order', () => {
const filter = [
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'event.action',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'event.action',
negate: false,
params: ['a', 'b', 'c'],
type: 'phrases',
value: ['a', 'b', 'c'],
},
query: {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'event.action': 'a',
},
},
{
match_phrase: {
'event.action': 'b',
},
},
{
match_phrase: {
'event.action': 'c',
},
},
],
},
},
},
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'another.range.field',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'another.range.field',
negate: false,
params: {
gte: '0',
lt: '10',
},
type: 'range',
value: {
gte: '0',
lt: '10',
},
},
query: {
range: {
'another.range.field': {
gte: '0',
lt: '10',
},
},
},
},
];
const updated = overrideTimeRange({
// @ts-expect-error
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toEqual([
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'event.action',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'event.action',
negate: false,
params: ['a', 'b', 'c'],
type: 'phrases',
value: ['a', 'b', 'c'],
},
query: {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'event.action': 'a',
},
},
{
match_phrase: {
'event.action': 'b',
},
},
{
match_phrase: {
'event.action': 'c',
},
},
],
},
},
},
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-06-18T19:30:00.000Z',
lte: '2025-06-18T19:55:00.000Z',
},
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'another.range.field',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'another.range.field',
negate: false,
params: {
gte: '0',
lt: '10',
},
type: 'range',
value: {
gte: '0',
lt: '10',
},
},
query: {
range: {
'another.range.field': {
gte: '0',
lt: '10',
},
},
},
},
]);
});
it('should return modified time range in the filter array range filters are present', () => {
const filter = [
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'event.action',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'event.action',
negate: false,
params: ['a', 'b', 'c'],
type: 'phrases',
value: ['a', 'b', 'c'],
},
query: {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'event.action': 'a',
},
},
{
match_phrase: {
'event.action': 'b',
},
},
{
match_phrase: {
'event.action': 'c',
},
},
],
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'another.range.field',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'another.range.field',
negate: false,
params: {
gte: '0',
lt: '10',
},
type: 'range',
value: {
gte: '0',
lt: '10',
},
},
query: {
range: {
'another.range.field': {
gte: '0',
lt: '10',
},
},
},
},
];
const updated = overrideTimeRange({
// @ts-expect-error
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toEqual([
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'event.start': {
format: 'strict_date_optional_time',
gte: '2025-06-18T19:30:00.000Z',
lte: '2025-06-18T19:55:00.000Z',
},
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'event.action',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'event.action',
negate: false,
params: ['a', 'b', 'c'],
type: 'phrases',
value: ['a', 'b', 'c'],
},
query: {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'event.action': 'a',
},
},
{
match_phrase: {
'event.action': 'b',
},
},
{
match_phrase: {
'event.action': 'c',
},
},
],
},
},
},
{
$state: {
store: 'appState',
},
meta: {
alias: null,
disabled: false,
field: 'another.range.field',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
key: 'another.range.field',
negate: false,
params: {
gte: '0',
lt: '10',
},
type: 'range',
value: {
gte: '0',
lt: '10',
},
},
query: {
range: {
'another.range.field': {
gte: '0',
lt: '10',
},
},
},
},
]);
});
it('should return undefined if unexpected time filter found', () => {
const filter = [
{
meta: {
field: 'event.start',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'another.field': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
];
const updated = overrideTimeRange({
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toBeUndefined();
});
it('should return undefined if no meta field found', () => {
const filter = [
{
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2025-01-01T19:38:24.286Z',
lte: '2025-01-01T20:03:24.286Z',
},
},
},
},
];
const updated = overrideTimeRange({
// @ts-expect-error missing meta field
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
});
expect(updated).toBeUndefined();
});
it('should return undefined if invalid time', () => {
const filter = [
{
meta: {
field: '@timestamp',
index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
params: {},
},
query: {
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: 'foo',
lte: 'bar',
},
},
},
},
];
const updated = overrideTimeRange({
currentFilters: filter,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toBeUndefined();
});
it('should return undefined for undefined filters', () => {
const updated = overrideTimeRange({
currentFilters: undefined,
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toBeUndefined();
});
it('should return undefined for empty filters', () => {
const updated = overrideTimeRange({
currentFilters: [],
forceNow: '2025-06-18T19:55:00.000Z',
logger: mockLogger,
});
expect(updated).toBeUndefined();
});
});

110
src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Filter } from '@kbn/es-query';
import { set } from '@kbn/safer-lodash-set';
import type { Logger } from '@kbn/core/server';
import { cloneDeep, get, has, isArray } from 'lodash';
const getTimeFieldAccessorString = (metaField: string): string => `query.range['${metaField}']`;
const getTimeFields = (filter: Filter) => {
const metaField = get(filter, 'meta.field');
if (metaField) {
const timeFieldAccessorString = getTimeFieldAccessorString(metaField);
const timeFormat = get(filter, `${timeFieldAccessorString}.format`);
const timeGte = get(filter, `${timeFieldAccessorString}.gte`);
const timeLte = get(filter, `${timeFieldAccessorString}.lte`);
return { metaField, timeFormat, timeGte, timeLte };
}
return {};
};
const isValidDateTime = (dateString: string): boolean => {
const date = Date.parse(dateString);
return !isNaN(date) && date > 0;
};
interface OverrideTimeRangeOpts {
currentFilters: Filter[] | Filter | undefined;
forceNow: string;
logger: Logger;
}
export const overrideTimeRange = ({
currentFilters,
forceNow,
logger,
}: OverrideTimeRangeOpts): Filter[] | undefined => {
if (!currentFilters) {
return;
}
const filters = isArray(currentFilters) ? currentFilters : [currentFilters];
if (filters.length === 0) {
return;
}
// Looking for filters with this format which indicate a time range:
// {
// "meta": {
// "field": <timeFieldName>,
// "index": <indexId>,
// "params": {}
// },
// "query": {
// "range": {
// <timeFieldName>: {
// "format": "strict_date_optional_time",
// "gte": "2025-06-18T18:29:53.537Z",
// "lte": "2025-06-18T18:54:53.537Z"
// }
// }
// }
// }
const timeFilterIndex = filters.findIndex((filter) => {
if (has(filter, '$state')) {
return false;
}
const {
timeFormat: maybeTimeFieldFormat,
timeGte: maybeTimeFieldGte,
timeLte: maybeTimeFieldLte,
} = getTimeFields(filter);
if (maybeTimeFieldFormat && maybeTimeFieldGte && maybeTimeFieldLte) {
return isValidDateTime(maybeTimeFieldGte) && isValidDateTime(maybeTimeFieldLte);
}
return false;
});
if (timeFilterIndex >= 0) {
try {
const timeFilter = cloneDeep(filters[timeFilterIndex]);
const { metaField, timeGte, timeLte } = getTimeFields(timeFilter);
if (metaField) {
const timeGteMs = Date.parse(timeGte);
const timeLteMs = Date.parse(timeLte);
const timeDiffMs = timeLteMs - timeGteMs;
const newLte = Date.parse(forceNow);
const newGte = newLte - timeDiffMs;
const timeFieldAccessorString = getTimeFieldAccessorString(metaField);
set(timeFilter, `${timeFieldAccessorString}.gte`, new Date(newGte).toISOString());
set(timeFilter, `${timeFieldAccessorString}.lte`, forceNow);
filters.splice(timeFilterIndex, 1, timeFilter);
return filters;
}
} catch (error) {
logger.warn(`Error calculating updated time range: ${error.message}`);
}
}
};

2
src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts
View file

@ -124,6 +124,8 @@ export class SearchCursorPit extends SearchCursor {
throw new Error('Could not retrieve the search body!');
}
this.logger.debug(() => `Executing search with body: ${JSON.stringify(searchBody)}`);
const response = await this.searchWithPit(searchBody);
if (!response) {

1
src/platform/packages/private/kbn-generate-csv/tsconfig.json
View file

@ -32,5 +32,6 @@
"@kbn/search-types",
"@kbn/task-manager-plugin",
"@kbn/esql-utils",
"@kbn/safer-lodash-set",
]
}

1
src/platform/packages/private/kbn-generate-csv/types.ts
View file

@ -17,5 +17,6 @@ export interface JobParamsCSV {
browserTimezone?: string;
searchSource: SerializedSearchSourceFields;
columns?: string[];
forceNow?: string;
pagingStrategy?: CsvPagingStrategy;
}