[Response Ops][Reporting] Fixing timestamp override for scheduled CSV reports (#224757)

## Summary

PDF, PNG and ES|QL CSV reports all use a relative date range based on
`now` so when we generate recurring exports, we override `now` with a
`forceNow` parameter. Non ES|QL CSV reports use a `SearchSource` with a
fixed time range, even when a relative time range is set in Discover.

This PR updates the CSV search source report generation to override the
fixed time range for recurring scheduled exports.

## To Verify

- create a dataview (trying creating one using a field other than
`@timestamp` as the time field)
- populate the dataview with some data
- schedule a CSV export and verify that the eventual CSV report has data
in the correct time range
- may be faster to schedule via the API to get a report generated
faster.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/platform/packages/private/kbn-generate-csv/src/generate_csv.ts

@@ -9,7 +9,7 @@
 
 import moment from 'moment';
 import type { Writable } from 'stream';
-
+import type { Filter } from '@kbn/es-query';
 import { errors as esErrors, estypes } from '@elastic/elasticsearch';
 import type { IScopedClusterClient, IUiSettingsClient, Logger } from '@kbn/core/server';
 import type { ISearchClient } from '@kbn/search-types';
@@ -34,6 +34,7 @@ import type { ReportingConfigType } from '@kbn/reporting-server';
 import { TaskErrorSource, createTaskRunError } from '@kbn/task-manager-plugin/server';
 import { CONTENT_TYPE_CSV } from '../constants';
 import type { JobParamsCSV } from '../types';
+import { overrideTimeRange } from './lib/override_time_range';
 import { getExportSettings, type CsvExportSettings } from './lib/get_export_settings';
 import { i18nTexts } from './lib/i18n_texts';
 import { MaxSizeStringBuilder } from './lib/max_size_string_builder';
@@ -277,6 +278,24 @@ export class CsvGenerator {
       throw new Error(`The search must have a reference to an index pattern!`);
     }
 
+    if (this.job.forceNow) {
+      this.logger.debug(`Overriding time range filter using forceNow: ${this.job.forceNow}`);
+
+      const currentFilters = searchSource.getField('filter') as Filter[] | Filter | undefined;
+      this.logger.debug(() => `Current filters: ${JSON.stringify(currentFilters)}`);
+      const updatedFilters = overrideTimeRange({
+        currentFilters,
+        forceNow: this.job.forceNow,
+        logger: this.logger,
+      });
+      this.logger.debug(() => `Updated filters: ${JSON.stringify(updatedFilters)}`);
+
+      if (updatedFilters) {
+        searchSource.removeField('filter'); // remove existing filters
+        searchSource.setField('filter', updatedFilters);
+      }
+    }
+
     const { maxSizeBytes, bom, escapeFormulaValues, timezone } = settings;
     const indexPatternTitle = index.getIndexPattern();
     const builder = new MaxSizeStringBuilder(this.stream, byteSizeValueToNumber(maxSizeBytes), bom);

src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.test.ts

@@ -0,0 +1,609 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { overrideTimeRange } from './override_time_range';
+import { loggingSystemMock } from '@kbn/core/server/mocks';
+
+const mockLogger = loggingSystemMock.createLogger();
+
+describe('overrideTimeRange', () => {
+  it('should return modified time range filter', () => {
+    const filter = {
+      meta: {
+        field: '@timestamp',
+        index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+        params: {},
+      },
+      query: {
+        range: {
+          '@timestamp': {
+            format: 'strict_date_optional_time',
+            gte: '2025-01-01T19:38:24.286Z',
+            lte: '2025-01-01T20:03:24.286Z',
+          },
+        },
+      },
+    };
+
+    const updated = overrideTimeRange({
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toEqual([
+      {
+        meta: {
+          field: '@timestamp',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: '2025-06-18T19:30:00.000Z',
+              lte: '2025-06-18T19:55:00.000Z',
+            },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('should return modified time range in filter array', () => {
+    const filter = [
+      {
+        meta: {
+          field: '@timestamp',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toEqual([
+      {
+        meta: {
+          field: '@timestamp',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: '2025-06-18T19:30:00.000Z',
+              lte: '2025-06-18T19:55:00.000Z',
+            },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('should return modified time range in the filter array when timestamp field is not @timestamp', () => {
+    const filter = [
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toEqual([
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-06-18T19:30:00.000Z',
+              lte: '2025-06-18T19:55:00.000Z',
+            },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('should maintain the same filter order', () => {
+    const filter = [
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'event.action',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'event.action',
+          negate: false,
+          params: ['a', 'b', 'c'],
+          type: 'phrases',
+          value: ['a', 'b', 'c'],
+        },
+        query: {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_phrase: {
+                  'event.action': 'a',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'b',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'c',
+                },
+              },
+            ],
+          },
+        },
+      },
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'another.range.field',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'another.range.field',
+          negate: false,
+          params: {
+            gte: '0',
+            lt: '10',
+          },
+          type: 'range',
+          value: {
+            gte: '0',
+            lt: '10',
+          },
+        },
+        query: {
+          range: {
+            'another.range.field': {
+              gte: '0',
+              lt: '10',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      // @ts-expect-error
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toEqual([
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'event.action',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'event.action',
+          negate: false,
+          params: ['a', 'b', 'c'],
+          type: 'phrases',
+          value: ['a', 'b', 'c'],
+        },
+        query: {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_phrase: {
+                  'event.action': 'a',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'b',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'c',
+                },
+              },
+            ],
+          },
+        },
+      },
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-06-18T19:30:00.000Z',
+              lte: '2025-06-18T19:55:00.000Z',
+            },
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'another.range.field',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'another.range.field',
+          negate: false,
+          params: {
+            gte: '0',
+            lt: '10',
+          },
+          type: 'range',
+          value: {
+            gte: '0',
+            lt: '10',
+          },
+        },
+        query: {
+          range: {
+            'another.range.field': {
+              gte: '0',
+              lt: '10',
+            },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('should return modified time range in the filter array range filters are present', () => {
+    const filter = [
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'event.action',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'event.action',
+          negate: false,
+          params: ['a', 'b', 'c'],
+          type: 'phrases',
+          value: ['a', 'b', 'c'],
+        },
+        query: {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_phrase: {
+                  'event.action': 'a',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'b',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'c',
+                },
+              },
+            ],
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'another.range.field',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'another.range.field',
+          negate: false,
+          params: {
+            gte: '0',
+            lt: '10',
+          },
+          type: 'range',
+          value: {
+            gte: '0',
+            lt: '10',
+          },
+        },
+        query: {
+          range: {
+            'another.range.field': {
+              gte: '0',
+              lt: '10',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      // @ts-expect-error
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toEqual([
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'event.start': {
+              format: 'strict_date_optional_time',
+              gte: '2025-06-18T19:30:00.000Z',
+              lte: '2025-06-18T19:55:00.000Z',
+            },
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'event.action',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'event.action',
+          negate: false,
+          params: ['a', 'b', 'c'],
+          type: 'phrases',
+          value: ['a', 'b', 'c'],
+        },
+        query: {
+          bool: {
+            minimum_should_match: 1,
+            should: [
+              {
+                match_phrase: {
+                  'event.action': 'a',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'b',
+                },
+              },
+              {
+                match_phrase: {
+                  'event.action': 'c',
+                },
+              },
+            ],
+          },
+        },
+      },
+      {
+        $state: {
+          store: 'appState',
+        },
+        meta: {
+          alias: null,
+          disabled: false,
+          field: 'another.range.field',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          key: 'another.range.field',
+          negate: false,
+          params: {
+            gte: '0',
+            lt: '10',
+          },
+          type: 'range',
+          value: {
+            gte: '0',
+            lt: '10',
+          },
+        },
+        query: {
+          range: {
+            'another.range.field': {
+              gte: '0',
+              lt: '10',
+            },
+          },
+        },
+      },
+    ]);
+  });
+
+  it('should return undefined if unexpected time filter found', () => {
+    const filter = [
+      {
+        meta: {
+          field: 'event.start',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            'another.field': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toBeUndefined();
+  });
+
+  it('should return undefined if no meta field found', () => {
+    const filter = [
+      {
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: '2025-01-01T19:38:24.286Z',
+              lte: '2025-01-01T20:03:24.286Z',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      // @ts-expect-error missing meta field
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+    });
+    expect(updated).toBeUndefined();
+  });
+
+  it('should return undefined if invalid time', () => {
+    const filter = [
+      {
+        meta: {
+          field: '@timestamp',
+          index: '0bde9920-4ade-4c19-8043-368aa37f1dae',
+          params: {},
+        },
+        query: {
+          range: {
+            '@timestamp': {
+              format: 'strict_date_optional_time',
+              gte: 'foo',
+              lte: 'bar',
+            },
+          },
+        },
+      },
+    ];
+
+    const updated = overrideTimeRange({
+      currentFilters: filter,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toBeUndefined();
+  });
+
+  it('should return undefined for undefined filters', () => {
+    const updated = overrideTimeRange({
+      currentFilters: undefined,
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toBeUndefined();
+  });
+
+  it('should return undefined for empty filters', () => {
+    const updated = overrideTimeRange({
+      currentFilters: [],
+      forceNow: '2025-06-18T19:55:00.000Z',
+      logger: mockLogger,
+    });
+    expect(updated).toBeUndefined();
+  });
+});

src/platform/packages/private/kbn-generate-csv/src/lib/override_time_range.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Filter } from '@kbn/es-query';
+import { set } from '@kbn/safer-lodash-set';
+import type { Logger } from '@kbn/core/server';
+import { cloneDeep, get, has, isArray } from 'lodash';
+
+const getTimeFieldAccessorString = (metaField: string): string => `query.range['${metaField}']`;
+const getTimeFields = (filter: Filter) => {
+  const metaField = get(filter, 'meta.field');
+  if (metaField) {
+    const timeFieldAccessorString = getTimeFieldAccessorString(metaField);
+    const timeFormat = get(filter, `${timeFieldAccessorString}.format`);
+    const timeGte = get(filter, `${timeFieldAccessorString}.gte`);
+    const timeLte = get(filter, `${timeFieldAccessorString}.lte`);
+
+    return { metaField, timeFormat, timeGte, timeLte };
+  }
+
+  return {};
+};
+
+const isValidDateTime = (dateString: string): boolean => {
+  const date = Date.parse(dateString);
+  return !isNaN(date) && date > 0;
+};
+
+interface OverrideTimeRangeOpts {
+  currentFilters: Filter[] | Filter | undefined;
+  forceNow: string;
+  logger: Logger;
+}
+export const overrideTimeRange = ({
+  currentFilters,
+  forceNow,
+  logger,
+}: OverrideTimeRangeOpts): Filter[] | undefined => {
+  if (!currentFilters) {
+    return;
+  }
+
+  const filters = isArray(currentFilters) ? currentFilters : [currentFilters];
+  if (filters.length === 0) {
+    return;
+  }
+
+  // Looking for filters with this format which indicate a time range:
+  //   {
+  //     "meta": {
+  //         "field": <timeFieldName>,
+  //         "index": <indexId>,
+  //         "params": {}
+  //     },
+  //     "query": {
+  //         "range": {
+  //             <timeFieldName>: {
+  //                 "format": "strict_date_optional_time",
+  //                 "gte": "2025-06-18T18:29:53.537Z",
+  //                 "lte": "2025-06-18T18:54:53.537Z"
+  //             }
+  //         }
+  //     }
+  // }
+  const timeFilterIndex = filters.findIndex((filter) => {
+    if (has(filter, '$state')) {
+      return false;
+    }
+
+    const {
+      timeFormat: maybeTimeFieldFormat,
+      timeGte: maybeTimeFieldGte,
+      timeLte: maybeTimeFieldLte,
+    } = getTimeFields(filter);
+
+    if (maybeTimeFieldFormat && maybeTimeFieldGte && maybeTimeFieldLte) {
+      return isValidDateTime(maybeTimeFieldGte) && isValidDateTime(maybeTimeFieldLte);
+    }
+    return false;
+  });
+
+  if (timeFilterIndex >= 0) {
+    try {
+      const timeFilter = cloneDeep(filters[timeFilterIndex]);
+      const { metaField, timeGte, timeLte } = getTimeFields(timeFilter);
+      if (metaField) {
+        const timeGteMs = Date.parse(timeGte);
+        const timeLteMs = Date.parse(timeLte);
+        const timeDiffMs = timeLteMs - timeGteMs;
+        const newLte = Date.parse(forceNow);
+        const newGte = newLte - timeDiffMs;
+
+        const timeFieldAccessorString = getTimeFieldAccessorString(metaField);
+        set(timeFilter, `${timeFieldAccessorString}.gte`, new Date(newGte).toISOString());
+        set(timeFilter, `${timeFieldAccessorString}.lte`, forceNow);
+
+        filters.splice(timeFilterIndex, 1, timeFilter);
+        return filters;
+      }
+    } catch (error) {
+      logger.warn(`Error calculating updated time range: ${error.message}`);
+    }
+  }
+};

src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts

@@ -124,6 +124,8 @@ export class SearchCursorPit extends SearchCursor {
       throw new Error('Could not retrieve the search body!');
     }
 
+    this.logger.debug(() => `Executing search with body: ${JSON.stringify(searchBody)}`);
+
     const response = await this.searchWithPit(searchBody);
 
     if (!response) {

src/platform/packages/private/kbn-generate-csv/tsconfig.json

@@ -32,5 +32,6 @@
     "@kbn/search-types",
     "@kbn/task-manager-plugin",
     "@kbn/esql-utils",
+    "@kbn/safer-lodash-set",
   ]
 }

src/platform/packages/private/kbn-generate-csv/types.ts

@@ -17,5 +17,6 @@ export interface JobParamsCSV {
   browserTimezone?: string;
   searchSource: SerializedSearchSourceFields;
   columns?: string[];
+  forceNow?: string;
   pagingStrategy?: CsvPagingStrategy;
 }