From e09ecd048ed0f44ad9965e872e46f2bdbd4bc588 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Wed, 22 Feb 2023 12:44:31 -0700 Subject: [PATCH] [Reporting] New API alternative endpoint for exporting CSV (#149172) ## Summary This new CSV endpoint will be usable for scripted reporting. As of this PR, the endpoint is not going to be used in the UI: that will be handled by [this issue](https://github.com/elastic/kibana/issues/151190). The new endpoint is not documented yet, but should be once the endpoint is used in the UI, according to [this issue](https://github.com/elastic/kibana/issues/151745). Depends on https://github.com/elastic/kibana/pull/150631 Closes https://github.com/elastic/kibana/issues/148775 - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ## Using the endpoint from a script A request to generate a report has a string of rison-encoded job parameters. The parameters can now be made into a template for customers to inject their own savedSearchID. The time range of the search, filters, and query can also be dynamically filled in at the time of requesting a report - much more easily than before. ```sh SAVED_SEARCH_ID="d779c5d0-a1c3-11ed-bd60-6957c24fc275" # Inject the search ID into the "jobParams" template JOB_PARAMS="(browserTimezone:UTC,locatorParams:!((id:DISCOVER_APP_LOCATOR,params:(savedSearchId:'${SAVED_SEARCH_ID}',timeRange:(from:now-1M,to:now)))),objectType:search)" # Send a request to generate a report curl -XPOST $ENDPOINT \ -H "${AUTH_HEADER}" \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: reporting' -d '{ "jobParams": "'$JOB_PARAMS'"}' ``` ## Limitations A locator without a `savedSearchId` is currently not supported. Getting that support will make it possible to create exports based on content in Discover tied only to a Data View (unsaved search support). ## (Demo) Developer Example app _The demo app is descoped from this PR_ ~~This PR updates the Reporting example app to feature a "CSV Explorer." In this app, you can select a saved search, build up a DiscoverAppLocatorParams object, and send a request to generate a report.~~ https://user-images.githubusercontent.com/908371/217356050-f556f897-33c6-4623-aa06-9af191378e48.mp4 ## Release Note We are skipping the release note for now, while waiting for documentation. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../common/constants/report_types.ts | 2 + .../common/types/export_types/csv_v2.ts | 23 + .../plugins/reporting/common/types/index.ts | 4 + x-pack/plugins/reporting/kibana.jsonc | 1 + x-pack/plugins/reporting/server/core.ts | 8 +- .../server/export_types/csv_v2/create_job.ts | 42 ++ .../server/export_types/csv_v2/execute_job.ts | 63 ++ .../server/export_types/csv_v2/index.ts | 40 ++ .../server/lib/export_types_registry.ts | 4 +- .../server/routes/lib/request_handler.ts | 4 +- .../create_mock_reportingplugin.ts | 2 + x-pack/plugins/reporting/server/types.ts | 17 +- .../reporting_usage_collector.test.ts.snap | 628 ++++++++++++++++++ .../server/usage/get_export_stats.ts | 1 + .../reporting/server/usage/schema.test.ts | 90 +++ .../plugins/reporting/server/usage/schema.ts | 6 + .../plugins/reporting/server/usage/types.ts | 5 + .../schema/xpack_plugins.json | 366 ++++++++++ .../functional/apps/discover/reporting.ts | 2 + .../fixtures/kbn_archiver/reporting/logs.json | 150 ++++- .../kbn_archiver/reporting/timeless.json | 126 ++++ .../__snapshots__/csv_v2.snap | 73 ++ .../reporting_and_security/csv_v2.ts | 250 +++++++ .../reporting_and_security/index.ts | 1 + x-pack/test/tsconfig.json | 1 + 25 files changed, 1892 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/reporting/common/types/export_types/csv_v2.ts create mode 100644 x-pack/plugins/reporting/server/export_types/csv_v2/create_job.ts create mode 100644 x-pack/plugins/reporting/server/export_types/csv_v2/execute_job.ts create mode 100644 x-pack/plugins/reporting/server/export_types/csv_v2/index.ts create mode 100644 x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_v2.snap create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/csv_v2.ts diff --git a/x-pack/plugins/reporting/common/constants/report_types.ts b/x-pack/plugins/reporting/common/constants/report_types.ts index baa9153e1a81..4e9b945aadb7 100644 --- a/x-pack/plugins/reporting/common/constants/report_types.ts +++ b/x-pack/plugins/reporting/common/constants/report_types.ts @@ -5,7 +5,9 @@ * 2.0. */ +// Export Type Definitions export const CSV_REPORT_TYPE = 'CSV'; +export const CSV_REPORT_TYPE_V2 = 'csv_v2'; export const PDF_REPORT_TYPE = 'printablePdf'; export const PDF_REPORT_TYPE_V2 = 'printablePdfV2'; diff --git a/x-pack/plugins/reporting/common/types/export_types/csv_v2.ts b/x-pack/plugins/reporting/common/types/export_types/csv_v2.ts new file mode 100644 index 000000000000..a712119a0736 --- /dev/null +++ b/x-pack/plugins/reporting/common/types/export_types/csv_v2.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BaseParamsV2, BasePayloadV2 } from '../base'; + +interface CsvFromSavedObjectBase { + objectType: 'search'; +} + +/** + * Makes title optional, as it can be derived from the saved search object + */ +export type JobParamsCsvFromSavedObject = CsvFromSavedObjectBase & + Omit & { title?: string }; + +/** + * + */ +export type TaskPayloadCsvFromSavedObject = CsvFromSavedObjectBase & BasePayloadV2; diff --git a/x-pack/plugins/reporting/common/types/index.ts b/x-pack/plugins/reporting/common/types/index.ts index 1f5abe084031..b42418019f1a 100644 --- a/x-pack/plugins/reporting/common/types/index.ts +++ b/x-pack/plugins/reporting/common/types/index.ts @@ -8,6 +8,10 @@ import type { PdfScreenshotResult, PngScreenshotResult } from '@kbn/screenshotting-plugin/server'; import type { BaseParams, BaseParamsV2, BasePayload, BasePayloadV2, JobId } from './base'; +export type { + JobParamsCsvFromSavedObject, + TaskPayloadCsvFromSavedObject, +} from './export_types/csv_v2'; export type { JobParamsPNGDeprecated } from './export_types/png'; export type { JobParamsPNGV2 } from './export_types/png_v2'; export type { JobAppParamsPDF, JobParamsPDFDeprecated } from './export_types/printable_pdf'; diff --git a/x-pack/plugins/reporting/kibana.jsonc b/x-pack/plugins/reporting/kibana.jsonc index 34304f8dc96a..a02db938c9ec 100644 --- a/x-pack/plugins/reporting/kibana.jsonc +++ b/x-pack/plugins/reporting/kibana.jsonc @@ -13,6 +13,7 @@ ], "requiredPlugins": [ "data", + "discover", "fieldFormats", "esUiShared", "home", diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index 045119240f1e..397c2c391d50 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -7,8 +7,11 @@ import type { DocLinksServiceSetup, + FakeRawRequest, + Headers, IBasePath, IClusterClient, + KibanaRequest, Logger, PackageInfo, PluginInitializerContext, @@ -16,12 +19,10 @@ import type { SavedObjectsServiceStart, StatusServiceSetup, UiSettingsServiceStart, - KibanaRequest, - FakeRawRequest, - Headers, } from '@kbn/core/server'; import { CoreKibanaRequest, ServiceStatusLevels } from '@kbn/core/server'; import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; +import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; @@ -69,6 +70,7 @@ export interface ReportingInternalStart { uiSettings: UiSettingsServiceStart; esClient: IClusterClient; data: DataPluginStart; + discover: DiscoverServerPluginStart; fieldFormats: FieldFormatsStart; licensing: LicensingPluginStart; logger: Logger; diff --git a/x-pack/plugins/reporting/server/export_types/csv_v2/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv_v2/create_job.ts new file mode 100644 index 000000000000..b0c886137edd --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_v2/create_job.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from '../../../common/types'; +import { CreateJobFn, CreateJobFnFactory } from '../../types'; + +type CreateJobFnType = CreateJobFn; + +export const createJobFnFactory: CreateJobFnFactory = function createJobFactoryFn( + reporting +) { + return async function createJob(jobParams, _context, req) { + // 1. Validation of locatorParams + const { locatorParams } = jobParams; + const { id, params } = locatorParams[0]; + if ( + !locatorParams || + !Array.isArray(locatorParams) || + locatorParams.length !== 1 || + id !== 'DISCOVER_APP_LOCATOR' || + !params + ) { + throw Boom.badRequest('Invalid Job params: must contain a single Discover App locator'); + } + + if (!params || !params.savedSearchId || typeof params.savedSearchId !== 'string') { + throw Boom.badRequest('Invalid Discover App locator: must contain a savedSearchId'); + } + + // use Discover contract to get the title of the report from job params + const { discover: discoverPluginStart } = await reporting.getPluginStartDeps(); + const locatorClient = await discoverPluginStart.locator.asScopedClient(req); + const title = await locatorClient.titleFromLocator(params); + + return { ...jobParams, title }; + }; +}; diff --git a/x-pack/plugins/reporting/server/export_types/csv_v2/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_v2/execute_job.ts new file mode 100644 index 000000000000..825b87c8b6d8 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_v2/execute_job.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TaskPayloadCsvFromSavedObject } from '../../../common/types'; +import { getFieldFormats } from '../../services'; +import type { RunTaskFn, RunTaskFnFactory } from '../../types'; +import { decryptJobHeaders } from '../common'; +import { CsvGenerator } from '../csv_searchsource/generate_csv'; + +type RunTaskFnType = RunTaskFn; + +export const runTaskFnFactory: RunTaskFnFactory = (reporting, _logger) => { + const config = reporting.getConfig(); + const encryptionKey = config.get('encryptionKey'); + const csvConfig = config.get('csv'); + + return async function runTask(jobId, job, cancellationToken, stream) { + const logger = _logger.get(`execute:${jobId}`); + + const headers = await decryptJobHeaders(encryptionKey, job.headers, logger); + const fakeRequest = reporting.getFakeRequest(headers, job.spaceId, logger); + const uiSettings = await reporting.getUiSettingsClient(fakeRequest, logger); + const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(uiSettings); + const { data: dataPluginStart, discover: discoverPluginStart } = + await reporting.getPluginStartDeps(); + const data = dataPluginStart.search.asScoped(fakeRequest); + + const { locatorParams } = job; + const { params } = locatorParams[0]; + + // use Discover contract to convert the job params into inputs for CsvGenerator + const locatorClient = await discoverPluginStart.locator.asScopedClient(fakeRequest); + const columns = await locatorClient.columnsFromLocator(params); + const searchSource = await locatorClient.searchSourceFromLocator(params); + + const [es, searchSourceStart] = await Promise.all([ + (await reporting.getEsClient()).asScoped(fakeRequest), + await dataPluginStart.search.searchSource.asScoped(fakeRequest), + ]); + + const clients = { uiSettings, data, es }; + const dependencies = { searchSourceStart, fieldFormatsRegistry }; + + const csv = new CsvGenerator( + { + columns, + searchSource: searchSource.getSerializedFields(true), + ...job, + }, + csvConfig, + clients, + dependencies, + cancellationToken, + logger, + stream + ); + return await csv.generateData(); + }; +}; diff --git a/x-pack/plugins/reporting/server/export_types/csv_v2/index.ts b/x-pack/plugins/reporting/server/export_types/csv_v2/index.ts new file mode 100644 index 000000000000..0e914affbf14 --- /dev/null +++ b/x-pack/plugins/reporting/server/export_types/csv_v2/index.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CSV_REPORT_TYPE_V2 as CSV_JOB_TYPE, + LICENSE_TYPE_BASIC, + LICENSE_TYPE_CLOUD_STANDARD, + LICENSE_TYPE_ENTERPRISE, + LICENSE_TYPE_GOLD, + LICENSE_TYPE_PLATINUM, + LICENSE_TYPE_TRIAL, +} from '../../../common/constants'; +import { JobParamsCsvFromSavedObject, TaskPayloadCsvFromSavedObject } from '../../../common/types'; +import { CreateJobFn, ExportTypeDefinition, RunTaskFn } from '../../types'; +import { createJobFnFactory } from './create_job'; +import { runTaskFnFactory } from './execute_job'; + +export const getExportType = (): ExportTypeDefinition< + CreateJobFn, + RunTaskFn +> => ({ + id: CSV_JOB_TYPE, + name: CSV_JOB_TYPE, + jobType: CSV_JOB_TYPE, + jobContentExtension: 'csv', + createJobFnFactory, + runTaskFnFactory, + validLicenses: [ + LICENSE_TYPE_TRIAL, + LICENSE_TYPE_BASIC, + LICENSE_TYPE_CLOUD_STANDARD, + LICENSE_TYPE_GOLD, + LICENSE_TYPE_PLATINUM, + LICENSE_TYPE_ENTERPRISE, + ], +}); diff --git a/x-pack/plugins/reporting/server/lib/export_types_registry.ts b/x-pack/plugins/reporting/server/lib/export_types_registry.ts index 0c245bde667d..96456ea93926 100644 --- a/x-pack/plugins/reporting/server/lib/export_types_registry.ts +++ b/x-pack/plugins/reporting/server/lib/export_types_registry.ts @@ -6,7 +6,8 @@ */ import { isString } from 'lodash'; -import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_searchsource_immediate'; +import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_v2'; +import { getExportType as getTypeCsvFromSavedObjectImmediate } from '../export_types/csv_searchsource_immediate'; import { getExportType as getTypeCsv } from '../export_types/csv_searchsource'; import { getExportType as getTypePng } from '../export_types/png'; import { getExportType as getTypePngV2 } from '../export_types/png_v2'; @@ -88,6 +89,7 @@ export function getExportTypesRegistry(): ExportTypesRegistry { const getTypeFns: Array<() => ExportTypeDefinition> = [ getTypeCsv, getTypeCsvFromSavedObject, + getTypeCsvFromSavedObjectImmediate, getTypePng, getTypePngV2, getTypePrintablePdf, diff --git a/x-pack/plugins/reporting/server/routes/lib/request_handler.ts b/x-pack/plugins/reporting/server/routes/lib/request_handler.ts index e54121c9dd97..0e0eba2a26ab 100644 --- a/x-pack/plugins/reporting/server/routes/lib/request_handler.ts +++ b/x-pack/plugins/reporting/server/routes/lib/request_handler.ts @@ -66,14 +66,14 @@ export class RequestHandler { throw new Error(`Export type ${exportTypeId} is not an async job type!`); } - // 1. ensure the incoming params have a version field + // 1. ensure the incoming params have a version field (should be set by the UI) jobParams.version = checkParamsVersion(jobParams, logger); // 2. encrypt request headers for the running report job to authenticate itself with Kibana // 3. call the export type's createJobFn to create the job payload const [headers, job] = await Promise.all([ this.encryptHeaders(), - createJob(jobParams, context), + createJob(jobParams, context, this.req), ]); const payload = { diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index aa1640274e4c..4c1d28e75ba1 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -18,6 +18,7 @@ import { statusServiceMock, } from '@kbn/core/server/mocks'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; +import { discoverPluginMock } from '@kbn/discover-plugin/server/mocks'; import { FieldFormatsRegistry } from '@kbn/field-formats-plugin/common'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; import { DeepPartial } from 'utility-types'; @@ -66,6 +67,7 @@ export const createMockPluginStart = async ( esClient: elasticsearchServiceMock.createClusterClient(), savedObjects: { getScopedClient: jest.fn() }, uiSettings: { asScopedToClient: () => ({ get: jest.fn() }) }, + discover: discoverPluginMock.createStartContract(), data: dataPluginMock.createStartContract(), fieldFormats: () => Promise.resolve(fieldFormatsMock), store: await createMockReportingStore(config), diff --git a/x-pack/plugins/reporting/server/types.ts b/x-pack/plugins/reporting/server/types.ts index ac81bb0393a0..9f7d48e94f41 100644 --- a/x-pack/plugins/reporting/server/types.ts +++ b/x-pack/plugins/reporting/server/types.ts @@ -5,17 +5,16 @@ * 2.0. */ -import type { IRouter, Logger, CustomRequestHandlerContext } from '@kbn/core/server'; +import type { CustomRequestHandlerContext, IRouter, KibanaRequest, Logger } from '@kbn/core/server'; import type { DataPluginStart } from '@kbn/data-plugin/server/plugin'; -import { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; -import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server'; -import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; -import type { Writable } from 'stream'; +import { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server'; import type { - PngScreenshotOptions as BasePngScreenshotOptions, PdfScreenshotOptions as BasePdfScreenshotOptions, + PngScreenshotOptions as BasePngScreenshotOptions, ScreenshottingStart, } from '@kbn/screenshotting-plugin/server'; import type { @@ -28,6 +27,8 @@ import type { TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; +import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import type { Writable } from 'stream'; import type { CancellationToken } from '../common/cancellation_token'; import type { BaseParams, BasePayload, TaskRunResult, UrlOrUrlLocatorTuple } from '../common/types'; import type { ReportingConfigType } from './config'; @@ -59,7 +60,8 @@ export type ScrollConfig = ReportingConfigType['csv']['scroll']; // default fn type for CreateJobFnFactory export type CreateJobFn = ( jobParams: JobParamsType, - context: ReportingRequestHandlerContext + context: ReportingRequestHandlerContext, + req: KibanaRequest ) => Promise>; // default fn type for RunTaskFnFactory @@ -105,6 +107,7 @@ export interface ReportingSetupDeps { export interface ReportingStartDeps { data: DataPluginStart; + discover: DiscoverServerPluginStart; fieldFormats: FieldFormatsStart; licensing: LicensingPluginStart; screenshotting: ScreenshottingStart; diff --git a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap index 6e6653253e26..732bbf7c43ef 100644 --- a/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap +++ b/x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap @@ -425,6 +425,95 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, + "available": Object { + "type": "boolean", + }, + "deprecated": Object { + "type": "long", + }, + "error_codes": Object { + "authentication_expired_error": Object { + "type": "long", + }, + "kibana_shutting_down_error": Object { + "type": "long", + }, + "queue_timeout_error": Object { + "type": "long", + }, + "unknown_error": Object { + "type": "long", + }, + }, + "execution_times": Object { + "avg": Object { + "type": "float", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, + "metrics": Object { + "csv_rows": Object { + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + }, + "output_size": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + "total": Object { + "type": "long", + }, + }, "enabled": Object { "type": "boolean", }, @@ -846,6 +935,95 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, + "available": Object { + "type": "boolean", + }, + "deprecated": Object { + "type": "long", + }, + "error_codes": Object { + "authentication_expired_error": Object { + "type": "long", + }, + "kibana_shutting_down_error": Object { + "type": "long", + }, + "queue_timeout_error": Object { + "type": "long", + }, + "unknown_error": Object { + "type": "long", + }, + }, + "execution_times": Object { + "avg": Object { + "type": "float", + }, + "max": Object { + "type": "long", + }, + "min": Object { + "type": "long", + }, + }, + "metrics": Object { + "csv_rows": Object { + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + }, + "output_size": Object { + "1.0": Object { + "type": "long", + }, + "25.0": Object { + "type": "long", + }, + "5.0": Object { + "type": "long", + }, + "50.0": Object { + "type": "long", + }, + "75.0": Object { + "type": "long", + }, + "95.0": Object { + "type": "long", + }, + "99.0": Object { + "type": "long", + }, + }, + "total": Object { + "type": "long", + }, + }, "output_size": Object { "1.0": Object { "type": "long", @@ -1247,6 +1425,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -1333,6 +1525,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -1419,6 +1625,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -1505,6 +1725,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -1591,6 +1825,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2023,6 +2271,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2109,6 +2371,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2195,6 +2471,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2281,6 +2571,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2367,6 +2671,20 @@ Object { "type": "long", }, }, + "csv_v2": Object { + "canvas workpad": Object { + "type": "long", + }, + "dashboard": Object { + "type": "long", + }, + "search": Object { + "type": "long", + }, + "visualization": Object { + "type": "long", + }, + }, "printable_pdf": Object { "canvas workpad": Object { "type": "long", @@ -2542,6 +2860,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "enabled": true, "last7Days": Object { "PNG": Object { @@ -2681,6 +3030,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "output_size": undefined, "printable_pdf": Object { "app": Object { @@ -3039,6 +3419,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "enabled": true, "last7Days": Object { "PNG": Object { @@ -3178,6 +3589,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "output_size": undefined, "printable_pdf": Object { "app": Object { @@ -3524,6 +3966,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "enabled": true, "last7Days": Object { "PNG": Object { @@ -3663,6 +4136,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "output_size": undefined, "printable_pdf": Object { "app": Object { @@ -4043,6 +4547,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "enabled": true, "last7Days": Object { "PNG": Object { @@ -4182,6 +4717,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "output_size": undefined, "printable_pdf": Object { "app": Object { @@ -4550,6 +5116,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "enabled": true, "last7Days": Object { "PNG": Object { @@ -4689,6 +5286,37 @@ Object { }, "total": 0, }, + "csv_v2": Object { + "app": Object { + "canvas workpad": 0, + "dashboard": 0, + "search": 0, + "visualization": 0, + }, + "available": true, + "deprecated": 0, + "error_codes": undefined, + "execution_times": undefined, + "layout": undefined, + "metrics": Object { + "csv_rows": Object { + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + }, + "output_size": Object { + "1.0": null, + "25.0": null, + "5.0": null, + "50.0": null, + "75.0": null, + "95.0": null, + "99.0": null, + }, + "total": 0, + }, "output_size": undefined, "printable_pdf": Object { "app": Object { diff --git a/x-pack/plugins/reporting/server/usage/get_export_stats.ts b/x-pack/plugins/reporting/server/usage/get_export_stats.ts index 40e749a9493a..4e2b1b6ceafc 100644 --- a/x-pack/plugins/reporting/server/usage/get_export_stats.ts +++ b/x-pack/plugins/reporting/server/usage/get_export_stats.ts @@ -54,6 +54,7 @@ const metricsSets = { const metricsForFeature: { [K in keyof JobTypes]: JobTypes[K]['metrics'] } = { csv_searchsource: metricsSets.csv, csv_searchsource_immediate: metricsSets.csv, + csv_v2: metricsSets.csv, PNG: metricsSets.png, PNGV2: metricsSets.png, printable_pdf: metricsSets.pdf, diff --git a/x-pack/plugins/reporting/server/usage/schema.test.ts b/x-pack/plugins/reporting/server/usage/schema.test.ts index 55d78c7d5b1e..41ef55ddb6be 100644 --- a/x-pack/plugins/reporting/server/usage/schema.test.ts +++ b/x-pack/plugins/reporting/server/usage/schema.test.ts @@ -146,6 +146,31 @@ describe('Reporting telemetry schema', () => { "csv_searchsource_immediate.output_size.95.0.type": "long", "csv_searchsource_immediate.output_size.99.0.type": "long", "csv_searchsource_immediate.total.type": "long", + "csv_v2.app.canvas workpad.type": "long", + "csv_v2.app.dashboard.type": "long", + "csv_v2.app.search.type": "long", + "csv_v2.app.visualization.type": "long", + "csv_v2.available.type": "boolean", + "csv_v2.deprecated.type": "long", + "csv_v2.error_codes.authentication_expired_error.type": "long", + "csv_v2.error_codes.kibana_shutting_down_error.type": "long", + "csv_v2.error_codes.queue_timeout_error.type": "long", + "csv_v2.error_codes.unknown_error.type": "long", + "csv_v2.execution_times.avg.type": "float", + "csv_v2.execution_times.max.type": "long", + "csv_v2.execution_times.min.type": "long", + "csv_v2.metrics.csv_rows.50.0.type": "long", + "csv_v2.metrics.csv_rows.75.0.type": "long", + "csv_v2.metrics.csv_rows.95.0.type": "long", + "csv_v2.metrics.csv_rows.99.0.type": "long", + "csv_v2.output_size.1.0.type": "long", + "csv_v2.output_size.25.0.type": "long", + "csv_v2.output_size.5.0.type": "long", + "csv_v2.output_size.50.0.type": "long", + "csv_v2.output_size.75.0.type": "long", + "csv_v2.output_size.95.0.type": "long", + "csv_v2.output_size.99.0.type": "long", + "csv_v2.total.type": "long", "enabled.type": "boolean", "last7Days.PNG.app.canvas workpad.type": "long", "last7Days.PNG.app.dashboard.type": "long", @@ -266,6 +291,31 @@ describe('Reporting telemetry schema', () => { "last7Days.csv_searchsource_immediate.output_size.95.0.type": "long", "last7Days.csv_searchsource_immediate.output_size.99.0.type": "long", "last7Days.csv_searchsource_immediate.total.type": "long", + "last7Days.csv_v2.app.canvas workpad.type": "long", + "last7Days.csv_v2.app.dashboard.type": "long", + "last7Days.csv_v2.app.search.type": "long", + "last7Days.csv_v2.app.visualization.type": "long", + "last7Days.csv_v2.available.type": "boolean", + "last7Days.csv_v2.deprecated.type": "long", + "last7Days.csv_v2.error_codes.authentication_expired_error.type": "long", + "last7Days.csv_v2.error_codes.kibana_shutting_down_error.type": "long", + "last7Days.csv_v2.error_codes.queue_timeout_error.type": "long", + "last7Days.csv_v2.error_codes.unknown_error.type": "long", + "last7Days.csv_v2.execution_times.avg.type": "float", + "last7Days.csv_v2.execution_times.max.type": "long", + "last7Days.csv_v2.execution_times.min.type": "long", + "last7Days.csv_v2.metrics.csv_rows.50.0.type": "long", + "last7Days.csv_v2.metrics.csv_rows.75.0.type": "long", + "last7Days.csv_v2.metrics.csv_rows.95.0.type": "long", + "last7Days.csv_v2.metrics.csv_rows.99.0.type": "long", + "last7Days.csv_v2.output_size.1.0.type": "long", + "last7Days.csv_v2.output_size.25.0.type": "long", + "last7Days.csv_v2.output_size.5.0.type": "long", + "last7Days.csv_v2.output_size.50.0.type": "long", + "last7Days.csv_v2.output_size.75.0.type": "long", + "last7Days.csv_v2.output_size.95.0.type": "long", + "last7Days.csv_v2.output_size.99.0.type": "long", + "last7Days.csv_v2.total.type": "long", "last7Days.output_size.1.0.type": "long", "last7Days.output_size.25.0.type": "long", "last7Days.output_size.5.0.type": "long", @@ -381,6 +431,10 @@ describe('Reporting telemetry schema', () => { "last7Days.statuses.completed.csv_searchsource_immediate.dashboard.type": "long", "last7Days.statuses.completed.csv_searchsource_immediate.search.type": "long", "last7Days.statuses.completed.csv_searchsource_immediate.visualization.type": "long", + "last7Days.statuses.completed.csv_v2.canvas workpad.type": "long", + "last7Days.statuses.completed.csv_v2.dashboard.type": "long", + "last7Days.statuses.completed.csv_v2.search.type": "long", + "last7Days.statuses.completed.csv_v2.visualization.type": "long", "last7Days.statuses.completed.printable_pdf.canvas workpad.type": "long", "last7Days.statuses.completed.printable_pdf.dashboard.type": "long", "last7Days.statuses.completed.printable_pdf.search.type": "long", @@ -405,6 +459,10 @@ describe('Reporting telemetry schema', () => { "last7Days.statuses.completed_with_warnings.csv_searchsource_immediate.dashboard.type": "long", "last7Days.statuses.completed_with_warnings.csv_searchsource_immediate.search.type": "long", "last7Days.statuses.completed_with_warnings.csv_searchsource_immediate.visualization.type": "long", + "last7Days.statuses.completed_with_warnings.csv_v2.canvas workpad.type": "long", + "last7Days.statuses.completed_with_warnings.csv_v2.dashboard.type": "long", + "last7Days.statuses.completed_with_warnings.csv_v2.search.type": "long", + "last7Days.statuses.completed_with_warnings.csv_v2.visualization.type": "long", "last7Days.statuses.completed_with_warnings.printable_pdf.canvas workpad.type": "long", "last7Days.statuses.completed_with_warnings.printable_pdf.dashboard.type": "long", "last7Days.statuses.completed_with_warnings.printable_pdf.search.type": "long", @@ -429,6 +487,10 @@ describe('Reporting telemetry schema', () => { "last7Days.statuses.failed.csv_searchsource_immediate.dashboard.type": "long", "last7Days.statuses.failed.csv_searchsource_immediate.search.type": "long", "last7Days.statuses.failed.csv_searchsource_immediate.visualization.type": "long", + "last7Days.statuses.failed.csv_v2.canvas workpad.type": "long", + "last7Days.statuses.failed.csv_v2.dashboard.type": "long", + "last7Days.statuses.failed.csv_v2.search.type": "long", + "last7Days.statuses.failed.csv_v2.visualization.type": "long", "last7Days.statuses.failed.printable_pdf.canvas workpad.type": "long", "last7Days.statuses.failed.printable_pdf.dashboard.type": "long", "last7Days.statuses.failed.printable_pdf.search.type": "long", @@ -453,6 +515,10 @@ describe('Reporting telemetry schema', () => { "last7Days.statuses.pending.csv_searchsource_immediate.dashboard.type": "long", "last7Days.statuses.pending.csv_searchsource_immediate.search.type": "long", "last7Days.statuses.pending.csv_searchsource_immediate.visualization.type": "long", + "last7Days.statuses.pending.csv_v2.canvas workpad.type": "long", + "last7Days.statuses.pending.csv_v2.dashboard.type": "long", + "last7Days.statuses.pending.csv_v2.search.type": "long", + "last7Days.statuses.pending.csv_v2.visualization.type": "long", "last7Days.statuses.pending.printable_pdf.canvas workpad.type": "long", "last7Days.statuses.pending.printable_pdf.dashboard.type": "long", "last7Days.statuses.pending.printable_pdf.search.type": "long", @@ -477,6 +543,10 @@ describe('Reporting telemetry schema', () => { "last7Days.statuses.processing.csv_searchsource_immediate.dashboard.type": "long", "last7Days.statuses.processing.csv_searchsource_immediate.search.type": "long", "last7Days.statuses.processing.csv_searchsource_immediate.visualization.type": "long", + "last7Days.statuses.processing.csv_v2.canvas workpad.type": "long", + "last7Days.statuses.processing.csv_v2.dashboard.type": "long", + "last7Days.statuses.processing.csv_v2.search.type": "long", + "last7Days.statuses.processing.csv_v2.visualization.type": "long", "last7Days.statuses.processing.printable_pdf.canvas workpad.type": "long", "last7Days.statuses.processing.printable_pdf.dashboard.type": "long", "last7Days.statuses.processing.printable_pdf.search.type": "long", @@ -600,6 +670,10 @@ describe('Reporting telemetry schema', () => { "statuses.completed.csv_searchsource_immediate.dashboard.type": "long", "statuses.completed.csv_searchsource_immediate.search.type": "long", "statuses.completed.csv_searchsource_immediate.visualization.type": "long", + "statuses.completed.csv_v2.canvas workpad.type": "long", + "statuses.completed.csv_v2.dashboard.type": "long", + "statuses.completed.csv_v2.search.type": "long", + "statuses.completed.csv_v2.visualization.type": "long", "statuses.completed.printable_pdf.canvas workpad.type": "long", "statuses.completed.printable_pdf.dashboard.type": "long", "statuses.completed.printable_pdf.search.type": "long", @@ -624,6 +698,10 @@ describe('Reporting telemetry schema', () => { "statuses.completed_with_warnings.csv_searchsource_immediate.dashboard.type": "long", "statuses.completed_with_warnings.csv_searchsource_immediate.search.type": "long", "statuses.completed_with_warnings.csv_searchsource_immediate.visualization.type": "long", + "statuses.completed_with_warnings.csv_v2.canvas workpad.type": "long", + "statuses.completed_with_warnings.csv_v2.dashboard.type": "long", + "statuses.completed_with_warnings.csv_v2.search.type": "long", + "statuses.completed_with_warnings.csv_v2.visualization.type": "long", "statuses.completed_with_warnings.printable_pdf.canvas workpad.type": "long", "statuses.completed_with_warnings.printable_pdf.dashboard.type": "long", "statuses.completed_with_warnings.printable_pdf.search.type": "long", @@ -648,6 +726,10 @@ describe('Reporting telemetry schema', () => { "statuses.failed.csv_searchsource_immediate.dashboard.type": "long", "statuses.failed.csv_searchsource_immediate.search.type": "long", "statuses.failed.csv_searchsource_immediate.visualization.type": "long", + "statuses.failed.csv_v2.canvas workpad.type": "long", + "statuses.failed.csv_v2.dashboard.type": "long", + "statuses.failed.csv_v2.search.type": "long", + "statuses.failed.csv_v2.visualization.type": "long", "statuses.failed.printable_pdf.canvas workpad.type": "long", "statuses.failed.printable_pdf.dashboard.type": "long", "statuses.failed.printable_pdf.search.type": "long", @@ -672,6 +754,10 @@ describe('Reporting telemetry schema', () => { "statuses.pending.csv_searchsource_immediate.dashboard.type": "long", "statuses.pending.csv_searchsource_immediate.search.type": "long", "statuses.pending.csv_searchsource_immediate.visualization.type": "long", + "statuses.pending.csv_v2.canvas workpad.type": "long", + "statuses.pending.csv_v2.dashboard.type": "long", + "statuses.pending.csv_v2.search.type": "long", + "statuses.pending.csv_v2.visualization.type": "long", "statuses.pending.printable_pdf.canvas workpad.type": "long", "statuses.pending.printable_pdf.dashboard.type": "long", "statuses.pending.printable_pdf.search.type": "long", @@ -696,6 +782,10 @@ describe('Reporting telemetry schema', () => { "statuses.processing.csv_searchsource_immediate.dashboard.type": "long", "statuses.processing.csv_searchsource_immediate.search.type": "long", "statuses.processing.csv_searchsource_immediate.visualization.type": "long", + "statuses.processing.csv_v2.canvas workpad.type": "long", + "statuses.processing.csv_v2.dashboard.type": "long", + "statuses.processing.csv_v2.search.type": "long", + "statuses.processing.csv_v2.visualization.type": "long", "statuses.processing.printable_pdf.canvas workpad.type": "long", "statuses.processing.printable_pdf.dashboard.type": "long", "statuses.processing.printable_pdf.search.type": "long", diff --git a/x-pack/plugins/reporting/server/usage/schema.ts b/x-pack/plugins/reporting/server/usage/schema.ts index e9a0cbcca30b..d753afee9a4f 100644 --- a/x-pack/plugins/reporting/server/usage/schema.ts +++ b/x-pack/plugins/reporting/server/usage/schema.ts @@ -49,6 +49,7 @@ const layoutCountsSchema: MakeSchemaFrom = { const byAppCountsSchema: MakeSchemaFrom = { csv_searchsource: appCountsSchema, csv_searchsource_immediate: appCountsSchema, + csv_v2: appCountsSchema, PNG: appCountsSchema, PNGV2: appCountsSchema, printable_pdf: appCountsSchema, @@ -138,6 +139,11 @@ const jobTypesSchema: MakeSchemaFrom = { metrics: metricsSchemaCsv, error_codes: errorCodesSchemaCsv, }, + csv_v2: { + ...availableTotalSchema, + metrics: metricsSchemaCsv, + error_codes: errorCodesSchemaCsv, + }, PNG: { ...availableTotalSchema, metrics: metricsSchemaPng, error_codes: errorCodesSchemaPng }, PNGV2: { ...availableTotalSchema, metrics: metricsSchemaPng, error_codes: errorCodesSchemaPng }, printable_pdf: { diff --git a/x-pack/plugins/reporting/server/usage/types.ts b/x-pack/plugins/reporting/server/usage/types.ts index 2c189ff4ac2c..5751537590f2 100644 --- a/x-pack/plugins/reporting/server/usage/types.ts +++ b/x-pack/plugins/reporting/server/usage/types.ts @@ -50,6 +50,7 @@ interface QueueTimesMetrics { type BaseJobTypes = | 'csv_searchsource' | 'csv_searchsource_immediate' + | 'csv_v2' | 'PNG' | 'PNGV2' | 'printable_pdf' @@ -146,6 +147,10 @@ export interface JobTypes { metrics: MetricsStatsCsv; error_codes: ErrorCodesStatsCsv; }; + csv_v2: AvailableTotal & { + metrics: MetricsStatsCsv; + error_codes: ErrorCodesStatsCsv; + }; PNG: AvailableTotal & { metrics: MetricsStatsPng; error_codes: ErrorCodesStatsPng; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 2d2c67535918..5a94f1361380 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -6885,6 +6885,109 @@ } } }, + "csv_v2": { + "properties": { + "available": { + "type": "boolean" + }, + "total": { + "type": "long" + }, + "deprecated": { + "type": "long" + }, + "output_size": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, + "app": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, + "execution_times": { + "properties": { + "min": { + "type": "long" + }, + "max": { + "type": "long" + }, + "avg": { + "type": "float" + } + } + }, + "metrics": { + "properties": { + "csv_rows": { + "properties": { + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + } + } + }, + "error_codes": { + "properties": { + "authentication_expired_error": { + "type": "long" + }, + "queue_timeout_error": { + "type": "long" + }, + "unknown_error": { + "type": "long" + }, + "kibana_shutting_down_error": { + "type": "long" + } + } + } + } + }, "PNG": { "properties": { "available": { @@ -7543,6 +7646,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -7643,6 +7762,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -7743,6 +7878,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -7843,6 +7994,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -7943,6 +8110,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -8263,6 +8446,109 @@ } } }, + "csv_v2": { + "properties": { + "available": { + "type": "boolean" + }, + "total": { + "type": "long" + }, + "deprecated": { + "type": "long" + }, + "output_size": { + "properties": { + "1.0": { + "type": "long" + }, + "5.0": { + "type": "long" + }, + "25.0": { + "type": "long" + }, + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + }, + "app": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, + "execution_times": { + "properties": { + "min": { + "type": "long" + }, + "max": { + "type": "long" + }, + "avg": { + "type": "float" + } + } + }, + "metrics": { + "properties": { + "csv_rows": { + "properties": { + "50.0": { + "type": "long" + }, + "75.0": { + "type": "long" + }, + "95.0": { + "type": "long" + }, + "99.0": { + "type": "long" + } + } + } + } + }, + "error_codes": { + "properties": { + "authentication_expired_error": { + "type": "long" + }, + "queue_timeout_error": { + "type": "long" + }, + "unknown_error": { + "type": "long" + }, + "kibana_shutting_down_error": { + "type": "long" + } + } + } + } + }, "PNG": { "properties": { "available": { @@ -8921,6 +9207,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -9021,6 +9323,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -9121,6 +9439,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -9221,6 +9555,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { @@ -9321,6 +9671,22 @@ } } }, + "csv_v2": { + "properties": { + "search": { + "type": "long" + }, + "canvas workpad": { + "type": "long" + }, + "dashboard": { + "type": "long" + }, + "visualization": { + "type": "long" + } + } + }, "PNG": { "properties": { "search": { diff --git a/x-pack/test/functional/apps/discover/reporting.ts b/x-pack/test/functional/apps/discover/reporting.ts index 026129e7f077..b076fa586e24 100644 --- a/x-pack/test/functional/apps/discover/reporting.ts +++ b/x-pack/test/functional/apps/discover/reporting.ts @@ -51,6 +51,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { after(async () => { await reportingAPI.teardownEcommerce(); + await esArchiver.emptyKibanaIndex(); }); it('is available if new', async () => { @@ -72,6 +73,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { after(async () => { await reportingAPI.teardownEcommerce(); + await esArchiver.emptyKibanaIndex(); }); beforeEach(async () => { diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json index e90150fde0fd..e7121b78f6dd 100644 --- a/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/logs.json @@ -1,17 +1,24 @@ { "attributes": { - "fields": "[{\"name\":\"referer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"xss.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.lastname\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.dest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"utc_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.char\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"clientip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"machine.ram\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"links\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"id\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"phpmemory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.twitter:card.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"ip\",\"type\":\"ip\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:modified_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:site_name.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:tag\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"agent.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"headings\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.og:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"request\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"relatedContent.twitter:site\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.coordinates\",\"type\":\"geo_point\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"meta.related\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@message.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.article:section\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"xss\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"links.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"geo.srcdest\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"extension.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"machine.os.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@tags\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"geo.src\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"spaces.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:image:height.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"url\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:site_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:title\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@message\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.twitter:image.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bytes\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"response\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"meta.user.firstname\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"relatedContent.og:image:width.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.og:description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"relatedContent.article:published_time\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]", + "fieldAttrs": "{\"@message\":{\"count\":2},\"@tags\":{\"count\":2},\"url\":{\"count\":2}}", + "fieldFormatMap": "{}", + "fields": "[]", + "name": "logstash-*", + "runtimeFieldMap": "{}", + "sourceFilters": "[]", "timeFieldName": "@timestamp", - "title": "logstash-*" + "title": "logstash-*", + "typeMeta": "{}" }, - "coreMigrationVersion": "8.0.0", + "coreMigrationVersion": "8.7.0", "id": "logstash-*", "migrationVersion": { "index-pattern": "8.0.0" }, "references": [], "type": "index-pattern", - "version": "WzIsMl0=" + "updated_at": "2023-01-19T00:49:47.963Z", + "version": "WzMxNCwyXQ==" } { @@ -496,3 +503,138 @@ "updated_at": "2021-12-22T04:22:25.584Z", "version": "WzI0MjAsMV0=" } + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"type\":\"custom\",\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"query\",\"value\":\"{\\\"bool\\\":{\\\"minimum_should_match\\\":1,\\\"should\\\":[{\\\"match_phrase\\\":{\\\"@tags\\\":\\\"info\\\"}}]}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"minimum_should_match\":1,\"should\":[{\"match_phrase\":{\"@tags\":\"info\"}}]}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "A saved search with match_phrase filter and no columns selected", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T00:36:50.456Z", + "id": "5cfe1180-9791-11ed-97f7-93bc26fd388d", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T00:37:22.716Z", + "version": "WzE5NiwyXQ==" +} + +{ + "attributes": { + "columns": [ + "@message", + "url", + "@tags" + ], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"host:media-for-the-masses\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "refreshInterval": { + "pause": true, + "value": 60000 + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "timeRange": { + "from": "2015-07-30T15:49:33.702Z", + "to": "2015-11-01T01:39:01.782Z" + }, + "timeRestore": true, + "title": "A saved search with the time stored and a query", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T00:38:13.779Z", + "id": "8ea82630-9791-11ed-97f7-93bc26fd388d", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T00:50:04.124Z", + "version": "WzMxNiwyXQ==" +} + +{ + "attributes": { + "columns": [ + "@message", + "@tags", + "url" + ], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"host:media-for-the-masses\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "A saved search with a query", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T00:31:57.708Z", + "id": "ae8044c0-9790-11ed-97f7-93bc26fd388d", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "logstash-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T00:39:15.770Z", + "version": "WzI2MCwyXQ==" +} diff --git a/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json b/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json new file mode 100644 index 000000000000..2763aee3b841 --- /dev/null +++ b/x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless.json @@ -0,0 +1,126 @@ +{ + "attributes": { + "fieldAttrs": "{\"eon\":{\"count\":1},\"epoch\":{\"count\":1},\"era\":{\"count\":1},\"period\":{\"count\":1}}", + "fieldFormatMap": "{}", + "fields": "[]", + "name": "timeless test", + "runtimeFieldMap": "{}", + "sourceFilters": "[]", + "title": "timeless-*", + "typeMeta": "{}" + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T21:56:24.982Z", + "id": "b1c00e58-e6ba-4da2-9919-f5acf707b896", + "migrationVersion": { + "index-pattern": "8.0.0" + }, + "references": [], + "type": "index-pattern", + "updated_at": "2023-01-19T21:57:06.182Z", + "version": "WzE3MCwxXQ==" +} + +{ + "attributes": { + "columns": [ + "eon", + "epoch", + "era", + "period" + ], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [], + "timeRestore": false, + "title": "plain search with 4 columns", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T22:01:14.908Z", + "id": "cafc3dc0-9844-11ed-8e25-6b737289a7c8", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "b1c00e58-e6ba-4da2-9919-f5acf707b896", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T22:01:14.908Z", + "version": "WzIxMCwxXQ==" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [], + "timeRestore": false, + "title": "plain search", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T22:01:28.910Z", + "id": "d354c6e0-9844-11ed-8e25-6b737289a7c8", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "b1c00e58-e6ba-4da2-9919-f5acf707b896", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T22:01:28.910Z", + "version": "WzIxNywxXQ==" +} + +{ + "attributes": { + "columns": [], + "description": "", + "grid": {}, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"*zoic\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "sort": [], + "timeRestore": false, + "title": "*zoic", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.7.0", + "created_at": "2023-01-19T22:02:37.551Z", + "id": "fc3e8ff0-9844-11ed-8e25-6b737289a7c8", + "migrationVersion": { + "search": "8.0.0" + }, + "references": [ + { + "id": "b1c00e58-e6ba-4da2-9919-f5acf707b896", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "search", + "updated_at": "2023-01-19T22:02:37.551Z", + "version": "WzI0MCwxXQ==" +} \ No newline at end of file diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_v2.snap b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_v2.snap new file mode 100644 index 000000000000..a094bad84d44 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_v2.snap @@ -0,0 +1,73 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Reporting APIs CSV Generation from Saved Search ID export from non-timebased data view query stored in the saved search csv file matches 1`] = ` +"_id,_index,_score,eon,epoch,era,period +tvJJX4UBvD7uFsw9L2x4,timeless-test,1,Phanerozoic, Pliocene,Cenozoic,Neogene +t_JJX4UBvD7uFsw9L2x4,timeless-test,1,Phanerozoic, Holocene,Cenozoic,Quaternary +uPJJX4UBvD7uFsw9L2x4,timeless-test,1,Phanerozoic,-,Mesozoic,Cretaceous +ufJJX4UBvD7uFsw9L2x4,timeless-test,1,Phanerozoic,-,Mesozoic,Jurassic +uvJJX4UBvD7uFsw9L2x4,timeless-test,1,Phanerozoic,-,Paleozoic,Cambrian +u_JJX4UBvD7uFsw9L2x4,timeless-test,1,Proterozoic,-,Paleozoic,Permian +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from non-timebased data view query stored in the saved search job response data is correct 1`] = ` +Object { + "contentDisposition": "attachment; filename=\\"*zoic.csv\\"", + "contentType": "text/csv; charset=utf-8", + "title": "*zoic", +} +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view timezone formatting export with custom timezone and timeRange from locator params csv file matches 1`] = ` +"order_date,category,currency,customer_id,order_id,day_of_week_i,products.created_on,sku +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories,EUR,19,716724,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085 +2019-07-11 16:00:00.000,Women's Shoes, Women's Clothing,EUR,45,591503,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0006400064, ZO0150601506 +2019-07-11 16:00:00.000,Women's Clothing,EUR,12,591709,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0638206382, ZO0038800388 +2019-07-11 16:00:00.000,Men's Clothing,EUR,52,590937,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0297602976, ZO0565605656 +2019-07-11 16:00:00.000,Men's Clothing,EUR,29,590976,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0561405614, ZO0281602816 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,41,591636,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0385003850, ZO0408604086 +2019-07-11 16:00:00.000,Men's Shoes,EUR,30,591539,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0505605056, ZO0513605136 +2019-07-11 16:00:00.000,Men's Clothing,EUR,41,591598,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0276702767, ZO0291702917 +2019-07-11 16:00:00.000,Women's Clothing,EUR,44,590927,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0046600466, ZO0050800508 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,48,590970,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0455604556, ZO0680806808 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,46,591299,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0229002290, ZO0674406744 +2019-07-11 16:00:00.000,Men's Clothing,EUR,36,591133,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0529905299, ZO0617006170 +2019-07-11 16:00:00.000,Men's Clothing,EUR,13,591175,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0299402994, ZO0433504335 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,21,591297,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0257502575, ZO0451704517 +2019-07-11 16:00:00.000,Men's Clothing,EUR,14,591149,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0584905849, ZO0578405784 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,27,591754,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0335803358, ZO0325903259 +2019-07-11 16:00:00.000,Women's Clothing, Women's Shoes,EUR,42,591803,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0645906459, ZO0324303243 +2019-07-11 16:00:00.000,Women's Clothing,EUR,46,592082,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0034400344, ZO0492904929 +2019-07-11 16:00:00.000,Women's Shoes, Women's Accessories,EUR,27,591283,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0239302393, ZO0198501985 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,4,591148,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0290302903, ZO0513705137 +2019-07-11 16:00:00.000,Men's Accessories, Men's Clothing,EUR,51,591417,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0464504645, ZO0621006210 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,14,591562,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0544305443, ZO0108001080 +2019-07-11 16:00:00.000,Women's Clothing, Women's Accessories,EUR,5,590996,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0638106381, ZO0096900969 +2019-07-11 16:00:00.000,Women's Shoes,EUR,27,591317,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0366203662, ZO0139501395 +2019-07-11 16:00:00.000,Men's Clothing,EUR,38,591362,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0541805418, ZO0594105941 +2019-07-11 16:00:00.000,Men's Shoes, Men's Clothing,EUR,30,591411,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0693506935, ZO0532405324 +2019-07-11 16:00:00.000,Men's Clothing, Men's Shoes,EUR,38,722629,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0424204242, ZO0403504035, ZO0506705067, ZO0395603956 +2019-07-11 16:00:00.000,Men's Clothing,EUR,16,591041,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0418704187, ZO0557105571 +2019-07-11 16:00:00.000,Women's Clothing,EUR,6,591074,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0268602686, ZO0484704847 +2019-07-11 16:00:00.000,Men's Clothing,EUR,7,591349,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0474804748, ZO0560705607 +2019-07-11 16:00:00.000,Women's Accessories, Women's Clothing,EUR,44,591374,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0206002060, ZO0268302683 +2019-07-11 16:00:00.000,Women's Clothing,EUR,12,591230,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0226902269, ZO0660106601 +2019-07-11 16:00:00.000,Women's Shoes, Women's Clothing,EUR,17,591717,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0248002480, ZO0646706467 +2019-07-11 16:00:00.000,Women's Shoes,EUR,42,591768,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0005800058, ZO0133901339 +2019-07-11 16:00:00.000,Men's Clothing,EUR,21,591810,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0587405874, ZO0590305903 +2019-07-11 16:00:00.000,Women's Accessories, Women's Shoes,EUR,22,592049,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0186201862, ZO0018800188 +2019-07-11 16:00:00.000,Women's Clothing, Women's Accessories,EUR,28,592097,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0265502655, ZO0201102011 +2019-07-11 16:00:00.000,Men's Clothing,EUR,19,590807,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0582105821, ZO0421304213 +2019-07-11 16:00:00.000,Men's Accessories, Men's Clothing,EUR,13,714827,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0314503145, ZO0444804448, ZO0538405384, ZO0630306303 +2019-07-11 16:00:00.000,Women's Shoes,EUR,22,591735,5,2016-12-30 15:00:00.000, 2016-12-30 15:00:00.000,ZO0248402484, ZO0382603826 +" +`; + +exports[`Reporting APIs CSV Generation from Saved Search ID export from timebased data view timezone formatting export with custom timezone and timeRange from locator params job response data is correct 1`] = ` +Object { + "contentDisposition": "attachment; filename=\\"Ecommerce Data.csv\\"", + "contentType": "text/csv; charset=utf-8", + "title": "Ecommerce Data", +} +`; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/csv_v2.ts b/x-pack/test/reporting_api_integration/reporting_and_security/csv_v2.ts new file mode 100644 index 000000000000..04ff7adc357e --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/csv_v2.ts @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DISCOVER_APP_LOCATOR } from '@kbn/discover-plugin/common'; +import expect from '@kbn/expect'; +import type { + JobParamsCsvFromSavedObject, + ReportApiJSON, +} from '@kbn/reporting-plugin/common/types'; +import rison from '@kbn/rison'; +import request from 'supertest'; +import { FtrProviderContext } from '../ftr_provider_context'; + +const LOGSTASH_DATA_ARCHIVE = 'test/functional/fixtures/es_archiver/logstash_functional'; +const LOGSTASH_SAVED_OBJECTS = 'x-pack/test/functional/fixtures/kbn_archiver/reporting/logs'; +const ECOM_SAVED_SEARCH_ID = '6091ead0-1c6d-11ea-a100-8589bb9d7c6b'; +const TIMELESS_SAVED_OBJECTS = 'x-pack/test/functional/fixtures/kbn_archiver/reporting/timeless'; +const TIMELESS_SAVED_SEARCH_WITH_QUERY = 'fc3e8ff0-9844-11ed-8e25-6b737289a7c8'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const es = getService('es'); + const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); + const esArchiver = getService('esArchiver'); + const reportingAPI = getService('reportingAPI'); + const log = getService('log'); + + const requestCsvFromSavedSearch = async ( + params: Omit + ) => { + const job: JobParamsCsvFromSavedObject = { + browserTimezone: (params as JobParamsCsvFromSavedObject).browserTimezone ?? 'UTC', + objectType: 'search', + version: '8.7.0', + ...params, + }; + log.info(`sending request for saved search: ${job.locatorParams[0].params.savedSearchId}`); + const jobParams = rison.encode(job); + return await supertest + .post(`/api/reporting/generate/csv_v2`) + .set('kbn-xsrf', 'xxx') + .send({ jobParams }); + }; + + const cleanupLogstash = async () => { + const logstashIndices = await es.indices.get({ + index: 'logstash-*', + allow_no_indices: true, + expand_wildcards: 'all', + ignore_unavailable: true, + }); + await Promise.all( + Object.keys(logstashIndices.body ?? {}).map(async (logstashIndex) => { + log.info(`deleting ${logstashIndex}`); + await es.indices.delete({ + index: logstashIndex, + }); + }) + ); + }; + + const timelessIndexName = 'timeless-test'; + const loadTimelessData = async () => { + log.info(`loading test data`); + await es.indices.create({ + index: timelessIndexName, + body: { + settings: { number_of_shards: 1 }, + mappings: { + properties: { + eon: { type: 'keyword' }, + era: { type: 'keyword' }, + period: { type: 'keyword' }, + epoch: { type: 'keyword' }, + }, + }, + }, + }); + await es.bulk({ + refresh: 'wait_for', + body: [ + { index: { _index: timelessIndexName, _id: 'tvJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Cenozoic', period: 'Neogene', epoch: ' Pliocene' }, + { index: { _index: timelessIndexName, _id: 't_JJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Cenozoic', period: 'Quaternary', epoch: ' Holocene' }, + { index: { _index: timelessIndexName, _id: 'uPJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Mesozoic', period: 'Cretaceous' }, + { index: { _index: timelessIndexName, _id: 'ufJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Mesozoic', period: 'Jurassic' }, + { index: { _index: timelessIndexName, _id: 'uvJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Phanerozoic', era: 'Paleozoic', period: 'Cambrian' }, + { index: { _index: timelessIndexName, _id: 'u_JJX4UBvD7uFsw9L2x4' } }, + { eon: 'Proterozoic', era: 'Paleozoic', period: 'Permian' }, + { index: { _index: timelessIndexName, _id: 'vPJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Archean' }, + { index: { _index: timelessIndexName, _id: 'vfJJX4UBvD7uFsw9L2x4' } }, + { eon: 'Hadean' }, + ], + }); + }; + + describe('CSV Generation from Saved Search ID', () => { + before(async () => { + // clear any previous UI Settings + await kibanaServer.uiSettings.replace({}); + + // explicitly delete all pre-existing logstash indices, since we have exports with no time filter + log.info(`deleting logstash indices`); + await cleanupLogstash(); + + log.info(`updating Advanced Settings`); + await kibanaServer.uiSettings.update({ + 'csv:quoteValues': false, + 'dateFormat:tz': 'UTC', + dateFormat: 'YYYY-MM-DD HH:mm:ss.SSS', + }); + }); + + after(async () => { + await kibanaServer.uiSettings.replace({}); + }); + + describe('export from non-timebased data view', () => { + before(async () => { + await kibanaServer.importExport.load(TIMELESS_SAVED_OBJECTS); + await loadTimelessData(); + }); + + after(async () => { + await kibanaServer.importExport.unload(TIMELESS_SAVED_OBJECTS); + + log.info(`loading test data`); + await es.indices.delete({ + index: timelessIndexName, + }); + }); + + describe('query stored in the saved search', () => { + let response: request.Response; + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + const { text, status } = await requestCsvFromSavedSearch({ + locatorParams: [ + { + id: DISCOVER_APP_LOCATOR, + version: 'reporting', + params: { + savedSearchId: TIMELESS_SAVED_SEARCH_WITH_QUERY, + }, + }, + ], + }); + expect(status).to.eql(200); + ({ job, path } = JSON.parse(text)); + await reportingAPI.waitForJobToFinish(path); + response = await supertest.get(path); + csvFile = response.text; + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expectSnapshot({ + contentType: response.header['content-type'], + contentDisposition: response.header['content-disposition'], + title: job.payload.title, + }).toMatch(); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + }); + + describe('export from timebased data view', () => { + before(async () => { + log.info(`loading archives and fixtures`); + await esArchiver.load(LOGSTASH_DATA_ARCHIVE); + await kibanaServer.importExport.load(LOGSTASH_SAVED_OBJECTS); + }); + + after(async () => { + await esArchiver.unload(LOGSTASH_DATA_ARCHIVE); + await kibanaServer.importExport.unload(LOGSTASH_SAVED_OBJECTS); + }); + + describe('timezone formatting', () => { + const savedSearchId = ECOM_SAVED_SEARCH_ID; + const timeRange = { from: '2019-07-11T00:00:00.000Z', to: '2019-07-12T00:00:00.000Z' }; + + describe('export with custom timezone and timeRange from locator params', () => { + let response: request.Response; + let job: ReportApiJSON; + let path: string; + let csvFile: string; + + before(async () => { + await reportingAPI.initEcommerce(); + const { text, status } = await requestCsvFromSavedSearch({ + locatorParams: [ + { + id: DISCOVER_APP_LOCATOR, + version: 'reporting', + params: { savedSearchId, timeRange }, + }, + ], + browserTimezone: 'US/Alaska', + objectType: 'search', + version: 'reporting', + } as JobParamsCsvFromSavedObject); + expect(status).to.eql(200); + ({ job, path } = JSON.parse(text)); + await reportingAPI.waitForJobToFinish(path); + response = await supertest.get(path); + csvFile = response.text; + }); + + after(async () => { + await reportingAPI.teardownEcommerce(); + }); + + it('job response data is correct', () => { + expect(path).to.be.a('string'); + expect(job).to.be.an('object'); + expect(job.attempts).equal(0); + expectSnapshot({ + contentType: response.header['content-type'], + contentDisposition: response.header['content-disposition'], + title: job.payload.title, + }).toMatch(); + }); + + it('csv file matches', () => { + expectSnapshot(csvFile).toMatch(); + }); + }); + }); + }); + }); +}; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index 97c05e0af765..f0f11c93e62d 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -23,6 +23,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./security_roles_privileges')); loadTestFile(require.resolve('./download_csv_dashboard')); loadTestFile(require.resolve('./generate_csv_discover')); + loadTestFile(require.resolve('./csv_v2')); loadTestFile(require.resolve('./network_policy')); loadTestFile(require.resolve('./spaces')); loadTestFile(require.resolve('./usage')); diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 95f90a10734d..476b6223b959 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -115,5 +115,6 @@ "@kbn/cloud-security-posture-plugin", "@kbn/cloud-integration-saml-provider-plugin", "@kbn/security-api-integration-helpers", + "@kbn/discover-plugin", ] }