github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[7.x] [Reporting-CSV Export] Re-write CSV Export using SearchSource (#88303) (#94757)

Browse source 
* [Reporting-CSV Export] Re-write CSV Export using SearchSource (#88303)

* [Reporting-CSV Export] Re-write CSV Export using SearchSource

* replace PIT solution with scan-and-scroll

* update tests

* cleanup

* simplify pr

* update docs

* update docs

* update telemetry schema

* use getSearchRequestBody instead of flatten

* Revert "update docs"

This reverts commit ab9f4d9642.

* optimize some async calls

* cleanup

* --wip-- [skip ci]

* fix telemetry schema

* fix telemetry tests

* fix snapshot

* api docs

* api doc updates

* use import type

* format the data through chains of maps

* add another saved search to reporting/ecommerce_kibana

* add a failing test

* add error logging to query failures

* put clear scroll in a finally so the ES error can be captured

* log dat error

* set dat fieldsFromSource

* --wip-- [skip ci]

* Revert "add another saved search to reporting/ecommerce_kibana"

This reverts commit 6edf26eff2.

* functional test fixes

* clean up ecommerce test archive

* add test for new search with fieldsFromSource set

* add tests and refactor tests

* cleanup redundant conditionals

* add GenerateCsv.getFields

* fix some tests

* fix double-escaping

* fix test snapshots and refactoring

* fix other tests

* fix test

* fix default index pattern in functional tests

* fix ts and sort fields when they come from API response

* --wip-- [skip ci]

* fix formatting and increase maxSizeBytes for testing

* remove client-side logic for sanitizing fields

* do not prepend timefield name if it already is a column

* test the logic to prepend timeField

* test the logic to sort the fields

* fix functional test

* preserve the error from data.search

* add functional test for ES returning an error

* fix snapshot

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

* simplify logging tags

* update snapshots

* --wip-- [skip ci]

* fix types

* update jest snapshot

* Revert "simplify logging tags"

This reverts commit e844dbd6ab.

* Revert "update jest snapshot"

This reverts commit 88497529de.

* fix types again

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Tim Sullivan 2021-03-16 19:43:18 -07:00 • committed by GitHub
parent 5f500702e2
commit 60c8c0233d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
80 changed files with 6388 additions and 5790 deletions
Download patch file Download diff file Expand all files Collapse all files

50
api_docs/data.json
View file

@ -22013,7 +22013,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 245
"lineNumber": 246
},
"signature": [
"typeof ",
@ -22034,7 +22034,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 246
"lineNumber": 247
},
"signature": [
"typeof ",
@ -22055,7 +22055,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 247
"lineNumber": 248
},
"signature": [
"({ display: string; val: string; enabled(agg: ",
@ -22077,7 +22077,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 248
"lineNumber": 249
},
"signature": [
"typeof ",
@ -22098,7 +22098,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 249
"lineNumber": 250
},
"signature": [
"typeof ",
@ -22119,7 +22119,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 250
"lineNumber": 251
},
"signature": [
"typeof ",
@ -22140,7 +22140,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 251
"lineNumber": 252
},
"signature": [
"(agg: ",
@ -22162,7 +22162,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 252
"lineNumber": 253
},
"signature": [
"(agg: ",
@ -22184,7 +22184,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 253
"lineNumber": 254
},
"signature": [
"(...types: string[]) => (agg: ",
@ -22206,7 +22206,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 254
"lineNumber": 255
},
"signature": [
"typeof ",
@ -22227,7 +22227,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 255
"lineNumber": 256
},
"signature": [
"typeof ",
@ -22248,7 +22248,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 256
"lineNumber": 257
}
},
{
@ -22259,7 +22259,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 257
"lineNumber": 258
},
"signature": [
"typeof ",
@ -22280,7 +22280,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 258
"lineNumber": 259
},
"signature": [
"typeof ",
@ -22301,7 +22301,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 259
"lineNumber": 260
},
"signature": [
"typeof ",
@ -22322,7 +22322,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 260
"lineNumber": 261
}
},
{
@ -22333,7 +22333,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 261
"lineNumber": 262
},
"signature": [
"string[]"
@ -22347,7 +22347,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 262
"lineNumber": 263
},
"signature": [
"typeof ",
@ -22368,7 +22368,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 263
"lineNumber": 264
},
"signature": [
"typeof ",
@ -22386,7 +22386,7 @@
"label": "aggs",
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 244
"lineNumber": 245
}
},
{
@ -22397,7 +22397,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 265
"lineNumber": 266
},
"signature": [
"typeof ",
@ -22418,7 +22418,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 266
"lineNumber": 267
},
"signature": [
"typeof ",
@ -22439,7 +22439,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 267
"lineNumber": 268
},
"signature": [
"typeof ",
@ -22460,7 +22460,7 @@
"description": [],
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 268
"lineNumber": 269
},
"signature": [
"typeof ",
@ -22478,7 +22478,7 @@
"label": "search",
"source": {
"path": "src/plugins/data/server/index.ts",
"lineNumber": 243
"lineNumber": 244
},
"initialIsOpen": false
},

177
api_docs/data_search.json
View file

@ -1573,6 +1573,183 @@
}
],
"interfaces": [
{
"id": "def-server.IScopedSearchClient",
"type": "Interface",
"label": "IScopedSearchClient",
"signature": [
{
"pluginId": "data",
"scope": "server",
"docId": "kibDataSearchPluginApi",
"section": "def-server.IScopedSearchClient",
"text": "IScopedSearchClient"
},
" extends ",
{
"pluginId": "data",
"scope": "common",
"docId": "kibDataSearchPluginApi",
"section": "def-common.ISearchClient",
"text": "ISearchClient"
}
],
"description": [],
"tags": [],
"children": [
{
"tags": [],
"id": "def-server.IScopedSearchClient.saveSession",
"type": "Function",
"label": "saveSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 90
},
"signature": [
"(sessionId: string, attributes: Partial<unknown>) => Promise<",
{
"pluginId": "core",
"scope": "common",
"docId": "kibCorePluginApi",
"section": "def-common.SavedObject",
"text": "SavedObject"
},
"<unknown> | undefined>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.getSession",
"type": "Function",
"label": "getSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 91
},
"signature": [
"(sessionId: string) => Promise<",
{
"pluginId": "core",
"scope": "common",
"docId": "kibCorePluginApi",
"section": "def-common.SavedObject",
"text": "SavedObject"
},
"<unknown>>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.findSessions",
"type": "Function",
"label": "findSessions",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 92
},
"signature": [
"(options: Pick<",
{
"pluginId": "core",
"scope": "server",
"docId": "kibCoreSavedObjectsPluginApi",
"section": "def-server.SavedObjectsFindOptions",
"text": "SavedObjectsFindOptions"
},
", \"filter\" | \"fields\" | \"searchAfter\" | \"search\" | \"page\" | \"perPage\" | \"sortField\" | \"sortOrder\" | \"searchFields\" | \"rootSearchFields\" | \"hasReference\" | \"hasReferenceOperator\" | \"defaultSearchOperator\" | \"namespaces\" | \"typeToNamespacesMap\" | \"preference\" | \"pit\">) => Promise<",
{
"pluginId": "core",
"scope": "server",
"docId": "kibCoreSavedObjectsPluginApi",
"section": "def-server.SavedObjectsFindResponse",
"text": "SavedObjectsFindResponse"
},
"<unknown>>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.updateSession",
"type": "Function",
"label": "updateSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 93
},
"signature": [
"(sessionId: string, attributes: Partial<unknown>) => Promise<",
{
"pluginId": "core",
"scope": "server",
"docId": "kibCoreSavedObjectsPluginApi",
"section": "def-server.SavedObjectsUpdateResponse",
"text": "SavedObjectsUpdateResponse"
},
"<unknown>>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.cancelSession",
"type": "Function",
"label": "cancelSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 94
},
"signature": [
"(sessionId: string) => Promise<{}>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.deleteSession",
"type": "Function",
"label": "deleteSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 95
},
"signature": [
"(sessionId: string) => Promise<{}>"
]
},
{
"tags": [],
"id": "def-server.IScopedSearchClient.extendSession",
"type": "Function",
"label": "extendSession",
"description": [],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 96
},
"signature": [
"(sessionId: string, expires: Date) => Promise<",
{
"pluginId": "core",
"scope": "server",
"docId": "kibCoreSavedObjectsPluginApi",
"section": "def-server.SavedObjectsUpdateResponse",
"text": "SavedObjectsUpdateResponse"
},
"<unknown>>"
]
}
],
"source": {
"path": "src/plugins/data/server/search/types.ts",
"lineNumber": 89
},
"initialIsOpen": false
},
{
"id": "def-server.ISearchSessionService",
"type": "Interface",

19
api_docs/discover.json
View file

@ -40,6 +40,25 @@
"lineNumber": 18
},
"initialIsOpen": false
},
{
"id": "def-public.loadSharingDataHelpers",
"type": "Function",
"label": "loadSharingDataHelpers",
"signature": [
"() => Promise<typeof ",
"src/plugins/discover/public/application/helpers/get_sharing_data",
">"
],
"description": [],
"children": [],
"tags": [],
"returnComment": [],
"source": {
"path": "src/plugins/discover/public/shared/index.ts",
"lineNumber": 12
},
"initialIsOpen": false
}
],
"interfaces": [

283
api_docs/reporting.json
View file

@ -851,7 +851,29 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 58
"lineNumber": 69
}
},
{
"type": "Object",
"label": "context",
"isRequired": true,
"signature": [
{
"pluginId": "core",
"scope": "server",
"docId": "kibCorePluginApi",
"section": "def-server.PluginInitializerContext",
"text": "PluginInitializerContext"
},
"<Readonly<{ encryptionKey?: string | undefined; } & { enabled: boolean; index: string; capture: Readonly<{} & { browser: Readonly<{} & { type: string; chromium: Readonly<{ inspect?: boolean | undefined; disableSandbox?: boolean | undefined; } & { proxy: Readonly<{ server?: string | undefined; bypass?: string[] | undefined; } & { enabled: boolean; }>; }>; autoDownload: boolean; }>; timeouts: Readonly<{} & { openUrl: number | moment.Duration; waitForElements: number | moment.Duration; renderComplete: number | moment.Duration; }>; networkPolicy: Readonly<{} & { enabled: boolean; rules: Readonly<{ host?: string | undefined; protocol?: string | undefined; } & { allow: boolean; }>[]; }>; zoom: number; viewport: Readonly<{} & { height: number; width: number; }>; loadDelay: number | moment.Duration; maxAttempts: number; }>; kibanaServer: Readonly<{ hostname?: string | undefined; port?: number | undefined; protocol?: string | undefined; } & {}>; queue: Readonly<{} & { timeout: number | moment.Duration; pollInterval: number | moment.Duration; indexInterval: string; pollEnabled: boolean; pollIntervalErrorMultiplier: number; }>; csv: Readonly<{} & { scroll: Readonly<{} & { size: number; duration: string; }>; checkForFormulas: boolean; escapeFormulaValues: boolean; enablePanelActionDownload: boolean; maxSizeBytes: number | ",
"ByteSizeValue",
"; useByteOrderMarkEncoding: boolean; }>; roles: Readonly<{} & { allow: string[]; }>; poll: Readonly<{} & { jobCompletionNotifier: Readonly<{} & { interval: number; intervalErrorMultiplier: number; }>; jobsRefresh: Readonly<{} & { interval: number; intervalErrorMultiplier: number; }>; }>; }>>"
],
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 69
}
}
],
@ -859,7 +881,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 58
"lineNumber": 69
}
},
{
@ -895,7 +917,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 63
"lineNumber": 79
}
}
],
@ -903,7 +925,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 63
"lineNumber": 79
}
},
{
@ -939,7 +961,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 71
"lineNumber": 93
}
}
],
@ -947,7 +969,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 71
"lineNumber": 93
}
},
{
@ -963,7 +985,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 79
"lineNumber": 106
}
},
{
@ -979,7 +1001,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 90
"lineNumber": 117
}
},
{
@ -995,7 +1017,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 97
"lineNumber": 124
}
},
{
@ -1031,7 +1053,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 104
"lineNumber": 131
}
}
],
@ -1039,7 +1061,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 104
"lineNumber": 131
}
},
{
@ -1057,7 +1079,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 112
"lineNumber": 139
}
},
{
@ -1080,7 +1102,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 132
"lineNumber": 159
}
},
{
@ -1104,7 +1126,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 142
"lineNumber": 169
}
},
{
@ -1127,7 +1149,76 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 150
"lineNumber": 177
}
},
{
"id": "def-server.ReportingCore.scheduleTask",
"type": "Function",
"label": "scheduleTask",
"signature": [
"(report: ",
{
"pluginId": "reporting",
"scope": "server",
"docId": "kibReportingPluginApi",
"section": "def-server.ReportTaskParams",
"text": "ReportTaskParams"
},
"<",
{
"pluginId": "reporting",
"scope": "server",
"docId": "kibReportingPluginApi",
"section": "def-server.BasePayload",
"text": "BasePayload"
},
">) => Promise<",
{
"pluginId": "taskManager",
"scope": "server",
"docId": "kibTaskManagerPluginApi",
"section": "def-server.ConcreteTaskInstance",
"text": "ConcreteTaskInstance"
},
">"
],
"description": [],
"children": [
{
"type": "Object",
"label": "report",
"isRequired": true,
"signature": [
{
"pluginId": "reporting",
"scope": "server",
"docId": "kibReportingPluginApi",
"section": "def-server.ReportTaskParams",
"text": "ReportTaskParams"
},
"<",
{
"pluginId": "reporting",
"scope": "server",
"docId": "kibReportingPluginApi",
"section": "def-server.BasePayload",
"text": "BasePayload"
},
">"
],
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 181
}
}
],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 181
}
},
{
@ -1151,7 +1242,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 154
"lineNumber": 185
}
},
{
@ -1175,7 +1266,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 158
"lineNumber": 189
}
},
{
@ -1199,7 +1290,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 168
"lineNumber": 199
}
},
{
@ -1222,7 +1313,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 177
"lineNumber": 208
}
},
{
@ -1245,7 +1336,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 184
"lineNumber": 216
}
},
{
@ -1291,7 +1382,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 193
"lineNumber": 225
}
}
],
@ -1299,7 +1390,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 193
"lineNumber": 225
}
},
{
@ -1344,7 +1435,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 199
"lineNumber": 231
}
},
{
@ -1363,7 +1454,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 199
"lineNumber": 231
}
}
],
@ -1371,7 +1462,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 199
"lineNumber": 231
}
},
{
@ -1409,7 +1500,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 213
"lineNumber": 245
}
},
{
@ -1422,7 +1513,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 213
"lineNumber": 245
}
},
{
@ -1441,7 +1532,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 213
"lineNumber": 245
}
}
],
@ -1449,7 +1540,7 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 213
"lineNumber": 245
}
},
{
@ -1502,7 +1593,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 233
"lineNumber": 265
}
},
{
@ -1521,7 +1612,7 @@
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 233
"lineNumber": 265
}
}
],
@ -1529,13 +1620,137 @@
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 233
"lineNumber": 265
}
},
{
"id": "def-server.ReportingCore.getDataService",
"type": "Function",
"label": "getDataService",
"signature": [
"() => Promise<",
{
"pluginId": "data",
"scope": "server",
"docId": "kibDataPluginApi",
"section": "def-server.DataPluginStart",
"text": "DataPluginStart"
},
">"
],
"description": [],
"children": [],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 275
}
},
{
"id": "def-server.ReportingCore.getEsClient",
"type": "Function",
"label": "getEsClient",
"signature": [
"() => Promise<",
{
"pluginId": "core",
"scope": "server",
"docId": "kibCorePluginApi",
"section": "def-server.IClusterClient",
"text": "IClusterClient"
},
">"
],
"description": [],
"children": [],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 280
}
},
{
"id": "def-server.ReportingCore.trackReport",
"type": "Function",
"label": "trackReport",
"signature": [
"(reportId: string) => void"
],
"description": [],
"children": [
{
"type": "string",
"label": "reportId",
"isRequired": true,
"signature": [
"string"
],
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 285
}
}
],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 285
}
},
{
"id": "def-server.ReportingCore.untrackReport",
"type": "Function",
"label": "untrackReport",
"signature": [
"(reportId: string) => void"
],
"description": [],
"children": [
{
"type": "string",
"label": "reportId",
"isRequired": true,
"signature": [
"string"
],
"description": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 289
}
}
],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 289
}
},
{
"id": "def-server.ReportingCore.countConcurrentReports",
"type": "Function",
"label": "countConcurrentReports",
"signature": [
"() => number"
],
"description": [],
"children": [],
"tags": [],
"returnComment": [],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 293
}
}
],
"source": {
"path": "x-pack/plugins/reporting/server/core.ts",
"lineNumber": 50
"lineNumber": 58
},
"initialIsOpen": false
},
@ -2230,4 +2445,4 @@
}
]
}
}
}

3
src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts
View file

@ -97,8 +97,7 @@ export const getTopNavLinks = ({
const sharingData = await getSharingData(
searchSource,
state.appStateContainer.getState(),
services.uiSettings,
getFieldCounts
services.uiSettings
);
services.share.toggleShareContextMenu({
anchorElement,

157
src/plugins/discover/public/application/helpers/get_sharing_data.test.ts
View file

@ -11,59 +11,130 @@ import { getSharingData, showPublicUrlSwitch } from './get_sharing_data';
import { IUiSettingsClient } from 'kibana/public';
import { createSearchSourceMock } from '../../../../data/common/search/search_source/mocks';
import { indexPatternMock } from '../../__mocks__/index_pattern';
import { SORT_DEFAULT_ORDER_SETTING } from '../../../common';
import { DOC_HIDE_TIME_COLUMN_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common';
import { IndexPattern } from 'src/plugins/data/public';
describe('getSharingData', () => {
let mockConfig: IUiSettingsClient;
beforeEach(() => {
mockConfig = ({
get: (key: string) => {
if (key === SORT_DEFAULT_ORDER_SETTING) {
return 'desc';
}
if (key === DOC_HIDE_TIME_COLUMN_SETTING) {
return false;
}
return false;
},
} as unknown) as IUiSettingsClient;
});
test('returns valid data for sharing', async () => {
const searchSourceMock = createSearchSourceMock({ index: indexPatternMock });
const result = await getSharingData(searchSourceMock, { columns: [] }, mockConfig);
expect(result).toMatchInlineSnapshot(`
Object {
"searchSource": Object {
"index": "the-index-pattern-id",
"sort": Array [
Object {
"_score": "desc",
},
],
},
}
`);
});
test('fields have prepended timeField', async () => {
const index = { ...indexPatternMock } as IndexPattern;
index.timeFieldName = 'cool-timefield';
const searchSourceMock = createSearchSourceMock({ index });
const result = await getSharingData(
searchSourceMock,
{ columns: [] },
({
get: (key: string) => {
if (key === SORT_DEFAULT_ORDER_SETTING) {
return 'desc';
}
return false;
},
} as unknown) as IUiSettingsClient,
() => Promise.resolve({})
{
columns: [
'cool-field-1',
'cool-field-2',
'cool-field-3',
'cool-field-4',
'cool-field-5',
'cool-field-6',
],
},
mockConfig
);
expect(result).toMatchInlineSnapshot(`
Object {
"conflictedTypesFields": Array [],
"fields": Array [],
"indexPatternId": "the-index-pattern-id",
"metaFields": Array [
"_index",
"_score",
],
"searchRequest": Object {
"body": Object {
"_source": Object {},
"fields": Array [],
"query": Object {
"bool": Object {
"filter": Array [],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
"searchSource": Object {
"fields": Array [
"cool-timefield",
"cool-field-1",
"cool-field-2",
"cool-field-3",
"cool-field-4",
"cool-field-5",
"cool-field-6",
],
"index": "the-index-pattern-id",
"sort": Array [
Object {
"_doc": "desc",
},
"runtime_mappings": Object {},
"script_fields": Object {},
"sort": Array [
Object {
"_score": Object {
"order": "desc",
},
},
],
"stored_fields": Array [
"*",
],
},
"index": "the-index-pattern-title",
],
},
}
`);
});
test('fields conditionally do not have prepended timeField', async () => {
mockConfig = ({
get: (key: string) => {
if (key === DOC_HIDE_TIME_COLUMN_SETTING) {
return true;
}
return false;
},
} as unknown) as IUiSettingsClient;
const index = { ...indexPatternMock } as IndexPattern;
index.timeFieldName = 'cool-timefield';
const searchSourceMock = createSearchSourceMock({ index });
const result = await getSharingData(
searchSourceMock,
{
columns: [
'cool-field-1',
'cool-field-2',
'cool-field-3',
'cool-field-4',
'cool-field-5',
'cool-field-6',
],
},
mockConfig
);
expect(result).toMatchInlineSnapshot(`
Object {
"searchSource": Object {
"fields": Array [
"cool-field-1",
"cool-field-2",
"cool-field-3",
"cool-field-4",
"cool-field-5",
"cool-field-6",
],
"index": "the-index-pattern-id",
"sort": Array [
Object {
"_doc": false,
},
],
},
}
`);

73
src/plugins/discover/public/application/helpers/get_sharing_data.ts
View file

@ -6,57 +6,28 @@
* Side Public License, v 1.
*/
import { Capabilities, IUiSettingsClient } from 'kibana/public';
import type { Capabilities, IUiSettingsClient } from 'kibana/public';
import { DOC_HIDE_TIME_COLUMN_SETTING, SORT_DEFAULT_ORDER_SETTING } from '../../../common';
import { getSortForSearchSource } from '../angular/doc_table';
import { ISearchSource } from '../../../../data/common';
import { AppState } from '../angular/discover_state';
import { SortOrder } from '../../saved_searches/types';
const getSharingDataFields = async (
getFieldCounts: () => Promise<Record<string, number>>,
selectedFields: string[],
timeFieldName: string,
hideTimeColumn: boolean
) => {
if (
selectedFields.length === 0 ||
(selectedFields.length === 1 && selectedFields[0] === '_source')
) {
const fieldCounts = await getFieldCounts();
return {
searchFields: undefined,
selectFields: Object.keys(fieldCounts).sort(),
};
}
const fields =
timeFieldName && !hideTimeColumn ? [timeFieldName, ...selectedFields] : selectedFields;
return {
searchFields: fields,
selectFields: fields,
};
};
import type { SavedSearch, SortOrder } from '../../saved_searches/types';
/**
* Preparing data to share the current state as link or CSV/Report
*/
export async function getSharingData(
currentSearchSource: ISearchSource,
state: AppState,
config: IUiSettingsClient,
getFieldCounts: () => Promise<Record<string, number>>
state: AppState | SavedSearch,
config: IUiSettingsClient
) {
const searchSource = currentSearchSource.createCopy();
const index = searchSource.getField('index')!;
const fields = {
fields: searchSource.getField('fields'),
fieldsFromSource: searchSource.getField('fieldsFromSource'),
};
const { searchFields, selectFields } = await getSharingDataFields(
getFieldCounts,
state.columns || [],
index.timeFieldName || '',
config.get(DOC_HIDE_TIME_COLUMN_SETTING)
);
searchSource.setField('fieldsFromSource', searchFields);
searchSource.setField(
'sort',
getSortForSearchSource(state.sort as SortOrder[], index, config.get(SORT_DEFAULT_ORDER_SETTING))
@ -66,17 +37,27 @@ export async function getSharingData(
searchSource.removeField('aggs');
searchSource.removeField('size');
const body = await searchSource.getSearchRequestBody();
// fields get re-set to match the saved search columns
let columns = state.columns || [];
if (columns && columns.length > 0) {
// conditionally add the time field column:
let timeFieldName: string | undefined;
const hideTimeColumn = config.get(DOC_HIDE_TIME_COLUMN_SETTING);
if (!hideTimeColumn && index && index.timeFieldName) {
timeFieldName = index.timeFieldName;
}
if (timeFieldName && !columns.includes(timeFieldName)) {
columns = [timeFieldName, ...columns];
}
// if columns were selected in the saved search, use them for the searchSource's fields
const fieldsKey = fields.fieldsFromSource ? 'fieldsFromSource' : 'fields';
searchSource.setField(fieldsKey, columns);
}
return {
searchRequest: {
index: index.title,
body,
},
fields: selectFields,
metaFields: index.metaFields,
conflictedTypesFields: index.fields.filter((f) => f.type === 'conflict').map((f) => f.name),
indexPatternId: index.id,
searchSource: searchSource.getSerializedFields(true),
};
}

10
x-pack/plugins/reporting/common/constants.ts
View file

@ -50,6 +50,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';
export const UI_SETTINGS_DATEFORMAT_TZ = 'dateFormat:tz';
export const LAYOUT_TYPES = {
PRESERVE_LAYOUT: 'preserve_layout',
@ -57,13 +58,16 @@ export const LAYOUT_TYPES = {
};
// Export Type Definitions
export const CSV_REPORT_TYPE = 'CSV';
export const CSV_JOB_TYPE = 'csv_searchsource';
export const PDF_REPORT_TYPE = 'printablePdf';
export const PDF_JOB_TYPE = 'printable_pdf';
export const PNG_REPORT_TYPE = 'PNG';
export const PNG_JOB_TYPE = 'PNG';
export const CSV_FROM_SAVEDOBJECT_JOB_TYPE = 'csv_from_savedobject';
export const CSV_SEARCHSOURCE_IMMEDIATE_TYPE = 'csv_searchsource_immediate';
// This is deprecated because it lacks support for runtime fields
// but the extension points are still needed for pre-existing scripted automation, until 8.0
@ -86,9 +90,9 @@ export const API_BASE_GENERATE = `${API_BASE_URL}/generate`;
export const API_LIST_URL = `${API_BASE_URL}/jobs`;
export const API_DIAGNOSE_URL = `${API_BASE_URL}/diagnose`;
// hacky endpoint
// hacky endpoint: download CSV without queueing a report
export const API_BASE_URL_V1 = '/api/reporting/v1'; //
export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv/saved-object`;
export const API_GENERATE_IMMEDIATE = `${API_BASE_URL_V1}/generate/immediate/csv_searchsource`;
// Management UI route
export const REPORTING_MANAGEMENT_HOME = '/app/management/insightsAndAlerting/reporting';

3
x-pack/plugins/reporting/common/types.ts
View file

@ -47,9 +47,10 @@ export interface ReportDocumentHead {
export interface TaskRunResult {
content_type: string | null;
content: string | null;
csv_contains_formulas?: boolean;
size: number;
csv_contains_formulas?: boolean;
max_size_reached?: boolean;
needs_sorting?: boolean;
warnings?: string[];
}

10
x-pack/plugins/reporting/public/components/reporting_panel_content.tsx
View file

@ -11,11 +11,7 @@ import React, { Component, ReactElement } from 'react';
import { ToastsSetup } from 'src/core/public';
import url from 'url';
import { toMountPoint } from '../../../../../src/plugins/kibana_react/public';
import {
CSV_REPORT_TYPE_DEPRECATED,
PDF_REPORT_TYPE,
PNG_REPORT_TYPE,
} from '../../common/constants';
import { CSV_REPORT_TYPE, PDF_REPORT_TYPE, PNG_REPORT_TYPE } from '../../common/constants';
import { BaseParams } from '../../common/types';
import { ReportingAPIClient } from '../lib/reporting_api_client';
@ -177,8 +173,8 @@ class ReportingPanelContentUi extends Component<Props, State> {
switch (this.props.reportType) {
case PDF_REPORT_TYPE:
return 'PDF';
case 'csv':
return CSV_REPORT_TYPE_DEPRECATED;
case 'csv_searchsource':
return CSV_REPORT_TYPE;
case 'png':
return PNG_REPORT_TYPE;
default:

15
x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.test.ts
View file

@ -52,7 +52,20 @@ describe('GetCsvReportPanelAction', () => {
context = {
embeddable: {
type: 'search',
getSavedSearch: () => ({ id: 'lebowski' }),
getSavedSearch: () => {
const searchSource = {
createCopy: () => searchSource,
removeField: jest.fn(),
setField: jest.fn(),
getField: jest.fn().mockImplementation((key: string) => {
if (key === 'index') {
return 'my-test-index-*';
}
}),
getSerializedFields: jest.fn().mockImplementation(() => ({})),
};
return { searchSource };
},
getTitle: () => `The Dude`,
getInspectorAdapters: () => null,
getInput: () => ({

62
x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx
View file

@ -5,13 +5,13 @@
* 2.0.
*/
import dateMath from '@elastic/datemath';
import { i18n } from '@kbn/i18n';
import _ from 'lodash';
import moment from 'moment-timezone';
import { CoreSetup } from 'src/core/public';
import {
loadSharingDataHelpers,
ISearchEmbeddable,
SavedSearch,
SEARCH_EMBEDDABLE_TYPE,
} from '../../../../../src/plugins/discover/public';
import { IEmbeddable, ViewMode } from '../../../../../src/plugins/embeddable/public';
@ -21,6 +21,7 @@ import {
} from '../../../../../src/plugins/ui_actions/public';
import { LicensingPluginSetup } from '../../../licensing/public';
import { API_GENERATE_IMMEDIATE, CSV_REPORTING_ACTION } from '../../common/constants';
import { JobParamsDownloadCSV } from '../../server/export_types/csv_searchsource_immediate/types';
import { checkLicense } from '../lib/license_check';
function isSavedSearchEmbeddable(
@ -61,17 +62,16 @@ export class GetCsvReportPanelAction implements ActionDefinition<ActionContext>
});
}
public getSearchRequestBody({ searchEmbeddable }: { searchEmbeddable: any }) {
const adapters = searchEmbeddable.getInspectorAdapters();
if (!adapters) {
return {};
}
public async getSearchSource(savedSearch: SavedSearch, embeddable: ISearchEmbeddable) {
const { getSharingData } = await loadSharingDataHelpers();
const searchSource = savedSearch.searchSource.createCopy();
const { searchSource: serializedSearchSource } = await getSharingData(
searchSource,
savedSearch, // TODO: get unsaved state (using embeddale.searchScope): https://github.com/elastic/kibana/issues/43977
this.core.uiSettings
);
if (adapters.requests.requests.length === 0) {
return {};
}
return searchEmbeddable.getSavedSearch().searchSource.getSearchRequestBody();
return serializedSearchSource;
}
public isCompatible = async (context: ActionContext) => {
@ -95,34 +95,18 @@ export class GetCsvReportPanelAction implements ActionDefinition<ActionContext>
return;
}
const {
timeRange: { to, from },
} = embeddable.getInput();
const savedSearch = embeddable.getSavedSearch();
const searchSource = await this.getSearchSource(savedSearch, embeddable);
const searchEmbeddable = embeddable;
const searchRequestBody = await this.getSearchRequestBody({ searchEmbeddable });
const state = _.pick(searchRequestBody, ['sort', 'docvalue_fields', 'query']);
const kibanaTimezone = this.core.uiSettings.get('dateFormat:tz');
const browserTimezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone;
const immediateJobParams: JobParamsDownloadCSV = {
searchSource,
browserTimezone,
title: savedSearch.title,
};
const id = `search:${embeddable.getSavedSearch().id}`;
const timezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone;
const fromTime = dateMath.parse(from);
const toTime = dateMath.parse(to, { roundUp: true });
if (!fromTime || !toTime) {
return this.onGenerationFail(
new Error(`Invalid time range: From: ${fromTime}, To: ${toTime}`)
);
}
const body = JSON.stringify({
timerange: {
min: fromTime.format(),
max: toTime.format(),
timezone,
},
state,
});
const body = JSON.stringify(immediateJobParams);
this.isDownloading = true;
@ -137,11 +121,11 @@ export class GetCsvReportPanelAction implements ActionDefinition<ActionContext>
});
await this.core.http
.post(`${API_GENERATE_IMMEDIATE}/${id}`, { body })
.post(`${API_GENERATE_IMMEDIATE}`, { body })
.then((rawResponse: string) => {
this.isDownloading = false;
const download = `${embeddable.getSavedSearch().title}.csv`;
const download = `${savedSearch.title}.csv`;
const blob = new Blob([rawResponse], { type: 'text/csv;charset=utf-8;' });
// Hack for IE11 Support

20
x-pack/plugins/reporting/public/share_context_menu/register_csv_reporting.tsx
View file

@ -11,10 +11,8 @@ import React from 'react';
import { IUiSettingsClient, ToastsSetup } from 'src/core/public';
import { ShareContext } from '../../../../../src/plugins/share/public';
import { LicensingPluginSetup } from '../../../licensing/public';
import {
JobParamsDeprecatedCSV,
SearchRequestDeprecatedCSV,
} from '../../server/export_types/csv/types';
import { CSV_JOB_TYPE } from '../../common/constants';
import { JobParamsCSV } from '../../server/export_types/csv_searchsource/types';
import { ReportingPanelContent } from '../components/reporting_panel_content_lazy';
import { checkLicense } from '../lib/license_check';
import { ReportingAPIClient } from '../lib/reporting_api_client';
@ -56,22 +54,18 @@ export const csvReportingProvider = ({
objectType,
objectId,
sharingData,
isDirty,
onClose,
isDirty,
}: ShareContext) => {
if ('search' !== objectType) {
return [];
}
const jobParams: JobParamsDeprecatedCSV = {
const jobParams: JobParamsCSV = {
browserTimezone,
objectType,
title: sharingData.title as string,
indexPatternId: sharingData.indexPatternId as string,
searchRequest: sharingData.searchRequest as SearchRequestDeprecatedCSV,
fields: sharingData.fields as string[],
metaFields: sharingData.metaFields as string[],
conflictedTypesFields: sharingData.conflictedTypesFields as string[],
objectType,
searchSource: sharingData.searchSource,
};
const getJobParams = () => jobParams;
@ -99,7 +93,7 @@ export const csvReportingProvider = ({
<ReportingPanelContent
apiClient={apiClient}
toasts={toasts}
reportType="csv"
reportType={CSV_JOB_TYPE}
layoutId={undefined}
objectId={objectId}
getJobParams={getJobParams}

21
x-pack/plugins/reporting/server/core.ts
View file

@ -11,6 +11,7 @@ import { first, map, take } from 'rxjs/operators';
import {
BasePath,
ElasticsearchServiceSetup,
IClusterClient,
KibanaRequest,
SavedObjectsClientContract,
SavedObjectsServiceStart,
@ -28,6 +29,7 @@ import { ESQueueInstance } from './lib/create_queue';
import { screenshotsObservableFactory, ScreenshotsObservableFn } from './lib/screenshots';
import { ReportingStore } from './lib/store';
import { ReportingPluginRouter } from './types';
import { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server';
export interface ReportingInternalSetup {
basePath: Pick<BasePath, 'set'>;
@ -41,10 +43,12 @@ export interface ReportingInternalSetup {
export interface ReportingInternalStart {
browserDriverFactory: HeadlessChromiumDriverFactory;
esqueue: ESQueueInstance;
store: ReportingStore;
savedObjects: SavedObjectsServiceStart;
uiSettings: UiSettingsServiceStart;
esClient: IClusterClient;
data: DataPluginStart;
esqueue: ESQueueInstance;
}
export class ReportingCore {
@ -155,6 +159,10 @@ export class ReportingCore {
return (await this.getPluginStartDeps()).esqueue;
}
public async getStore() {
return (await this.getPluginStartDeps()).store;
}
public async getLicenseInfo() {
const { licensing } = this.getPluginSetupDeps();
return await licensing.license$
@ -181,6 +189,7 @@ export class ReportingCore {
return this.pluginSetupDeps;
}
// NOTE: Uses the Legacy API
public getElasticsearchService() {
return this.getPluginSetupDeps().elasticsearch;
}
@ -239,4 +248,14 @@ export class ReportingCore {
const savedObjectsClient = await this.getSavedObjectsClient(request);
return await this.getUiSettingsServiceFactory(savedObjectsClient);
}
public async getDataService() {
const startDeps = await this.getPluginStartDeps();
return startDeps.data;
}
public async getEsClient() {
const startDeps = await this.getPluginStartDeps();
return startDeps.esClient;
}
}

1
x-pack/plugins/reporting/server/export_types/common/index.ts
View file

@ -12,7 +12,6 @@ export { omitBlockedHeaders } from './omit_blocked_headers';
export { validateUrls } from './validate_urls';
export interface TimeRangeParams {
timezone: string;
min?: Date | string | number | null;
max?: Date | string | number | null;
}

2
x-pack/plugins/reporting/server/export_types/csv/generate_csv/check_cells_for_formulas.ts
View file

@ -6,7 +6,7 @@
*/
import { pick, keys, values, some } from 'lodash';
import { cellHasFormulas } from './cell_has_formula';
import { cellHasFormulas } from '../../csv_searchsource/generate_csv/cell_has_formula';
interface IFlattened {
[header: string]: string;

11
x-pack/plugins/reporting/server/export_types/csv/generate_csv/index.ts
View file

@ -13,15 +13,18 @@ import { CSV_BOM_CHARS } from '../../../../common/constants';
import { byteSizeValueToNumber } from '../../../../common/schema_utils';
import { LevelLogger } from '../../../lib';
import { getFieldFormats } from '../../../services';
import { IndexPatternSavedObjectDeprecatedCSV, SavedSearchGeneratorResult } from '../types';
import { createEscapeValue } from '../../csv_searchsource/generate_csv/escape_value';
import { MaxSizeStringBuilder } from '../../csv_searchsource/generate_csv/max_size_string_builder';
import {
IndexPatternSavedObjectDeprecatedCSV,
SavedSearchGeneratorResultDeprecatedCSV,
} from '../types';
import { checkIfRowsHaveFormulas } from './check_cells_for_formulas';
import { createEscapeValue } from './escape_value';
import { fieldFormatMapFactory } from './field_format_map';
import { createFlattenHit } from './flatten_hit';
import { createFormatCsvValues } from './format_csv_values';
import { getUiSettings } from './get_ui_settings';
import { createHitIterator, EndpointCaller } from './hit_iterator';
import { MaxSizeStringBuilder } from './max_size_string_builder';
interface SearchRequest {
index: string;
@ -55,7 +58,7 @@ export function createGenerateCsv(logger: LevelLogger) {
uiSettingsClient: IUiSettingsClient,
callEndpoint: EndpointCaller,
cancellationToken: CancellationToken
): Promise<SavedSearchGeneratorResult> {
): Promise<SavedSearchGeneratorResultDeprecatedCSV> {
const settings = await getUiSettings(job.browserTimezone, uiSettingsClient, config, logger);
const escapeValue = createEscapeValue(settings.quoteValues, settings.escapeFormulaValues);
const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';

7
x-pack/plugins/reporting/server/export_types/csv/types.d.ts vendored
View file

@ -77,15 +77,10 @@ type FormatsMapDeprecatedCSV = Map<
}
>;
export interface SavedSearchGeneratorResult {
export interface SavedSearchGeneratorResultDeprecatedCSV {
content: string;
size: number;
maxSizeReached: boolean;
csvContainsFormulas?: boolean;
warnings: string[];
}
export interface CsvResultFromSearch {
type: string;
result: SavedSearchGeneratorResult;
}

96
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts
View file

@ -1,96 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { notFound, notImplemented } from '@hapi/boom';
import { get } from 'lodash';
import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
import { CsvFromSavedObjectRequest } from '../../routes/generate_from_savedobject_immediate';
import { CreateJobFnFactory } from '../../types';
import {
JobParamsPanelCsv,
JobPayloadPanelCsv,
SavedObject,
SavedObjectReference,
SavedObjectServiceError,
VisObjectAttributesJSON,
} from './types';
import type { ReportingRequestHandlerContext } from '../../types';
export type ImmediateCreateJobFn = (
jobParams: JobParamsPanelCsv,
context: ReportingRequestHandlerContext,
req: CsvFromSavedObjectRequest
) => Promise<JobPayloadPanelCsv>;
export const createJobFnFactory: CreateJobFnFactory<ImmediateCreateJobFn> = function createJobFactoryFn(
reporting,
parentLogger
) {
const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'create-job']);
return async function createJob(jobParams, context, req) {
const { savedObjectType, savedObjectId } = jobParams;
const panel = await Promise.resolve()
.then(() => context.core.savedObjects.client.get(savedObjectType, savedObjectId))
.then(async (savedObject: SavedObject) => {
const { attributes, references } = savedObject;
const { kibanaSavedObjectMeta: kibanaSavedObjectMetaJSON } = attributes;
const { timerange } = req.body;
if (!kibanaSavedObjectMetaJSON) {
throw new Error('Could not parse saved object data!');
}
const kibanaSavedObjectMeta = {
...kibanaSavedObjectMetaJSON,
searchSource: JSON.parse(kibanaSavedObjectMetaJSON.searchSourceJSON),
};
const { visState: visStateJSON } = attributes as VisObjectAttributesJSON;
if (visStateJSON) {
throw notImplemented('Visualization types are not yet implemented');
}
// saved search type
const { searchSource } = kibanaSavedObjectMeta;
if (!searchSource || !references) {
throw new Error('The saved search object is missing configuration fields!');
}
const indexPatternMeta = references.find(
(ref: SavedObjectReference) => ref.type === 'index-pattern'
);
if (!indexPatternMeta) {
throw new Error('Could not find index pattern for the saved search!');
}
return {
attributes: {
...attributes,
kibanaSavedObjectMeta: { searchSource },
},
indexPatternSavedObjectId: indexPatternMeta.id,
timerange,
};
})
.catch((err: Error) => {
const boomErr = (err as unknown) as { isBoom: boolean };
if (boomErr.isBoom) {
throw err;
}
const errPayload: SavedObjectServiceError = get(err, 'output.payload', { statusCode: 0 });
if (errPayload.statusCode === 404) {
throw notFound(errPayload.message);
}
logger.error(err);
throw new Error(`Unable to create a job from saved object data! Error: ${err}`);
});
return { ...jobParams, panel };
};
};

80
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts
View file

@ -1,80 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { KibanaRequest } from 'src/core/server';
import { CancellationToken } from '../../../common';
import { CONTENT_TYPE_CSV, CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
import { TaskRunResult } from '../../lib/tasks';
import { RunTaskFnFactory } from '../../types';
import { createGenerateCsv } from '../csv/generate_csv';
import { getGenerateCsvParams } from './lib/get_csv_job';
import { JobPayloadPanelCsv } from './types';
import type { ReportingRequestHandlerContext } from '../../types';
/*
* ImmediateExecuteFn receives the job doc payload because the payload was
* generated in the ScheduleFn
*/
export type ImmediateExecuteFn = (
jobId: null,
job: JobPayloadPanelCsv,
context: ReportingRequestHandlerContext,
req: KibanaRequest
) => Promise<TaskRunResult>;
export const runTaskFnFactory: RunTaskFnFactory<ImmediateExecuteFn> = function executeJobFactoryFn(
reporting,
parentLogger
) {
const config = reporting.getConfig();
const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'execute-job']);
return async function runTask(jobId, jobPayload, context, req) {
const generateCsv = createGenerateCsv(logger);
const { panel } = jobPayload;
logger.debug(`Execute job generating saved search CSV`);
const savedObjectsClient = context.core.savedObjects.client;
const uiSettingsClient = await reporting.getUiSettingsServiceFactory(savedObjectsClient);
const job = await getGenerateCsvParams(
jobPayload,
panel,
savedObjectsClient,
uiSettingsClient,
logger
);
const elasticsearch = reporting.getElasticsearchService();
const { callAsCurrentUser } = elasticsearch.legacy.client.asScoped(req);
const { content, maxSizeReached, size, csvContainsFormulas, warnings } = await generateCsv(
job,
config,
uiSettingsClient,
callAsCurrentUser,
new CancellationToken() // can not be cancelled
);
if (csvContainsFormulas) {
logger.warn(`CSV may contain formulas whose values have been escaped`);
}
if (maxSizeReached) {
logger.warn(`Max size reached: CSV output truncated to ${size} bytes`);
}
return {
content_type: CONTENT_TYPE_CSV,
content,
max_size_reached: maxSizeReached,
size,
csv_contains_formulas: csvContainsFormulas,
warnings,
};
};
};

340
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.test.ts
View file

@ -1,340 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createMockLevelLogger } from '../../../test_helpers';
import { JobParamsPanelCsv, SearchPanel } from '../types';
import { getGenerateCsvParams } from './get_csv_job';
const logger = createMockLevelLogger();
describe('Get CSV Job', () => {
let mockJobParams: JobParamsPanelCsv;
let mockSearchPanel: SearchPanel;
let mockSavedObjectsClient: any;
let mockUiSettingsClient: any;
beforeEach(() => {
mockJobParams = { savedObjectType: 'search', savedObjectId: '234-ididid' };
mockSearchPanel = {
indexPatternSavedObjectId: '123-indexId',
attributes: {
title: 'my search',
sort: [],
kibanaSavedObjectMeta: {
searchSource: { query: { isSearchSourceQuery: true }, filter: [] },
},
uiState: 56,
},
timerange: { timezone: 'PST', min: 0, max: 100 },
};
mockSavedObjectsClient = {
get: () => ({
attributes: { fields: null, title: null, timeFieldName: null },
}),
};
mockUiSettingsClient = {
get: () => ({}),
};
});
it('creates a data structure needed by generateCsv', async () => {
const result = await getGenerateCsvParams(
mockJobParams,
mockSearchPanel,
mockSavedObjectsClient,
mockUiSettingsClient,
logger
);
expect(result).toMatchInlineSnapshot(`
Object {
"browserTimezone": "PST",
"conflictedTypesFields": Array [],
"fields": Array [],
"indexPatternSavedObject": Object {
"attributes": Object {
"fields": null,
"timeFieldName": null,
"title": null,
},
"fields": Array [],
"timeFieldName": null,
"title": null,
},
"metaFields": Array [],
"searchRequest": Object {
"body": Object {
"_source": Object {
"includes": Array [],
},
"docvalue_fields": undefined,
"query": Object {
"bool": Object {
"filter": Array [],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
},
"script_fields": Object {},
"sort": Array [],
},
"index": null,
},
}
`);
});
it('uses query and sort from the payload', async () => {
mockJobParams.post = {
state: {
query: ['this is the query'],
sort: ['this is the sort'],
},
};
const result = await getGenerateCsvParams(
mockJobParams,
mockSearchPanel,
mockSavedObjectsClient,
mockUiSettingsClient,
logger
);
expect(result).toMatchInlineSnapshot(`
Object {
"browserTimezone": "PST",
"conflictedTypesFields": Array [],
"fields": Array [],
"indexPatternSavedObject": Object {
"attributes": Object {
"fields": null,
"timeFieldName": null,
"title": null,
},
"fields": Array [],
"timeFieldName": null,
"title": null,
},
"metaFields": Array [],
"searchRequest": Object {
"body": Object {
"_source": Object {
"includes": Array [],
},
"docvalue_fields": undefined,
"query": Object {
"bool": Object {
"filter": Array [
Object {
"0": "this is the query",
},
],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
},
"script_fields": Object {},
"sort": Array [
"this is the sort",
],
},
"index": null,
},
}
`);
});
it('uses timerange timezone from the payload', async () => {
mockJobParams.post = {
timerange: { timezone: 'Africa/Timbuktu', min: 0, max: 9000 },
};
const result = await getGenerateCsvParams(
mockJobParams,
mockSearchPanel,
mockSavedObjectsClient,
mockUiSettingsClient,
logger
);
expect(result).toMatchInlineSnapshot(`
Object {
"browserTimezone": "Africa/Timbuktu",
"conflictedTypesFields": Array [],
"fields": Array [],
"indexPatternSavedObject": Object {
"attributes": Object {
"fields": null,
"timeFieldName": null,
"title": null,
},
"fields": Array [],
"timeFieldName": null,
"title": null,
},
"metaFields": Array [],
"searchRequest": Object {
"body": Object {
"_source": Object {
"includes": Array [],
},
"docvalue_fields": undefined,
"query": Object {
"bool": Object {
"filter": Array [],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
},
"script_fields": Object {},
"sort": Array [],
},
"index": null,
},
}
`);
});
it('uses timerange min and max (numeric) when index pattern has timefieldName', async () => {
mockJobParams.post = {
timerange: { timezone: 'Africa/Timbuktu', min: 0, max: 900000000 },
};
mockSavedObjectsClient = {
get: () => ({
attributes: { fields: null, title: 'test search', timeFieldName: '@test_time' },
}),
};
const result = await getGenerateCsvParams(
mockJobParams,
mockSearchPanel,
mockSavedObjectsClient,
mockUiSettingsClient,
logger
);
expect(result).toMatchInlineSnapshot(`
Object {
"browserTimezone": "Africa/Timbuktu",
"conflictedTypesFields": Array [],
"fields": Array [
"@test_time",
],
"indexPatternSavedObject": Object {
"attributes": Object {
"fields": null,
"timeFieldName": "@test_time",
"title": "test search",
},
"fields": Array [],
"timeFieldName": "@test_time",
"title": "test search",
},
"metaFields": Array [],
"searchRequest": Object {
"body": Object {
"_source": Object {
"includes": Array [
"@test_time",
],
},
"docvalue_fields": undefined,
"query": Object {
"bool": Object {
"filter": Array [
Object {
"range": Object {
"@test_time": Object {
"format": "strict_date_time",
"gte": "1970-01-01T00:00:00Z",
"lte": "1970-01-11T10:00:00Z",
},
},
},
],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
},
"script_fields": Object {},
"sort": Array [],
},
"index": "test search",
},
}
`);
});
it('uses timerange min and max (string) when index pattern has timefieldName', async () => {
mockJobParams.post = {
timerange: {
timezone: 'Africa/Timbuktu',
min: '1980-01-01T00:00:00Z',
max: '1990-01-01T00:00:00Z',
},
};
mockSavedObjectsClient = {
get: () => ({
attributes: { fields: null, title: 'test search', timeFieldName: '@test_time' },
}),
};
const result = await getGenerateCsvParams(
mockJobParams,
mockSearchPanel,
mockSavedObjectsClient,
mockUiSettingsClient,
logger
);
expect(result).toMatchInlineSnapshot(`
Object {
"browserTimezone": "Africa/Timbuktu",
"conflictedTypesFields": Array [],
"fields": Array [
"@test_time",
],
"indexPatternSavedObject": Object {
"attributes": Object {
"fields": null,
"timeFieldName": "@test_time",
"title": "test search",
},
"fields": Array [],
"timeFieldName": "@test_time",
"title": "test search",
},
"metaFields": Array [],
"searchRequest": Object {
"body": Object {
"_source": Object {
"includes": Array [
"@test_time",
],
},
"docvalue_fields": undefined,
"query": Object {
"bool": Object {
"filter": Array [
Object {
"range": Object {
"@test_time": Object {
"format": "strict_date_time",
"gte": "1980-01-01T00:00:00Z",
"lte": "1990-01-01T00:00:00Z",
},
},
},
],
"must": Array [],
"must_not": Array [],
"should": Array [],
},
},
"script_fields": Object {},
"sort": Array [],
},
"index": "test search",
},
}
`);
});
});

155
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_csv_job.ts
View file

@ -1,155 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IUiSettingsClient, SavedObjectsClientContract } from 'kibana/server';
import { EsQueryConfig } from 'src/plugins/data/server';
import { esQuery, Filter, Query } from '../../../../../../../src/plugins/data/server';
import { LevelLogger } from '../../../lib';
import { TimeRangeParams } from '../../common';
import { GenerateCsvParams } from '../../csv/generate_csv';
import {
DocValueFields,
IndexPatternField,
JobParamsPanelCsv,
QueryFilter,
SavedSearchObjectAttributes,
SearchPanel,
SearchSource,
} from '../types';
import { getDataSource } from './get_data_source';
import { getFilters } from './get_filters';
export const getEsQueryConfig = async (config: IUiSettingsClient) => {
const configs = await Promise.all([
config.get('query:allowLeadingWildcards'),
config.get('query:queryString:options'),
config.get('courier:ignoreFilterIfFieldNotInIndex'),
]);
const [allowLeadingWildcards, queryStringOptions, ignoreFilterIfFieldNotInIndex] = configs;
return {
allowLeadingWildcards,
queryStringOptions,
ignoreFilterIfFieldNotInIndex,
} as EsQueryConfig;
};
/*
* Create a CSV Job object for CSV From SavedObject to use as a job parameter
* for generateCsv
*/
export const getGenerateCsvParams = async (
jobParams: JobParamsPanelCsv,
panel: SearchPanel,
savedObjectsClient: SavedObjectsClientContract,
uiConfig: IUiSettingsClient,
logger: LevelLogger
): Promise<GenerateCsvParams> => {
let timerange: TimeRangeParams | null;
if (jobParams.post?.timerange) {
timerange = jobParams.post?.timerange;
} else {
timerange = panel.timerange || null;
}
const { indexPatternSavedObjectId } = panel;
const savedSearchObjectAttr = panel.attributes as SavedSearchObjectAttributes;
const { indexPatternSavedObject } = await getDataSource(
savedObjectsClient,
indexPatternSavedObjectId
);
const esQueryConfig = await getEsQueryConfig(uiConfig);
const {
kibanaSavedObjectMeta: {
searchSource: {
filter: [searchSourceFilter],
query: searchSourceQuery,
},
},
} = savedSearchObjectAttr as { kibanaSavedObjectMeta: { searchSource: SearchSource } };
const {
timeFieldName: indexPatternTimeField,
title: esIndex,
fields: indexPatternFields,
} = indexPatternSavedObject;
if (!indexPatternFields || indexPatternFields.length === 0) {
logger.error(
new Error(
`No fields are selected in the saved search! Please select fields as columns in the saved search and try again.`
)
);
}
let payloadQuery: QueryFilter | undefined;
let payloadSort: any[] = [];
let docValueFields: DocValueFields[] | undefined;
if (jobParams.post && jobParams.post.state) {
({
post: {
state: { query: payloadQuery, sort: payloadSort = [], docvalue_fields: docValueFields },
},
} = jobParams);
}
const { includes, combinedFilter } = getFilters(
indexPatternSavedObjectId,
indexPatternTimeField,
timerange,
savedSearchObjectAttr,
searchSourceFilter,
payloadQuery
);
const savedSortConfigs = savedSearchObjectAttr.sort;
const sortConfig = [...payloadSort];
savedSortConfigs.forEach(([savedSortField, savedSortOrder]) => {
sortConfig.push({ [savedSortField]: { order: savedSortOrder } });
});
const scriptFieldsConfig =
indexPatternFields &&
indexPatternFields
.filter((f: IndexPatternField) => f.scripted)
.reduce((accum: any, curr: IndexPatternField) => {
return {
...accum,
[curr.name]: {
script: {
source: curr.script,
lang: curr.lang,
},
},
};
}, {});
const searchRequest = {
index: esIndex,
body: {
_source: { includes },
docvalue_fields: docValueFields,
query: esQuery.buildEsQuery(
// compromise made while factoring out IIndexPattern type
// @ts-expect-error
indexPatternSavedObject,
(searchSourceQuery as unknown) as Query,
(combinedFilter as unknown) as Filter,
esQueryConfig
),
script_fields: scriptFieldsConfig,
sort: sortConfig,
},
};
return {
browserTimezone: timerange?.timezone,
indexPatternSavedObject,
searchRequest,
fields: includes,
metaFields: [],
conflictedTypesFields: [],
};
};

56
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_data_source.ts
View file

@ -1,56 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndexPatternSavedObjectDeprecatedCSV } from '../../csv/types';
import { SavedObjectReference, SavedSearchObjectAttributesJSON, SearchSource } from '../types';
export async function getDataSource(
savedObjectsClient: any,
indexPatternId?: string,
savedSearchObjectId?: string
): Promise<{
indexPatternSavedObject: IndexPatternSavedObjectDeprecatedCSV;
searchSource: SearchSource | null;
}> {
let indexPatternSavedObject: IndexPatternSavedObjectDeprecatedCSV;
let searchSource: SearchSource | null = null;
if (savedSearchObjectId) {
try {
const { attributes, references } = (await savedObjectsClient.get(
'search',
savedSearchObjectId
)) as { attributes: SavedSearchObjectAttributesJSON; references: SavedObjectReference[] };
searchSource = JSON.parse(attributes.kibanaSavedObjectMeta.searchSourceJSON);
const { id: indexPatternFromSearchId } = references.find(
({ type }) => type === 'index-pattern'
) as { id: string };
({ indexPatternSavedObject } = await getDataSource(
savedObjectsClient,
indexPatternFromSearchId
));
return { searchSource, indexPatternSavedObject };
} catch (err) {
throw new Error(`Could not get saved search info! ${err}`);
}
}
try {
const { attributes } = await savedObjectsClient.get('index-pattern', indexPatternId);
const { fields, title, timeFieldName } = attributes;
const parsedFields = fields ? JSON.parse(fields) : [];
indexPatternSavedObject = {
fields: parsedFields,
title,
timeFieldName,
attributes,
};
} catch (err) {
throw new Error(`Could not get index pattern saved object! ${err}`);
}
return { indexPatternSavedObject, searchSource };
}

208
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.test.ts
View file

@ -1,208 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TimeRangeParams } from '../../common';
import { QueryFilter, SavedSearchObjectAttributes, SearchSourceFilter } from '../types';
import { getFilters } from './get_filters';
interface Args {
indexPatternId: string;
indexPatternTimeField: string | null;
timerange: TimeRangeParams | null;
savedSearchObjectAttr: SavedSearchObjectAttributes;
searchSourceFilter: SearchSourceFilter;
queryFilter: QueryFilter;
}
describe('CSV from Saved Object: get_filters', () => {
let args: Args;
beforeEach(() => {
args = {
indexPatternId: 'logs-test-*',
indexPatternTimeField: 'testtimestamp',
timerange: {
timezone: 'UTC',
min: '1901-01-01T00:00:00.000Z',
max: '1902-01-01T00:00:00.000Z',
},
savedSearchObjectAttr: {
title: 'test',
sort: [{ sortField: { order: 'asc' } }],
kibanaSavedObjectMeta: {
searchSource: {
query: { isSearchSourceQuery: true },
filter: ['hello searchSource filter 1'],
},
},
columns: ['larry'],
uiState: null,
},
searchSourceFilter: { isSearchSourceFilter: true, isFilter: true },
queryFilter: { isQueryFilter: true, isFilter: true },
};
});
describe('search', () => {
it('for timebased search', () => {
const filters = getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr,
args.searchSourceFilter,
args.queryFilter
);
expect(filters).toEqual({
combinedFilter: [
{
range: {
testtimestamp: {
format: 'strict_date_time',
gte: '1901-01-01T00:00:00Z',
lte: '1902-01-01T00:00:00Z',
},
},
},
{ isFilter: true, isSearchSourceFilter: true },
{ isFilter: true, isQueryFilter: true },
],
includes: ['testtimestamp', 'larry'],
timezone: 'UTC',
});
});
it('for non-timebased search', () => {
args.indexPatternTimeField = null;
args.timerange = null;
const filters = getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr,
args.searchSourceFilter,
args.queryFilter
);
expect(filters).toEqual({
combinedFilter: [
{ isFilter: true, isSearchSourceFilter: true },
{ isFilter: true, isQueryFilter: true },
],
includes: ['larry'],
timezone: null,
});
});
});
describe('errors', () => {
it('throw if timebased and timerange is missing', () => {
args.timerange = null;
const throwFn = () =>
getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr,
args.searchSourceFilter,
args.queryFilter
);
expect(throwFn).toThrow(
'Time range params are required for index pattern [logs-test-*], using time field [testtimestamp]'
);
});
});
it('composes the defined filters', () => {
expect(
getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr,
undefined,
undefined
)
).toEqual({
combinedFilter: [
{
range: {
testtimestamp: {
format: 'strict_date_time',
gte: '1901-01-01T00:00:00Z',
lte: '1902-01-01T00:00:00Z',
},
},
},
],
includes: ['testtimestamp', 'larry'],
timezone: 'UTC',
});
expect(
getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr,
undefined,
args.queryFilter
)
).toEqual({
combinedFilter: [
{
range: {
testtimestamp: {
format: 'strict_date_time',
gte: '1901-01-01T00:00:00Z',
lte: '1902-01-01T00:00:00Z',
},
},
},
{ isFilter: true, isQueryFilter: true },
],
includes: ['testtimestamp', 'larry'],
timezone: 'UTC',
});
});
describe('timefilter', () => {
it('formats the datetime to the provided timezone', () => {
args.timerange = {
timezone: 'MST',
min: '1901-01-01T00:00:00Z',
max: '1902-01-01T00:00:00Z',
};
expect(
getFilters(
args.indexPatternId,
args.indexPatternTimeField,
args.timerange,
args.savedSearchObjectAttr
)
).toEqual({
combinedFilter: [
{
range: {
testtimestamp: {
format: 'strict_date_time',
gte: '1900-12-31T17:00:00-07:00',
lte: '1901-12-31T17:00:00-07:00',
},
},
},
],
includes: ['testtimestamp', 'larry'],
timezone: 'MST',
});
});
});
});

55
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/lib/get_filters.ts
View file

@ -1,55 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { badRequest } from '@hapi/boom';
import moment from 'moment-timezone';
import { TimeRangeParams } from '../../common';
import { Filter, QueryFilter, SavedSearchObjectAttributes, SearchSourceFilter } from '../types';
export function getFilters(
indexPatternId: string,
indexPatternTimeField: string | null,
timerange: TimeRangeParams | null,
savedSearchObjectAttr: SavedSearchObjectAttributes,
searchSourceFilter?: SearchSourceFilter,
queryFilter?: QueryFilter
) {
let includes: string[];
let timeFilter: any | null;
let timezone: string | null;
if (indexPatternTimeField) {
if (!timerange || timerange.min == null || timerange.max == null) {
throw badRequest(
`Time range params are required for index pattern [${indexPatternId}], using time field [${indexPatternTimeField}]`
);
}
timezone = timerange.timezone;
const { min: gte, max: lte } = timerange;
timeFilter = {
range: {
[indexPatternTimeField]: {
format: 'strict_date_time',
gte: moment.tz(moment(gte), timezone).format(),
lte: moment.tz(moment(lte), timezone).format(),
},
},
};
const savedSearchCols = savedSearchObjectAttr.columns || [];
includes = [indexPatternTimeField, ...savedSearchCols];
} else {
includes = savedSearchObjectAttr.columns || [];
timeFilter = null;
timezone = null;
}
const combinedFilter: Filter[] = [timeFilter, searchSourceFilter, queryFilter].filter(Boolean); // builds an array of defined filters
return { timezone, combinedFilter, includes };
}

153
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/types.d.ts vendored
View file

@ -1,153 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TimeRangeParams } from '../common';
export interface FakeRequest {
headers: Record<string, string>;
}
export interface JobParamsPanelCsvPost {
timerange?: TimeRangeParams;
state?: any;
}
export interface SearchPanel {
indexPatternSavedObjectId: string;
attributes: SavedSearchObjectAttributes;
timerange?: TimeRangeParams;
}
export interface JobPayloadPanelCsv extends JobParamsPanelCsv {
panel: SearchPanel;
}
export interface JobParamsPanelCsv {
savedObjectType: string;
savedObjectId: string;
post?: JobParamsPanelCsvPost;
visType?: string;
}
export interface SavedObjectServiceError {
statusCode: number;
error?: string;
message?: string;
}
export interface SavedObjectMetaJSON {
searchSourceJSON: string;
}
export interface SavedObjectMeta {
searchSource: SearchSource;
}
export interface SavedSearchObjectAttributesJSON {
title: string;
sort: any[];
columns: string[];
kibanaSavedObjectMeta: SavedObjectMetaJSON;
uiState: any;
}
export interface SavedSearchObjectAttributes {
title: string;
sort: any[];
columns?: string[];
kibanaSavedObjectMeta: SavedObjectMeta;
uiState: any;
}
export interface VisObjectAttributesJSON {
title: string;
visState: string; // JSON string
type: string;
params: any;
uiStateJSON: string; // also JSON string
aggs: any[];
sort: any[];
kibanaSavedObjectMeta: SavedObjectMeta;
}
export interface VisObjectAttributes {
title: string;
visState: string; // JSON string
type: string;
params: any;
uiState: {
vis: {
params: {
sort: {
columnIndex: string;
direction: string;
};
};
};
};
aggs: any[];
sort: any[];
kibanaSavedObjectMeta: SavedObjectMeta;
}
export interface SavedObjectReference {
name: string; // should be kibanaSavedObjectMeta.searchSourceJSON.index
type: string; // should be index-pattern
id: string;
}
export interface SavedObject {
attributes: any;
references: SavedObjectReference[];
}
export interface VisPanel {
indexPatternSavedObjectId?: string;
savedSearchObjectId?: string;
attributes: VisObjectAttributes;
timerange: TimeRangeParams;
}
export interface DocValueFields {
field: string;
format: string;
}
export interface SearchSourceQuery {
isSearchSourceQuery: boolean;
}
export interface SearchSource {
query: SearchSourceQuery;
filter: any[];
}
/*
* These filter types are stub types to help ensure things get passed to
* non-Typescript functions in the right order. An actual structure is not
* needed because the code doesn't look into the properties; just combines them
* and passes them through to other non-TS modules.
*/
export interface Filter {
isFilter: boolean;
}
export interface TimeFilter extends Filter {
isTimeFilter: boolean;
}
export interface QueryFilter extends Filter {
isQueryFilter: boolean;
}
export interface SearchSourceFilter extends Filter {
isSearchSourceFilter: boolean;
}
export interface IndexPatternField {
scripted: boolean;
lang?: string;
script?: string;
name: string;
}

30
x-pack/plugins/reporting/server/export_types/csv_searchsource/create_job.ts Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CSV_JOB_TYPE } from '../../../common/constants';
import { cryptoFactory } from '../../lib';
import { CreateJobFn, CreateJobFnFactory } from '../../types';
import { JobParamsCSV, TaskPayloadCSV } from './types';
export const createJobFnFactory: CreateJobFnFactory<
CreateJobFn<JobParamsCSV, TaskPayloadCSV>
> = function createJobFactoryFn(reporting, parentLogger) {
const logger = parentLogger.clone([CSV_JOB_TYPE, 'create-job']);
const config = reporting.getConfig();
const crypto = cryptoFactory(config.get('encryptionKey'));
return async function createJob(jobParams, context, request) {
const serializedEncryptedHeaders = await crypto.encrypt(request.headers);
return {
headers: serializedEncryptedHeaders,
spaceId: reporting.getSpaceId(request, logger),
...jobParams,
};
};
};

75
x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.test.ts Normal file
View file

@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
jest.mock('./generate_csv/generate_csv', () => ({
CsvGenerator: class CsvGeneratorMock {
generateData() {
return {
content: 'test\n123',
};
}
},
}));
import nodeCrypto from '@elastic/node-crypto';
import { ReportingCore } from '../../';
import { CancellationToken } from '../../../common';
import {
createMockConfig,
createMockConfigSchema,
createMockLevelLogger,
createMockReportingCore,
} from '../../test_helpers';
import { runTaskFnFactory } from './execute_job';
const logger = createMockLevelLogger();
const encryptionKey = 'tetkey';
const headers = { sid: 'cooltestheaders' };
let encryptedHeaders: string;
let reportingCore: ReportingCore;
beforeAll(async () => {
const crypto = nodeCrypto({ encryptionKey });
const config = createMockConfig(
createMockConfigSchema({
encryptionKey,
csv: {
checkForFormulas: true,
escapeFormulaValues: true,
maxSizeBytes: 180000,
scroll: { size: 500, duration: '30s' },
},
})
);
encryptedHeaders = await crypto.encrypt(headers);
reportingCore = await createMockReportingCore(config);
});
test('gets the csv content from job parameters', async () => {
const runTask = runTaskFnFactory(reportingCore, logger);
const payload = await runTask(
'cool-job-id',
{
headers: encryptedHeaders,
browserTimezone: 'US/Alaska',
searchSource: {},
objectType: 'search',
title: 'Test Search',
},
new CancellationToken()
);
expect(payload).toMatchInlineSnapshot(`
Object {
"content": "test
123",
}
`);
});

49
x-pack/plugins/reporting/server/export_types/csv_searchsource/execute_job.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CSV_JOB_TYPE } from '../../../common/constants';
import { getFieldFormats } from '../../services';
import { RunTaskFn, RunTaskFnFactory } from '../../types';
import { decryptJobHeaders } from '../common';
import { CsvGenerator } from './generate_csv/generate_csv';
import { TaskPayloadCSV } from './types';
export const runTaskFnFactory: RunTaskFnFactory<RunTaskFn<TaskPayloadCSV>> = (
reporting,
parentLogger
) => {
const config = reporting.getConfig();
return async function runTask(jobId, job, cancellationToken) {
const logger = parentLogger.clone([CSV_JOB_TYPE, 'execute-job', jobId]);
const encryptionKey = config.get('encryptionKey');
const headers = await decryptJobHeaders(encryptionKey, job.headers, logger);
const fakeRequest = reporting.getFakeRequest({ headers }, job.spaceId, logger);
const uiSettings = await reporting.getUiSettingsClient(fakeRequest, logger);
const dataPluginStart = await reporting.getDataService();
const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(uiSettings);
const [es, searchSourceStart] = await Promise.all([
(await reporting.getEsClient()).asScoped(fakeRequest),
await dataPluginStart.search.searchSource.asScoped(fakeRequest),
]);
const clients = {
uiSettings,
data: dataPluginStart.search.asScoped(fakeRequest),
es,
};
const dependencies = {
searchSourceStart,
fieldFormatsRegistry,
};
const csv = new CsvGenerator(job, config, clients, dependencies, cancellationToken, logger);
return await csv.generateData();
};
};

163
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/__snapshots__/generate_csv.test.ts.snap generated Normal file
View file

@ -0,0 +1,163 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`fields cells can be multi-value 1`] = `
"\\"_id\\",sku
\\"my-cool-id\\",\\"This is a cool SKU., This is also a cool SKU.\\"
"
`;
exports[`fields provides top-level underscored fields as columns 1`] = `
"\\"_id\\",\\"_index\\",date,message
\\"my-cool-id\\",\\"my-cool-index\\",\\"2020-12-31T00:14:28.000Z\\",\\"it's nice to see you\\"
"
`;
exports[`fields sorts the fields when they are to be used as table column names 1`] = `
"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",date,\\"message_t\\",\\"message_u\\",\\"message_v\\",\\"message_w\\",\\"message_x\\",\\"message_y\\",\\"message_z\\"
\\"my-cool-id\\",\\"my-cool-index\\",\\"'-\\",\\"'-\\",\\"2020-12-31T00:14:28.000Z\\",\\"test field T\\",\\"test field U\\",\\"test field V\\",\\"test field W\\",\\"test field X\\",\\"test field Y\\",\\"test field Z\\"
"
`;
exports[`formats a search result to CSV content 1`] = `
"date,ip,message
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is a great message!\\"
"
`;
exports[`formats an empty search result to CSV content 1`] = `
"date,ip,message
"
`;
exports[`formulas can check for formulas, without escaping them 1`] = `
"date,ip,message
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"=SUM(A1:A2)\\"
"
`;
exports[`formulas escapes formula values in a cell, doesn't warn the csv contains formulas 1`] = `
"date,ip,message
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"'=SUM(A1:A2)\\"
"
`;
exports[`formulas escapes formula values in a header, doesn't warn the csv contains formulas 1`] = `
"date,ip,\\"'=SUM(A1:A2)\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"This is great data\\"
"
`;
exports[`uses the scrollId to page all the data 1`] = `
"date,ip,message
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from the initial search\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"hit from a subsequent scroll\\"
"
`;
exports[`warns if max size was reached 1`] = `
"date,ip,message
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
\\"2020-12-31T00:14:28.000Z\\",\\"110.135.176.89\\",\\"super cali fragile istic XPLA docious\\"
"
`;

0
x-pack/plugins/reporting/server/export_types/csv/generate_csv/cell_has_formula.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/cell_has_formula.ts
View file

0
x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.test.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.test.ts
View file

0
x-pack/plugins/reporting/server/export_types/csv/generate_csv/escape_value.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/escape_value.ts
View file

645
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts Normal file
View file

@ -0,0 +1,645 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import * as Rx from 'rxjs';
import { identity, range } from 'lodash';
import { IScopedClusterClient, IUiSettingsClient, SearchResponse } from 'src/core/server';
import {
elasticsearchServiceMock,
savedObjectsClientMock,
uiSettingsServiceMock,
} from 'src/core/server/mocks';
import { FieldFormatsRegistry, ISearchStartSearchSource } from 'src/plugins/data/common';
import { searchSourceInstanceMock } from 'src/plugins/data/common/search/search_source/mocks';
import { IScopedSearchClient } from 'src/plugins/data/server';
import { dataPluginMock } from 'src/plugins/data/server/mocks';
import { ReportingConfig } from '../../../';
import { CancellationToken } from '../../../../common';
import {
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
UI_SETTINGS_DATEFORMAT_TZ,
} from '../../../../common/constants';
import {
createMockConfig,
createMockConfigSchema,
createMockLevelLogger,
} from '../../../test_helpers';
import { JobParamsCSV } from '../types';
import { CsvGenerator } from './generate_csv';
const createMockJob = (baseObj: any = {}): JobParamsCSV => ({
...baseObj,
});
let mockEsClient: IScopedClusterClient;
let mockDataClient: IScopedSearchClient;
let mockConfig: ReportingConfig;
let uiSettingsClient: IUiSettingsClient;
const searchSourceMock = { ...searchSourceInstanceMock };
const mockSearchSourceService: jest.Mocked<ISearchStartSearchSource> = {
create: jest.fn().mockReturnValue(searchSourceMock),
createEmpty: jest.fn().mockReturnValue(searchSourceMock),
};
const mockDataClientSearchDefault = jest.fn().mockImplementation(
(): Rx.Observable<{ rawResponse: SearchResponse<unknown> }> =>
Rx.of({
rawResponse: {
took: 1,
timed_out: false,
_shards: { total: 1, successful: 1, failed: 0, skipped: 0 },
hits: {
hits: [],
total: 0,
max_score: 0,
},
},
})
);
const mockSearchSourceGetFieldDefault = jest.fn().mockImplementation((key: string) => {
switch (key) {
case 'fields':
return ['date', 'ip', 'message'];
case 'index':
return {
fields: {
getByName: jest.fn().mockImplementation(() => []),
getByType: jest.fn().mockImplementation(() => []),
},
getFormatterForField: jest.fn(),
};
}
});
const mockFieldFormatsRegistry = ({
deserialize: jest
.fn()
.mockImplementation(() => ({ id: 'string', convert: jest.fn().mockImplementation(identity) })),
} as unknown) as FieldFormatsRegistry;
beforeEach(async () => {
mockEsClient = elasticsearchServiceMock.createScopedClusterClient();
mockDataClient = dataPluginMock.createStartContract().search.asScoped({} as any);
mockDataClient.search = mockDataClientSearchDefault;
uiSettingsClient = uiSettingsServiceMock
.createStartContract()
.asScopedToClient(savedObjectsClientMock.create());
uiSettingsClient.get = jest.fn().mockImplementation((key): any => {
switch (key) {
case UI_SETTINGS_CSV_QUOTE_VALUES:
return true;
case UI_SETTINGS_CSV_SEPARATOR:
return ',';
case UI_SETTINGS_DATEFORMAT_TZ:
return 'Browser';
}
});
mockConfig = createMockConfig(
createMockConfigSchema({
csv: {
checkForFormulas: true,
escapeFormulaValues: true,
maxSizeBytes: 180000,
scroll: { size: 500, duration: '30s' },
},
})
);
searchSourceMock.getField = mockSearchSourceGetFieldDefault;
});
const logger = createMockLevelLogger();
it('formats an empty search result to CSV content', async () => {
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
it('formats a search result to CSV content', async () => {
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
fields: {
date: `["2020-12-31T00:14:28.000Z"]`,
ip: `["110.135.176.89"]`,
message: `["This is a great message!"]`,
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
const HITS_TOTAL = 100;
it('calculates the bytes of the content', async () => {
searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
if (key === 'fields') {
return ['message'];
}
return mockSearchSourceGetFieldDefault(key);
});
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
fields: {
message: ['this is a great message'],
},
})),
total: HITS_TOTAL,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.size).toBe(2608);
expect(csvResult.max_size_reached).toBe(false);
expect(csvResult.warnings).toEqual([]);
});
it('warns if max size was reached', async () => {
const TEST_MAX_SIZE = 500;
mockConfig = createMockConfig(
createMockConfigSchema({
csv: {
checkForFormulas: true,
escapeFormulaValues: true,
maxSizeBytes: TEST_MAX_SIZE,
scroll: { size: 500, duration: '30s' },
},
})
);
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
message: ['super cali fragile istic XPLA docious'],
},
})),
total: HITS_TOTAL,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.max_size_reached).toBe(true);
expect(csvResult.warnings).toEqual([]);
expect(csvResult.content).toMatchSnapshot();
});
it('uses the scrollId to page all the data', async () => {
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
_scroll_id: 'awesome-scroll-hero',
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
message: ['hit from the initial search'],
},
})),
total: HITS_TOTAL,
},
},
})
);
mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
body: {
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
message: ['hit from a subsequent scroll'],
},
})),
},
},
});
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.warnings).toEqual([]);
expect(csvResult.content).toMatchSnapshot();
});
describe('fields', () => {
it('cells can be multi-value', async () => {
searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
if (key === 'fields') {
return ['_id', 'sku'];
}
return mockSearchSourceGetFieldDefault(key);
});
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
_id: 'my-cool-id',
_index: 'my-cool-index',
_version: 4,
fields: {
sku: [`This is a cool SKU.`, `This is also a cool SKU.`],
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({ searchSource: {} }),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
});
it('provides top-level underscored fields as columns', async () => {
searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
if (key === 'fields') {
return ['_id', '_index', 'date', 'message'];
}
return mockSearchSourceGetFieldDefault(key);
});
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
_id: 'my-cool-id',
_index: 'my-cool-index',
_version: 4,
fields: {
date: ['2020-12-31T00:14:28.000Z'],
message: [`it's nice to see you`],
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({
searchSource: {
query: { query: '', language: 'kuery' },
sort: [{ '@date': 'desc' }],
index: '93f4bc50-6662-11eb-98bc-f550e2308366',
fields: ['_id', '_index', '@date', 'message'],
filter: [],
},
}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
it('sorts the fields when they are to be used as table column names', async () => {
searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
if (key === 'fields') {
return ['*'];
}
return mockSearchSourceGetFieldDefault(key);
});
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
_id: 'my-cool-id',
_index: 'my-cool-index',
_version: 4,
fields: {
date: ['2020-12-31T00:14:28.000Z'],
message_z: [`test field Z`],
message_y: [`test field Y`],
message_x: [`test field X`],
message_w: [`test field W`],
message_v: [`test field V`],
message_u: [`test field U`],
message_t: [`test field T`],
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({
searchSource: {
query: { query: '', language: 'kuery' },
sort: [{ '@date': 'desc' }],
index: '93f4bc50-6662-11eb-98bc-f550e2308366',
fields: ['*'],
filter: [],
},
}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
});
describe('formulas', () => {
const TEST_FORMULA = '=SUM(A1:A2)';
it(`escapes formula values in a cell, doesn't warn the csv contains formulas`, async () => {
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
message: [TEST_FORMULA],
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
it(`escapes formula values in a header, doesn't warn the csv contains formulas`, async () => {
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
[TEST_FORMULA]: 'This is great data',
},
},
],
total: 1,
},
},
})
);
searchSourceMock.getField = jest.fn().mockImplementation((key: string) => {
if (key === 'fields') {
return ['date', 'ip', TEST_FORMULA];
}
return mockSearchSourceGetFieldDefault(key);
});
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(false);
});
it('can check for formulas, without escaping them', async () => {
mockConfig = createMockConfig(
createMockConfigSchema({
csv: {
checkForFormulas: true,
escapeFormulaValues: false,
maxSizeBytes: 180000,
scroll: { size: 500, duration: '30s' },
},
})
);
mockDataClient.search = jest.fn().mockImplementation(() =>
Rx.of({
rawResponse: {
hits: {
hits: [
{
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
message: [TEST_FORMULA],
},
},
],
total: 1,
},
},
})
);
const generateCsv = new CsvGenerator(
createMockJob({}),
mockConfig,
{
es: mockEsClient,
data: mockDataClient,
uiSettings: uiSettingsClient,
},
{
searchSourceStart: mockSearchSourceService,
fieldFormatsRegistry: mockFieldFormatsRegistry,
},
new CancellationToken(),
logger
);
const csvResult = await generateCsv.generateData();
expect(csvResult.content).toMatchSnapshot();
expect(csvResult.csv_contains_formulas).toBe(true);
});
});

400
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts Normal file
View file

@ -0,0 +1,400 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { SearchResponse } from 'elasticsearch';
import { IScopedClusterClient, IUiSettingsClient } from 'src/core/server';
import { IScopedSearchClient } from 'src/plugins/data/server';
import { Datatable } from 'src/plugins/expressions/server';
import { ReportingConfig } from '../../..';
import {
ES_SEARCH_STRATEGY,
FieldFormat,
FieldFormatConfig,
IFieldFormatsRegistry,
IndexPattern,
ISearchSource,
ISearchStartSearchSource,
SearchFieldValue,
tabifyDocs,
} from '../../../../../../../src/plugins/data/common';
import { KbnServerError } from '../../../../../../../src/plugins/kibana_utils/server';
import { CancellationToken } from '../../../../common';
import { CONTENT_TYPE_CSV } from '../../../../common/constants';
import { byteSizeValueToNumber } from '../../../../common/schema_utils';
import { LevelLogger } from '../../../lib';
import { TaskRunResult } from '../../../lib/tasks';
import { JobParamsCSV } from '../types';
import { cellHasFormulas } from './cell_has_formula';
import { CsvExportSettings, getExportSettings } from './get_export_settings';
import { MaxSizeStringBuilder } from './max_size_string_builder';
interface Clients {
es: IScopedClusterClient;
data: IScopedSearchClient;
uiSettings: IUiSettingsClient;
}
interface Dependencies {
searchSourceStart: ISearchStartSearchSource;
fieldFormatsRegistry: IFieldFormatsRegistry;
}
// Function to check if the field name values can be used as the header row
function isPlainStringArray(
fields: SearchFieldValue[] | string | boolean | undefined
): fields is string[] {
let result = true;
if (Array.isArray(fields)) {
fields.forEach((field) => {
if (typeof field !== 'string' || field === '*' || field === '_source') {
result = false;
}
});
}
return result;
}
export class CsvGenerator {
private _formatters: Record<string, FieldFormat> | null = null;
private csvContainsFormulas = false;
private maxSizeReached = false;
private csvRowCount = 0;
constructor(
private job: JobParamsCSV,
private config: ReportingConfig,
private clients: Clients,
private dependencies: Dependencies,
private cancellationToken: CancellationToken,
private logger: LevelLogger
) {}
private async scan(
index: IndexPattern,
searchSource: ISearchSource,
scrollSettings: CsvExportSettings['scroll']
) {
const searchBody = await searchSource.getSearchRequestBody();
this.logger.debug(`executing search request`);
const searchParams = {
params: {
body: searchBody,
index: index.title,
scroll: scrollSettings.duration,
size: scrollSettings.size,
},
};
const results = (
await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
).rawResponse;
return results;
}
private async scroll(scrollId: string, scrollSettings: CsvExportSettings['scroll']) {
this.logger.debug(`executing scroll request`);
const results = (
await this.clients.es.asCurrentUser.scroll({
scroll: scrollSettings.duration,
scroll_id: scrollId,
})
).body as SearchResponse<unknown>;
return results;
}
/*
* Load field formats for each field in the list
*/
private getFormatters(table: Datatable) {
if (this._formatters) {
return this._formatters;
}
// initialize field formats
const formatters: Record<string, FieldFormat> = {};
table.columns.forEach((c) => {
const fieldFormat = this.dependencies.fieldFormatsRegistry.deserialize(c.meta.params);
formatters[c.id] = fieldFormat;
});
this._formatters = formatters;
return this._formatters;
}
private escapeValues(settings: CsvExportSettings) {
return (value: string) => {
if (settings.checkForFormulas && cellHasFormulas(value)) {
this.csvContainsFormulas = true; // set warning if cell value has a formula
}
return settings.escapeValue(value);
};
}
// use fields/fieldsFromSource from the searchSource to get the ordering of columns
// otherwise use the table columns as they are
private getFields(searchSource: ISearchSource, table: Datatable): string[] {
const fieldValues: Record<string, string | boolean | SearchFieldValue[] | undefined> = {
fields: searchSource.getField('fields'),
fieldsFromSource: searchSource.getField('fieldsFromSource'),
};
const fieldSource = fieldValues.fieldsFromSource ? 'fieldsFromSource' : 'fields';
this.logger.debug(`Getting search source fields from: '${fieldSource}'`);
const fields = fieldValues[fieldSource];
// Check if field name values are string[] and if the fields are user-defined
if (isPlainStringArray(fields)) {
return fields;
}
// Default to using the table column IDs as the fields
const columnIds = table.columns.map((c) => c.id);
// Fields in the API response don't come sorted - they need to be sorted client-side
columnIds.sort();
return columnIds;
}
private formatCellValues(formatters: Record<string, FieldFormat>) {
return ({
column: tableColumn,
data: dataTableCell,
}: {
column: string;
data: any;
}): string => {
let cell: string[] | string | object;
// check truthiness to guard against _score, _type, etc
if (tableColumn && dataTableCell) {
try {
cell = formatters[tableColumn].convert(dataTableCell);
} catch (err) {
this.logger.error(err);
cell = '-';
}
try {
// expected values are a string of JSON where the value(s) is in an array
cell = JSON.parse(cell);
} catch (e) {
// ignore
}
// We have to strip singular array values out of their array wrapper,
// So that the value appears the visually the same as seen in Discover
if (Array.isArray(cell)) {
cell = cell.map((c) => (typeof c === 'object' ? JSON.stringify(c) : c)).join(', ');
}
// Check for object-type value (geoip)
if (typeof cell === 'object') {
cell = JSON.stringify(cell);
}
return cell;
}
return '-'; // Unknown field: it existed in searchSource but has no value in the result
};
}
/*
* Use the list of fields to generate the header row
*/
private generateHeader(
fields: string[],
table: Datatable,
builder: MaxSizeStringBuilder,
settings: CsvExportSettings
) {
this.logger.debug(`Building CSV header row...`);
const header = fields.map(this.escapeValues(settings)).join(settings.separator) + '\n';
if (!builder.tryAppend(header)) {
return {
size: 0,
content: '',
maxSizeReached: true,
warnings: [],
};
}
}
/*
* Format a Datatable into rows of CSV content
*/
private generateRows(
fields: string[],
table: Datatable,
builder: MaxSizeStringBuilder,
formatters: Record<string, FieldFormat>,
settings: CsvExportSettings
) {
this.logger.debug(`Building ${table.rows.length} CSV data rows...`);
for (const dataTableRow of table.rows) {
if (this.cancellationToken.isCancelled()) {
break;
}
const row =
fields
.map((f) => ({ column: f, data: dataTableRow[f] }))
.map(this.formatCellValues(formatters))
.map(this.escapeValues(settings))
.join(settings.separator) + '\n';
if (!builder.tryAppend(row)) {
this.logger.warn(`Max Size Reached after ${this.csvRowCount} rows.`);
this.maxSizeReached = true;
if (this.cancellationToken) {
this.cancellationToken.cancel();
}
break;
}
this.csvRowCount++;
}
}
public async generateData(): Promise<TaskRunResult> {
const [settings, searchSource] = await Promise.all([
getExportSettings(
this.clients.uiSettings,
this.config,
this.job.browserTimezone,
this.logger
),
this.dependencies.searchSourceStart.create(this.job.searchSource),
]);
const index = searchSource.getField('index');
if (!index) {
throw new Error(`The search must have a revference to an index pattern!`);
}
const { maxSizeBytes, bom, escapeFormulaValues, scroll: scrollSettings } = settings;
const builder = new MaxSizeStringBuilder(byteSizeValueToNumber(maxSizeBytes), bom);
const warnings: string[] = [];
let first = true;
let currentRecord = -1;
let totalRecords = 0;
let scrollId: string | undefined;
// apply timezone from the job to all date field formatters
try {
index.fields.getByType('date').forEach(({ name }) => {
this.logger.debug(`setting timezone on ${name}`);
const format: FieldFormatConfig = {
...index.fieldFormatMap[name],
id: index.fieldFormatMap[name]?.id || 'date', // allow id: date_nanos
params: {
...index.fieldFormatMap[name]?.params,
timezone: settings.timezone,
},
};
index.setFieldFormat(name, format);
});
} catch (err) {
this.logger.error(err);
}
try {
do {
if (this.cancellationToken.isCancelled()) {
break;
}
let results: SearchResponse<unknown> | undefined;
if (scrollId == null) {
// open a scroll cursor in Elasticsearch
results = await this.scan(index, searchSource, scrollSettings);
scrollId = results?._scroll_id;
if (results.hits?.total != null) {
totalRecords = results.hits.total;
this.logger.debug(`Total search results: ${totalRecords}`);
}
} else {
// use the scroll cursor in Elasticsearch
results = await this.scroll(scrollId, scrollSettings);
}
if (!results) {
this.logger.warning(`Search results are undefined!`);
break;
}
let table: Datatable | undefined;
try {
table = tabifyDocs(results, index, { shallow: true, meta: true });
} catch (err) {
this.logger.error(err);
}
if (!table) {
break;
}
const fields = this.getFields(searchSource, table);
if (first) {
first = false;
this.generateHeader(fields, table, builder, settings);
}
if (table.rows.length < 1) {
break; // empty report with just the header
}
const formatters = this.getFormatters(table);
this.generateRows(fields, table, builder, formatters, settings);
// update iterator
currentRecord += table.rows.length;
} while (currentRecord < totalRecords - 1);
// Add warnings to be logged
if (this.csvContainsFormulas && escapeFormulaValues) {
warnings.push(
i18n.translate('xpack.reporting.exportTypes.csv.generateCsv.escapedFormulaValues', {
defaultMessage: 'CSV may contain formulas whose values have been escaped',
})
);
}
} catch (err) {
this.logger.error(err);
if (err instanceof KbnServerError && err.errBody) {
throw JSON.stringify(err.errBody.error);
}
} finally {
// clear scrollID
if (scrollId) {
this.logger.debug(`executing clearScroll request`);
try {
await this.clients.es.asCurrentUser.clearScroll({ scroll_id: [scrollId] });
} catch (err) {
this.logger.error(err);
}
} else {
this.logger.warn(`No scrollId to clear!`);
}
}
const size = builder.getSizeInBytes();
this.logger.debug(
`Finished generating. Total size in bytes: ${size}. Row count: ${this.csvRowCount}.`
);
return {
content: builder.getString(),
content_type: CONTENT_TYPE_CSV,
csv_contains_formulas: this.csvContainsFormulas && !escapeFormulaValues,
max_size_reached: this.maxSizeReached,
size,
warnings,
};
}
}

83
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
} from '../../../../common/constants';
import { IUiSettingsClient } from 'kibana/server';
import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
import {
createMockConfig,
createMockConfigSchema,
createMockLevelLogger,
} from '../../../test_helpers';
import { getExportSettings } from './get_export_settings';
describe('getExportSettings', () => {
let uiSettingsClient: IUiSettingsClient;
const config = createMockConfig(createMockConfigSchema({}));
const logger = createMockLevelLogger();
beforeEach(() => {
uiSettingsClient = uiSettingsServiceMock
.createStartContract()
.asScopedToClient(savedObjectsClientMock.create());
uiSettingsClient.get = jest.fn().mockImplementation((key: string) => {
switch (key) {
case UI_SETTINGS_CSV_QUOTE_VALUES:
return true;
case UI_SETTINGS_CSV_SEPARATOR:
return ',';
case UI_SETTINGS_DATEFORMAT_TZ:
return 'Browser';
}
return 'helo world';
});
});
test('getExportSettings: returns the expected result', async () => {
expect(await getExportSettings(uiSettingsClient, config, '', logger)).toMatchInlineSnapshot(`
Object {
"bom": "",
"checkForFormulas": undefined,
"escapeFormulaValues": undefined,
"escapeValue": [Function],
"maxSizeBytes": undefined,
"scroll": Object {
"duration": undefined,
"size": undefined,
},
"separator": ",",
"timezone": "UTC",
}
`);
});
test('escapeValue function', async () => {
const { escapeValue } = await getExportSettings(uiSettingsClient, config, '', logger);
expect(escapeValue(`test`)).toBe(`test`);
expect(escapeValue(`this is, a test`)).toBe(`"this is, a test"`);
expect(escapeValue(`"tet"`)).toBe(`"""tet"""`);
expect(escapeValue(`@foo`)).toBe(`"@foo"`);
});
test('non-default timezone', async () => {
uiSettingsClient.get = jest.fn().mockImplementation((key: string) => {
switch (key) {
case UI_SETTINGS_DATEFORMAT_TZ:
return `America/Aruba`;
}
});
expect(
await getExportSettings(uiSettingsClient, config, '', logger).then(({ timezone }) => timezone)
).toBe(`America/Aruba`);
});
});

85
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ByteSizeValue } from '@kbn/config-schema';
import { i18n } from '@kbn/i18n';
import { IUiSettingsClient } from 'kibana/server';
import { ReportingConfig } from '../../../';
import {
CSV_BOM_CHARS,
UI_SETTINGS_DATEFORMAT_TZ,
UI_SETTINGS_CSV_QUOTE_VALUES,
UI_SETTINGS_CSV_SEPARATOR,
} from '../../../../common/constants';
import { LevelLogger } from '../../../lib';
import { createEscapeValue } from './escape_value';
export interface CsvExportSettings {
timezone: string;
scroll: {
size: number;
duration: string;
};
bom: string;
separator: string;
maxSizeBytes: number | ByteSizeValue;
checkForFormulas: boolean;
escapeFormulaValues: boolean;
escapeValue: (value: string) => string;
}
export const getExportSettings = async (
client: IUiSettingsClient,
config: ReportingConfig,
timezone: string | undefined,
logger: LevelLogger
): Promise<CsvExportSettings> => {
// Timezone
let setTimezone: string;
// timezone in job params?
if (timezone) {
setTimezone = timezone;
} else {
// timezone in settings?
setTimezone = await client.get(UI_SETTINGS_DATEFORMAT_TZ);
if (setTimezone === 'Browser') {
// if `Browser`, hardcode it to 'UTC' so the export has data that makes sense
logger.warn(
i18n.translate('xpack.reporting.exportTypes.csv.executeJob.dateFormateSetting', {
defaultMessage:
'Kibana Advanced Setting "{dateFormatTimezone}" is set to "Browser". Dates will be formatted as UTC to avoid ambiguity.',
values: { dateFormatTimezone: 'dateFormat:tz' },
})
);
setTimezone = 'UTC';
}
}
// Separator, QuoteValues
const [separator, quoteValues] = await Promise.all([
client.get(UI_SETTINGS_CSV_SEPARATOR),
client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
]);
const escapeFormulaValues = config.get('csv', 'escapeFormulaValues');
const escapeValue = createEscapeValue(quoteValues, escapeFormulaValues);
const bom = config.get('csv', 'useByteOrderMarkEncoding') ? CSV_BOM_CHARS : '';
return {
timezone: setTimezone,
scroll: {
size: config.get('csv', 'scroll', 'size'),
duration: config.get('csv', 'scroll', 'duration'),
},
bom,
separator,
maxSizeBytes: config.get('csv', 'maxSizeBytes'),
checkForFormulas: config.get('csv', 'checkForFormulas'),
escapeFormulaValues,
escapeValue,
};
};

8
x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { CsvGenerator } from './generate_csv';

0
x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.test.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.test.ts
View file

0
x-pack/plugins/reporting/server/export_types/csv/generate_csv/max_size_string_builder.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/max_size_string_builder.ts
View file

40
x-pack/plugins/reporting/server/export_types/csv_searchsource/index.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
CSV_JOB_TYPE as jobType,
LICENSE_TYPE_BASIC,
LICENSE_TYPE_ENTERPRISE,
LICENSE_TYPE_GOLD,
LICENSE_TYPE_PLATINUM,
LICENSE_TYPE_STANDARD,
LICENSE_TYPE_TRIAL,
} from '../../../common/constants';
import { CreateJobFn, ExportTypeDefinition, RunTaskFn } from '../../types';
import { createJobFnFactory } from './create_job';
import { runTaskFnFactory } from './execute_job';
import { metadata } from './metadata';
import { JobParamsCSV, TaskPayloadCSV } from './types';
export const getExportType = (): ExportTypeDefinition<
CreateJobFn<JobParamsCSV>,
RunTaskFn<TaskPayloadCSV>
> => ({
...metadata,
jobType,
jobContentExtension: 'csv',
createJobFnFactory,
runTaskFnFactory,
validLicenses: [
LICENSE_TYPE_TRIAL,
LICENSE_TYPE_BASIC,
LICENSE_TYPE_STANDARD,
LICENSE_TYPE_GOLD,
LICENSE_TYPE_PLATINUM,
LICENSE_TYPE_ENTERPRISE,
],
});

6
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/metadata.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource/metadata.ts
View file

@ -5,9 +5,9 @@
* 2.0.
*/
import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants';
import { CSV_JOB_TYPE } from '../../../common/constants';
export const metadata = {
id: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
name: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
id: 'csv_searchsource',
name: CSV_JOB_TYPE,
};

18
x-pack/plugins/reporting/server/export_types/csv_searchsource/types.d.ts vendored Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { BaseParams, BasePayload } from '../../types';
export type RawValue = string | object | null | undefined;
interface BaseParamsCSV {
browserTimezone: string;
searchSource: any;
}
export type JobParamsCSV = BaseParamsCSV & BaseParams;
export type TaskPayloadCSV = BaseParamsCSV & BasePayload;

81
x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/execute_job.ts Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { KibanaRequest } from 'src/core/server';
import { CancellationToken } from '../../../common';
import { CSV_SEARCHSOURCE_IMMEDIATE_TYPE } from '../../../common/constants';
import { TaskRunResult } from '../../lib/tasks';
import { getFieldFormats } from '../../services';
import { ReportingRequestHandlerContext, RunTaskFnFactory } from '../../types';
import { CsvGenerator } from '../csv_searchsource/generate_csv/generate_csv';
import { JobParamsDownloadCSV } from './types';
/*
* ImmediateExecuteFn receives the job doc payload because the payload was
* generated in the ScheduleFn
*/
export type ImmediateExecuteFn = (
jobId: null,
job: JobParamsDownloadCSV,
context: ReportingRequestHandlerContext,
req: KibanaRequest
) => Promise<TaskRunResult>;
export const runTaskFnFactory: RunTaskFnFactory<ImmediateExecuteFn> = function executeJobFactoryFn(
reporting,
parentLogger
) {
const config = reporting.getConfig();
const logger = parentLogger.clone([CSV_SEARCHSOURCE_IMMEDIATE_TYPE, 'execute-job']);
return async function runTask(jobId, immediateJobParams, context, req) {
const job = {
objectType: 'immediate-search',
...immediateJobParams,
};
const savedObjectsClient = context.core.savedObjects.client;
const uiSettings = await reporting.getUiSettingsServiceFactory(savedObjectsClient);
const dataPluginStart = await reporting.getDataService();
const fieldFormatsRegistry = await getFieldFormats().fieldFormatServiceFactory(uiSettings);
const [es, searchSourceStart] = await Promise.all([
(await reporting.getEsClient()).asScoped(req),
await dataPluginStart.search.searchSource.asScoped(req),
]);
const clients = {
uiSettings,
data: dataPluginStart.search.asScoped(req),
es,
};
const dependencies = {
fieldFormatsRegistry,
searchSourceStart,
};
const cancellationToken = new CancellationToken();
const csv = new CsvGenerator(job, config, clients, dependencies, cancellationToken, logger);
const result = await csv.generateData();
if (result.csv_contains_formulas) {
logger.warn(`CSV may contain formulas whose values have been escaped`);
}
if (result.max_size_reached) {
logger.warn(`Max size reached: CSV output truncated to ${result.size} bytes`);
}
const { warnings } = result;
if (warnings) {
warnings.forEach((warning) => {
logger.warning(warning);
});
}
return result;
};
};

13
x-pack/plugins/reporting/server/export_types/csv_from_savedobject/index.ts → x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/index.ts
View file

@ -6,7 +6,7 @@
*/
import {
CSV_FROM_SAVEDOBJECT_JOB_TYPE,
CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
LICENSE_TYPE_BASIC,
LICENSE_TYPE_ENTERPRISE,
LICENSE_TYPE_GOLD,
@ -15,7 +15,6 @@ import {
LICENSE_TYPE_TRIAL,
} from '../../../common/constants';
import { ExportTypeDefinition } from '../../types';
import { createJobFnFactory, ImmediateCreateJobFn } from './create_job';
import { ImmediateExecuteFn, runTaskFnFactory } from './execute_job';
import { metadata } from './metadata';
@ -23,17 +22,13 @@ import { metadata } from './metadata';
* These functions are exported to share with the API route handler that
* generates csv from saved object immediately on request.
*/
export { createJobFnFactory } from './create_job';
export { runTaskFnFactory } from './execute_job';
export const getExportType = (): ExportTypeDefinition<
ImmediateCreateJobFn,
ImmediateExecuteFn
> => ({
export const getExportType = (): ExportTypeDefinition<null, ImmediateExecuteFn> => ({
...metadata,
jobType: CSV_FROM_SAVEDOBJECT_JOB_TYPE,
jobType: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
jobContentExtension: 'csv',
createJobFnFactory,
createJobFnFactory: null,
runTaskFnFactory,
validLicenses: [
LICENSE_TYPE_TRIAL,

13
x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/metadata.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CSV_SEARCHSOURCE_IMMEDIATE_TYPE } from '../../../common/constants';
export const metadata = {
id: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
name: CSV_SEARCHSOURCE_IMMEDIATE_TYPE,
};

24
x-pack/plugins/reporting/server/export_types/csv_searchsource_immediate/types.d.ts vendored Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TimeRangeParams } from '../common';
export interface FakeRequest {
headers: Record<string, string>;
}
export interface JobParamsDownloadCSV {
browserTimezone: string;
title: string;
searchSource: any;
}
export interface SavedObjectServiceError {
statusCode: number;
error?: string;
message?: string;
}

14
x-pack/plugins/reporting/server/lib/enqueue_job.ts
View file

@ -47,12 +47,16 @@ export function enqueueJobFactory(
throw new Error(`Export type ${exportTypeId} does not exist in the registry!`);
}
const [createJob, { store }] = await Promise.all([
exportType.createJobFnFactory(reporting, logger),
reporting.getPluginStartDeps(),
if (!exportType.createJobFnFactory) {
throw new Error(`Export type ${exportTypeId} is not an async job type!`);
}
const [createJob, store] = await Promise.all([
exportType.createJobFnFactory(reporting, logger.clone([exportType.id])),
reporting.getStore(),
]);
const job = await createJob(jobParams, context, request);
const job = await createJob!(jobParams, context, request);
const pendingReport = new Report({
jobtype: exportType.jobType,
created_by: user ? user.username : false,
@ -67,7 +71,7 @@ export function enqueueJobFactory(
// store the pending report, puts it in the Reporting Management UI table
const report = await store.addReport(pendingReport);
logger.info(`Scheduled ${exportType.name} report: ${report._id}`);
logger.info(`Queued ${exportType.name} report: ${report._id}`);
return report;
};

8
x-pack/plugins/reporting/server/lib/export_types_registry.ts
View file

@ -6,8 +6,9 @@
*/
import { isString } from 'lodash';
import { getExportType as getTypeCsv } from '../export_types/csv';
import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_from_savedobject';
import { getExportType as getTypeCsvDeprecated } from '../export_types/csv';
import { getExportType as getTypeCsvFromSavedObject } from '../export_types/csv_searchsource_immediate';
import { getExportType as getTypeCsv } from '../export_types/csv_searchsource';
import { getExportType as getTypePng } from '../export_types/png';
import { getExportType as getTypePrintablePdf } from '../export_types/printable_pdf';
import { CreateJobFn, ExportTypeDefinition } from '../types';
@ -82,8 +83,9 @@ export function getExportTypesRegistry(): ExportTypesRegistry {
const registry = new ExportTypesRegistry();
type CreateFnType = CreateJobFn<any, any>; // can not specify params types because different type of params are not assignable to each other
type RunFnType = any; // can not specify because ImmediateExecuteFn is not assignable to RunTaskFn
const getTypeFns: Array<() => ExportTypeDefinition<CreateFnType, RunFnType>> = [
const getTypeFns: Array<() => ExportTypeDefinition<CreateFnType | null, RunFnType>> = [
getTypeCsv,
getTypeCsvDeprecated,
getTypeCsvFromSavedObject,
getTypePng,
getTypePrintablePdf,

6
x-pack/plugins/reporting/server/plugin.ts
View file

@ -29,8 +29,8 @@ export class ReportingPlugin
constructor(context: PluginInitializerContext<ReportingConfigType>) {
this.logger = new LevelLogger(context.logger.get());
this.initializerContext = context;
this.reportingCore = new ReportingCore(this.logger);
this.initializerContext = context;
}
public setup(core: CoreSetup, plugins: ReportingSetupDeps) {
@ -121,8 +121,10 @@ export class ReportingPlugin
browserDriverFactory,
savedObjects: core.savedObjects,
uiSettings: core.uiSettings,
esqueue,
store,
esClient: core.elasticsearch.client,
data: plugins.data,
esqueue,
});
this.logger.debug('Start complete');

44
x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts → x-pack/plugins/reporting/server/routes/csv_searchsource_immediate.ts
View file

@ -8,26 +8,17 @@
import { schema } from '@kbn/config-schema';
import { KibanaRequest } from 'src/core/server';
import { ReportingCore } from '../';
import { createJobFnFactory } from '../export_types/csv_from_savedobject/create_job';
import { runTaskFnFactory } from '../export_types/csv_from_savedobject/execute_job';
import {
JobParamsPanelCsv,
JobParamsPanelCsvPost,
} from '../export_types/csv_from_savedobject/types';
import { runTaskFnFactory } from '../export_types/csv_searchsource_immediate/execute_job';
import { JobParamsDownloadCSV } from '../export_types/csv_searchsource_immediate/types';
import { LevelLogger as Logger } from '../lib';
import { TaskRunResult } from '../lib/tasks';
import { authorizedUserPreRoutingFactory } from './lib/authorized_user_pre_routing';
import { getJobParamsFromRequest } from './lib/get_job_params_from_request';
import { HandlerErrorFunction } from './types';
const API_BASE_URL_V1 = '/api/reporting/v1';
const API_BASE_GENERATE_V1 = `${API_BASE_URL_V1}/generate`;
export type CsvFromSavedObjectRequest = KibanaRequest<
JobParamsPanelCsv,
unknown,
JobParamsPanelCsvPost
>;
export type CsvFromSavedObjectRequest = KibanaRequest<unknown, unknown, JobParamsDownloadCSV>;
/*
* This function registers API Endpoints for immediate Reporting jobs. The API inputs are:
@ -47,43 +38,28 @@ export function registerGenerateCsvFromSavedObjectImmediate(
const userHandler = authorizedUserPreRoutingFactory(reporting);
const { router } = setupDeps;
/*
* CSV export with the `immediate` option does not queue a job with Reporting's ESQueue to run the job async. Instead, this does:
* - re-use the createJob function to build up es query config
* - re-use the runTask function to run the scan and scroll queries and capture the entire CSV in a result object.
*/
// This API calls run the SearchSourceImmediate export type's runTaskFn directly
router.post(
{
path: `${API_BASE_GENERATE_V1}/immediate/csv/saved-object/{savedObjectType}:{savedObjectId}`,
path: `${API_BASE_GENERATE_V1}/immediate/csv_searchsource`,
validate: {
params: schema.object({
savedObjectType: schema.string({ minLength: 5 }),
savedObjectId: schema.string({ minLength: 5 }),
}),
body: schema.object({
state: schema.object({}, { unknowns: 'allow' }),
timerange: schema.object({
timezone: schema.string({ defaultValue: 'UTC' }),
min: schema.nullable(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])),
max: schema.nullable(schema.oneOf([schema.number(), schema.string({ minLength: 5 })])),
}),
searchSource: schema.object({}, { unknowns: 'allow' }),
browserTimezone: schema.string({ defaultValue: 'UTC' }),
title: schema.string(),
}),
},
},
userHandler(async (user, context, req: CsvFromSavedObjectRequest, res) => {
const logger = parentLogger.clone(['savedobject-csv']);
const jobParams = getJobParamsFromRequest(req);
const createJob = createJobFnFactory(reporting, logger);
const logger = parentLogger.clone(['csv_searchsource_immediate']);
const runTaskFn = runTaskFnFactory(reporting, logger);
try {
// FIXME: no create job for immediate download
const payload = await createJob(jobParams, context, req);
const {
content_type: jobOutputContentType,
content: jobOutputContent,
size: jobOutputSize,
}: TaskRunResult = await runTaskFn(null, payload, context, req);
}: TaskRunResult = await runTaskFn(null, req.body, context, req);
logger.info(`Job output size: ${jobOutputSize} bytes`);

2
x-pack/plugins/reporting/server/routes/generation.ts
View file

@ -12,8 +12,8 @@ import { ReportingCore } from '../';
import { API_BASE_URL } from '../../common/constants';
import { LevelLogger as Logger } from '../lib';
import { enqueueJobFactory } from '../lib/enqueue_job';
import { registerGenerateCsvFromSavedObjectImmediate } from './csv_searchsource_immediate';
import { registerGenerateFromJobParams } from './generate_from_jobparams';
import { registerGenerateCsvFromSavedObjectImmediate } from './generate_from_savedobject_immediate';
import { registerLegacy } from './legacy';
import { HandlerFunction } from './types';

4
x-pack/plugins/reporting/server/routes/lib/get_document_payload.ts
View file

@ -8,7 +8,7 @@
// @ts-ignore
import contentDisposition from 'content-disposition';
import { get } from 'lodash';
import { CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants';
import { CSV_JOB_TYPE, CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants';
import { ExportTypesRegistry, statuses } from '../../lib';
import { ReportDocument } from '../../lib/store';
import { TaskRunResult } from '../../lib/tasks';
@ -34,7 +34,7 @@ const getTitle = (exportType: ExportTypeDefinition, title?: string): string =>
const getReportingHeaders = (output: TaskRunResult, exportType: ExportTypeDefinition) => {
const metaDataHeaders: Record<string, boolean> = {};
if (exportType.jobType === CSV_JOB_TYPE_DEPRECATED) {
if (exportType.jobType === CSV_JOB_TYPE || exportType.jobType === CSV_JOB_TYPE_DEPRECATED) {
const csvContainsFormulas = get(output, 'csv_contains_formulas', false);
const maxSizedReach = get(output, 'max_size_reached', false);

22
x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts
View file

@ -1,22 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { JobParamsPanelCsv } from '../../export_types/csv_from_savedobject/types';
import { CsvFromSavedObjectRequest } from '../generate_from_savedobject_immediate';
export function getJobParamsFromRequest(request: CsvFromSavedObjectRequest): JobParamsPanelCsv {
const { savedObjectType, savedObjectId } = request.params;
const { timerange, state } = request.body;
const post = timerange || state ? { timerange, state } : undefined;
return {
savedObjectType,
savedObjectId,
post,
};
}

35
x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts
View file

@ -12,7 +12,10 @@ jest.mock('../lib/create_queue');
import _ from 'lodash';
import * as Rx from 'rxjs';
import { coreMock } from 'src/core/server/mocks';
import { coreMock, elasticsearchServiceMock } from 'src/core/server/mocks';
import { fieldFormats } from 'src/plugins/data/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { dataPluginMock } from 'src/plugins/data/server/mocks';
import { ReportingConfig, ReportingCore } from '../';
import { featuresPluginMock } from '../../../features/server/mocks';
import {
@ -23,6 +26,7 @@ import {
import { ReportingConfigType } from '../config';
import { ReportingInternalSetup, ReportingInternalStart } from '../core';
import { ReportingStore } from '../lib';
import { setFieldFormats } from '../services';
import { createMockLevelLogger } from './create_mock_levellogger';
(initializeBrowserDriverFactory as jest.Mock<
@ -45,16 +49,23 @@ export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup =
const logger = createMockLevelLogger();
const createMockPluginStart = (
mockReportingCore: ReportingCore,
const createMockReportingStore = () => ({} as ReportingStore);
export const createMockPluginStart = (
mockReportingCore: ReportingCore | undefined,
startMock?: any
): ReportingInternalStart => {
const store = new ReportingStore(mockReportingCore, logger);
const store = mockReportingCore
? new ReportingStore(mockReportingCore, logger)
: createMockReportingStore();
return {
browserDriverFactory: startMock.browserDriverFactory,
esqueue: startMock.esqueue,
esClient: elasticsearchServiceMock.createClusterClient(),
savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() },
uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) },
data: startMock.data || dataPluginMock.createStartContract(),
store,
...startMock,
};
@ -121,11 +132,18 @@ export const createMockReportingCore = async (
setupDepsMock: ReportingInternalSetup | undefined = undefined,
startDepsMock: ReportingInternalStart | undefined = undefined
) => {
config = config || {};
const mockReportingCore = ({
getConfig: () => config,
getElasticsearchService: () => setupDepsMock?.elasticsearch,
getDataService: () => startDepsMock?.data,
} as unknown) as ReportingCore;
if (!setupDepsMock) {
setupDepsMock = createMockPluginSetup({});
}
if (!startDepsMock) {
startDepsMock = createMockPluginStart(mockReportingCore, {});
}
const context = coreMock.createPluginInitializerContext(createMockConfigSchema());
const core = new ReportingCore(logger);
@ -140,5 +158,12 @@ export const createMockReportingCore = async (
await core.pluginStart(startDepsMock);
await core.pluginStartsUp();
setFieldFormats({
fieldFormatServiceFactory() {
const fieldFormatsRegistry = new fieldFormats.FieldFormatsRegistry();
return Promise.resolve(fieldFormatsRegistry);
},
});
return core;
};

7
x-pack/plugins/reporting/server/types.ts
View file

@ -80,13 +80,16 @@ export type RunTaskFnFactory<RunTaskFnType> = (
logger: LevelLogger
) => RunTaskFnType;
export interface ExportTypeDefinition<CreateJobFnType = CreateJobFn, RunTaskFnType = RunTaskFn> {
export interface ExportTypeDefinition<
CreateJobFnType = CreateJobFn | null,
RunTaskFnType = RunTaskFn
> {
id: string;
name: string;
jobType: string;
jobContentEncoding?: string;
jobContentExtension: string;
createJobFnFactory: CreateJobFnFactory<CreateJobFnType>;
createJobFnFactory: CreateJobFnFactory<CreateJobFnType> | null; // immediate job does not have a "create" phase
runTaskFnFactory: RunTaskFnFactory<RunTaskFnType>;
validLicenses: string[];
}

24
x-pack/plugins/reporting/server/usage/__snapshots__/reporting_usage_collector.test.ts.snap generated
View file

@ -13,6 +13,10 @@ Object {
"available": true,
"total": 0,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"enabled": true,
"last7Days": Object {
"PNG": Object {
@ -24,6 +28,10 @@ Object {
"available": true,
"total": 0,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"printable_pdf": Object {
"app": Object {
"dashboard": 0,
@ -75,6 +83,10 @@ Object {
"available": true,
"total": 0,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"enabled": true,
"last7Days": Object {
"PNG": Object {
@ -86,6 +98,10 @@ Object {
"available": true,
"total": 0,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"printable_pdf": Object {
"app": Object {
"dashboard": 0,
@ -166,6 +182,10 @@ Object {
"available": true,
"total": 1,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"enabled": true,
"last7Days": Object {
"PNG": Object {
@ -177,6 +197,10 @@ Object {
"available": true,
"total": 1,
},
"csv_searchsource": Object {
"available": true,
"total": 0,
},
"printable_pdf": Object {
"app": Object {
"canvas workpad": 1,

8
x-pack/plugins/reporting/server/usage/decorate_range_stats.ts
View file

@ -6,7 +6,12 @@
*/
import { uniq } from 'lodash';
import { CSV_JOB_TYPE_DEPRECATED, PDF_JOB_TYPE, PNG_JOB_TYPE } from '../../common/constants';
import {
CSV_JOB_TYPE,
CSV_JOB_TYPE_DEPRECATED,
PDF_JOB_TYPE,
PNG_JOB_TYPE,
} from '../../common/constants';
import { AvailableTotal, ExportType, FeatureAvailabilityMap, RangeStats } from './types';
function getForFeature(
@ -55,6 +60,7 @@ export const decorateRangeStats = (
// combine the known types with any unknown type found in reporting data
const keysBasic = uniq([
CSV_JOB_TYPE,
CSV_JOB_TYPE_DEPRECATED,
PNG_JOB_TYPE,
...Object.keys(rangeStatsBasic),

154
x-pack/plugins/reporting/server/usage/reporting_usage_collector.test.ts
View file

@ -354,11 +354,13 @@ describe('data modeling', () => {
available: true,
browser_type: 'chromium',
csv: { available: true, total: 4 },
csv_searchsource: { available: true, total: 4 },
enabled: true,
last7Days: {
PNG: { available: true, total: 0 },
_all: 0,
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
printable_pdf: {
app: { dashboard: 0, visualization: 0 },
available: true,
@ -389,11 +391,13 @@ describe('data modeling', () => {
available: true,
browser_type: 'chromium',
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
enabled: true,
last7Days: {
PNG: { available: true, total: 3 },
_all: 4,
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
printable_pdf: {
app: { 'canvas workpad': 1, dashboard: 0, visualization: 0 },
available: true,
@ -431,6 +435,7 @@ describe('data modeling', () => {
layout: { preserve_layout: 0, print: 0 },
},
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
PNG: { available: true, total: 0 },
},
_all: 0,
@ -443,6 +448,7 @@ describe('data modeling', () => {
layout: { preserve_layout: 0, print: 0 },
},
csv: { available: true, total: 0 },
csv_searchsource: { available: true, total: 0 },
PNG: { available: true, total: 0 },
});
});
@ -491,6 +497,14 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"available": Object {
"type": "boolean",
},
"total": Object {
"type": "long",
},
},
"enabled": Object {
"type": "boolean",
},
@ -514,6 +528,14 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"available": Object {
"type": "boolean",
},
"total": Object {
"type": "long",
},
},
"printable_pdf": Object {
"app": Object {
"canvas workpad": Object {
@ -585,6 +607,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -620,6 +653,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -655,6 +699,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -690,6 +745,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -725,6 +791,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -760,6 +837,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -845,6 +933,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -880,6 +979,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -915,6 +1025,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -950,6 +1071,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -985,6 +1117,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",
@ -1020,6 +1163,17 @@ describe('Ready for collection observable', () => {
"type": "long",
},
},
"csv_searchsource": Object {
"canvas workpad": Object {
"type": "long",
},
"dashboard": Object {
"type": "long",
},
"visualization": Object {
"type": "long",
},
},
"printable_pdf": Object {
"canvas workpad": Object {
"type": "long",

2
x-pack/plugins/reporting/server/usage/schema.ts
View file

@ -16,6 +16,7 @@ const appCountsSchema: MakeSchemaFrom<AppCounts> = {
const byAppCountsSchema: MakeSchemaFrom<RangeStats['statuses']['cancelled']> = {
csv: appCountsSchema,
csv_searchsource: appCountsSchema,
PNG: appCountsSchema,
printable_pdf: appCountsSchema,
};
@ -27,6 +28,7 @@ const availableTotalSchema: MakeSchemaFrom<AvailableTotal> = {
const jobTypesSchema: MakeSchemaFrom<JobTypes> = {
csv: availableTotalSchema,
csv_searchsource: availableTotalSchema,
PNG: availableTotalSchema,
printable_pdf: {
...availableTotalSchema,

4
x-pack/plugins/reporting/server/usage/types.ts
View file

@ -59,7 +59,7 @@ export interface AvailableTotal {
total: number;
}
type BaseJobTypes = 'csv' | 'PNG' | 'printable_pdf';
type BaseJobTypes = 'csv' | 'csv_searchsource' | 'PNG' | 'printable_pdf';
export interface LayoutCounts {
print: number;
preserve_layout: number;
@ -106,7 +106,7 @@ export type ReportingUsageType = RangeStats & {
last7Days: RangeStats;
};
export type ExportType = 'csv' | 'printable_pdf' | 'PNG';
export type ExportType = 'csv' | 'csv_searchsource' | 'printable_pdf' | 'PNG';
export type FeatureAvailabilityMap = { [F in ExportType]: boolean };
export interface KeyCountBucket {

176
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -2436,6 +2436,16 @@
}
}
},
"csv_searchsource": {
"properties": {
"available": {
"type": "boolean"
},
"total": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"available": {
@ -2521,6 +2531,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2564,6 +2587,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2607,6 +2643,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2650,6 +2699,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2693,6 +2755,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2736,6 +2811,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2787,6 +2875,16 @@
}
}
},
"csv_searchsource": {
"properties": {
"available": {
"type": "boolean"
},
"total": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"available": {
@ -2872,6 +2970,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2915,6 +3026,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -2958,6 +3082,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -3001,6 +3138,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -3044,6 +3194,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {
@ -3087,6 +3250,19 @@
}
}
},
"csv_searchsource": {
"properties": {
"canvas workpad": {
"type": "long"
},
"dashboard": {
"type": "long"
},
"visualization": {
"type": "long"
}
}
},
"PNG": {
"properties": {
"canvas workpad": {

53
x-pack/test/functional/apps/dashboard/reporting/__snapshots__/download_csv.snap generated Normal file
View file

@ -0,0 +1,53 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`dashboard Reporting Download CSV E-Commerce Data Download CSV export of a saved search panel 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
"
`;
exports[`dashboard Reporting Download CSV E-Commerce Data Downloads a filtered CSV export of a saved search panel 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
"
`;
exports[`dashboard Reporting Download CSV Field Formatters and Scripted Fields Download CSV export of a saved search panel 1`] = `
"date,\\"_id\\",name,gender,value,year,\\"years_ago\\",\\"date_informal\\"
\\"Jan 1, 1984 @ 00:00:00.000\\",\\"1984-Fethany-F\\",Fethany,F,5,1984,\\"35.00000000000000000000\\",\\"Jan 1st 84\\"
\\"Jan 1, 1983 @ 00:00:00.000\\",\\"1983-Fethany-F\\",Fethany,F,\\"1,043\\",1983,\\"36.00000000000000000000\\",\\"Jan 1st 83\\"
\\"Jan 1, 1982 @ 00:00:00.000\\",\\"1982-Fethany-F\\",Fethany,F,780,1982,\\"37.00000000000000000000\\",\\"Jan 1st 82\\"
\\"Jan 1, 1981 @ 00:00:00.000\\",\\"1981-Fethany-F\\",Fethany,F,655,1981,\\"38.00000000000000000000\\",\\"Jan 1st 81\\"
\\"Jan 1, 1980 @ 00:00:00.000\\",\\"1980-Fethany-F\\",Fethany,F,702,1980,\\"39.00000000000000000000\\",\\"Jan 1st 80\\"
"
`;

158
x-pack/test/functional/apps/dashboard/reporting/download_csv.ts
View file

@ -9,91 +9,139 @@ import { REPO_ROOT } from '@kbn/utils';
import expect from '@kbn/expect';
import fs from 'fs';
import path from 'path';
import * as Rx from 'rxjs';
import { filter, first, map, timeout } from 'rxjs/operators';
import { FtrProviderContext } from '../../../ftr_provider_context';
const csvPath = path.resolve(REPO_ROOT, 'target/functional-tests/downloads/Ecommerce Data.csv');
// checks every 100ms for the file to exist in the download dir
// just wait up to 5 seconds
const getDownload$ = (filePath: string) => {
return Rx.interval(100).pipe(
map(() => fs.existsSync(filePath)),
filter((value) => value === true),
first(),
timeout(5000)
);
};
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const browser = getService('browser');
const dashboardPanelActions = getService('dashboardPanelActions');
const log = getService('log');
const testSubjects = getService('testSubjects');
const filterBar = getService('filterBar');
const find = getService('find');
const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']);
const retry = getService('retry');
const PageObjects = getPageObjects(['reporting', 'common', 'dashboard', 'timePicker']);
const getCsvPath = (name: string) =>
path.resolve(REPO_ROOT, `target/functional-tests/downloads/${name}.csv`);
// checks every 100ms for the file to exist in the download dir
// just wait up to 5 seconds
const getDownload = (filePath: string) => {
return retry.tryForTime(5000, async () => {
expect(fs.existsSync(filePath)).to.be(true);
return fs.readFileSync(filePath).toString();
});
};
const clickActionsMenu = async (headingTestSubj: string) => {
const savedSearchPanel = await testSubjects.find('embeddablePanelHeading-' + headingTestSubj);
await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
};
const clickDownloadCsv = async () => {
log.debug('click "More"');
await dashboardPanelActions.clickContextMenuMoreItem();
const actionItemTestSubj = 'embeddablePanelAction-downloadCsvReport';
await testSubjects.existOrFail(actionItemTestSubj); // wait for the full panel to display or else the test runner could click the wrong option!
log.debug('click "Download CSV"');
await testSubjects.click(actionItemTestSubj);
await testSubjects.existOrFail('csvDownloadStarted'); // validate toast panel
};
describe('Download CSV', () => {
before('initialize tests', async () => {
log.debug('ReportingPage:initTests');
await esArchiver.loadIfNeeded('reporting/ecommerce');
await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
await browser.setWindowSize(1600, 850);
});
after('clean up archives and previous file download', async () => {
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
afterEach('remove download', () => {
try {
fs.unlinkSync(csvPath);
fs.unlinkSync(getCsvPath('Ecommerce Data'));
} catch (e) {
// it might not have been there to begin with
}
});
it('Downloads a CSV export of a saved search panel', async function () {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
const savedSearchPanel = await testSubjects.find('embeddablePanelHeading-EcommerceData');
await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
describe('E-Commerce Data', () => {
before(async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
});
after(async () => {
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
const actionExists = await testSubjects.exists('embeddablePanelAction-downloadCsvReport');
if (!actionExists) {
await dashboardPanelActions.clickContextMenuMoreItem();
}
await testSubjects.existOrFail('embeddablePanelAction-downloadCsvReport'); // wait for the full panel to display or else the test runner could click the wrong option!
await testSubjects.click('embeddablePanelAction-downloadCsvReport');
await testSubjects.existOrFail('csvDownloadStarted'); // validate toast panel
it('Download CSV export of a saved search panel', async function () {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
await clickActionsMenu('EcommerceData');
await clickDownloadCsv();
const fileExists = await getDownload$(csvPath).toPromise();
expect(fileExists).to.be(true);
const csvFile = await getDownload(getCsvPath('Ecommerce Data'));
expectSnapshot(csvFile).toMatch();
});
// no need to validate download contents, API Integration tests do that some different variations
it('Downloads a filtered CSV export of a saved search panel', async function () {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
// add a filter
await filterBar.addFilter('currency', 'is', 'EUR');
await clickActionsMenu('EcommerceData');
await clickDownloadCsv();
const csvFile = await getDownload(getCsvPath('Ecommerce Data'));
expectSnapshot(csvFile).toMatch();
});
it('Gets the correct filename if panel titles are hidden', async () => {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard Hidden Panel Titles');
const savedSearchPanel = await find.byCssSelector(
'[data-test-embeddable-id="94eab06f-60ac-4a85-b771-3a8ed475c9bb"]'
); // panel title is hidden
await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
await clickDownloadCsv();
await testSubjects.existOrFail('csvDownloadStarted');
const csvFile = await getDownload(getCsvPath('Ecommerce Data')); // file exists with proper name
expect(csvFile).to.not.be(null);
});
});
it('Gets the correct filename if panel titles are hidden', async () => {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard Hidden Panel Titles');
const savedSearchPanel = await find.byCssSelector(
'[data-test-embeddable-id="94eab06f-60ac-4a85-b771-3a8ed475c9bb"]'
); // panel title is hidden
await dashboardPanelActions.toggleContextMenu(savedSearchPanel);
describe('Field Formatters and Scripted Fields', () => {
before(async () => {
await esArchiver.load('reporting/hugedata');
});
after(async () => {
await esArchiver.unload('reporting/hugedata');
});
const actionExists = await testSubjects.exists('embeddablePanelAction-downloadCsvReport');
if (!actionExists) {
await dashboardPanelActions.clickContextMenuMoreItem();
}
await testSubjects.existOrFail('embeddablePanelAction-downloadCsvReport');
await testSubjects.click('embeddablePanelAction-downloadCsvReport');
await testSubjects.existOrFail('csvDownloadStarted');
it('Download CSV export of a saved search panel', async () => {
await PageObjects.common.navigateToApp('dashboard');
await PageObjects.dashboard.loadSavedDashboard('names dashboard');
await PageObjects.timePicker.setAbsoluteRange(
'Jan 01, 1980 @ 00:00:00.000',
'Dec 31, 1984 @ 23:59:59.000'
);
const fileExists = await getDownload$(csvPath).toPromise(); // file exists with proper name
expect(fileExists).to.be(true);
await PageObjects.common.sleep(1000);
await filterBar.addFilter('name.keyword', 'is', 'Fethany');
await PageObjects.common.sleep(1000);
await clickActionsMenu('namessearch');
await clickDownloadCsv();
const csvFile = await getDownload(getCsvPath('namessearch'));
expectSnapshot(csvFile).toMatch();
});
});
});
}

106
x-pack/test/functional/apps/discover/__snapshots__/reporting.snap generated
View file

@ -1,40 +1,88 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`discover Discover Generate CSV: archived search generates a report with data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\",\\"\\"Women's Accessories\\"\\",\\"\\"Men's Accessories\\"\\"]\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0687606876\\"\\",\\"\\"ZO0290502905\\"\\",\\"\\"ZO0126701267\\"\\",\\"\\"ZO0308503085\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Shoes\\"\\",\\"\\"Women's Clothing\\"\\"]\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0006400064\\"\\",\\"\\"ZO0150601506\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0638206382\\"\\",\\"\\"ZO0038800388\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0297602976\\"\\",\\"\\"ZO0565605656\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0561405614\\"\\",\\"\\"ZO0281602816\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\"]\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0385003850\\"\\",\\"\\"ZO0408604086\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0505605056\\"\\",\\"\\"ZO0513605136\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0276702767\\"\\",\\"\\"ZO0291702917\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0046600466\\"\\",\\"\\"ZO0050800508\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Clothing\\"\\",\\"\\"Men's Shoes\\"\\"]\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0455604556\\"\\",\\"\\"ZO0680806808\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Clothing\\"\\",\\"\\"Women's Shoes\\"\\"]\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0229002290\\"\\",\\"\\"ZO0674406744\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0529905299\\"\\",\\"\\"ZO0617006170\\"\\"]\\"
exports[`discover Discover CSV Export Generate CSV: archived search generates a report with data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
"
`;
exports[`discover Discover Generate CSV: archived search generates a report with filtered data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"order_date\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\",\\"\\"Women's Accessories\\"\\",\\"\\"Men's Accessories\\"\\"]\\",EUR,19,716724,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0687606876\\"\\",\\"\\"ZO0290502905\\"\\",\\"\\"ZO0126701267\\"\\",\\"\\"ZO0308503085\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Shoes\\"\\",\\"\\"Women's Clothing\\"\\"]\\",EUR,45,591503,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0006400064\\"\\",\\"\\"ZO0150601506\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0638206382\\"\\",\\"\\"ZO0038800388\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0297602976\\"\\",\\"\\"ZO0565605656\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0561405614\\"\\",\\"\\"ZO0281602816\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Shoes\\"\\",\\"\\"Men's Clothing\\"\\"]\\",EUR,41,591636,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0385003850\\"\\",\\"\\"ZO0408604086\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0505605056\\"\\",\\"\\"ZO0513605136\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0276702767\\"\\",\\"\\"ZO0291702917\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0046600466\\"\\",\\"\\"ZO0050800508\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Men's Clothing\\"\\",\\"\\"Men's Shoes\\"\\"]\\",EUR,48,590970,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0455604556\\"\\",\\"\\"ZO0680806808\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Women's Clothing\\"\\",\\"\\"Women's Shoes\\"\\"]\\",EUR,46,591299,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0229002290\\"\\",\\"\\"ZO0674406744\\"\\"]\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Jul 12, 2019 @ 00:00:00.000\\",\\"[\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\",\\"\\"Dec 31, 2016 @ 00:00:00.000\\"\\"]\\",\\"[\\"\\"ZO0529905299\\"\\",\\"\\"ZO0617006170\\"\\"]\\"
exports[`discover Discover CSV Export Generate CSV: archived search generates a report with discover:searchFieldsFromSource = true 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
"
`;
exports[`discover Discover Generate CSV: new search generates a report with data 1`] = `
exports[`discover Discover CSV Export Generate CSV: archived search generates a report with filtered data 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
"
`;
exports[`discover Discover CSV Export Generate CSV: new search generates a report from a new search with data: default 1`] = `
"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,,Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"{
\\"\\"coordinates\\"\\": [
54.4,
24.5
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.39, 32.99, 10.34, 6.11\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"173.96\\",\\"173.96\\",4,4,order,sultan
"
`;
exports[`discover Discover CSV Export Generate CSV: new search generates a report from a new search with data: discover:searchFieldsFromSource 1`] = `
"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,,Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"{
\\"\\"coordinates\\"\\": [
54.4,
24.5
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.39, 32.99, 10.34, 6.11\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"173.96\\",\\"173.96\\",4,4,order,sultan
"
`;

100
x-pack/test/functional/apps/discover/reporting.ts
View file

@ -12,18 +12,25 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const log = getService('log');
const es = getService('es');
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const browser = getService('browser');
const PageObjects = getPageObjects(['reporting', 'common', 'discover', 'timePicker']);
const filterBar = getService('filterBar');
describe('Discover', () => {
const setFieldsFromSource = async (setValue: boolean) => {
await kibanaServer.uiSettings.update({ 'discover:searchFieldsFromSource': setValue });
};
describe('Discover CSV Export', () => {
before('initialize tests', async () => {
log.debug('ReportingPage:initTests');
await esArchiver.loadIfNeeded('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
await browser.setWindowSize(1600, 850);
});
after('clean up archives', async () => {
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
await es.deleteByQuery({
index: '.reporting-*',
refresh: true,
@ -31,7 +38,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
});
describe('Generate CSV: new search', () => {
describe('Check Available', () => {
beforeEach(() => PageObjects.common.navigateToApp('discover'));
it('is not available if new', async () => {
@ -63,8 +70,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
await PageObjects.reporting.openCsvReportingPanel();
expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
});
});
it('generates a report with data', async () => {
describe('Generate CSV: new search', () => {
beforeEach(async () => {
await esArchiver.load('reporting/ecommerce_kibana'); // reload the archive to wipe out changes made by each test
await PageObjects.common.navigateToApp('discover');
});
it('generates a report from a new search with data: default', async () => {
await PageObjects.discover.clickNewSearchButton();
await PageObjects.reporting.setTimepickerInDataRange();
await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated');
@ -79,6 +93,25 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
expectSnapshot(res.text).toMatch();
});
it('generates a report from a new search with data: discover:searchFieldsFromSource', async () => {
await setFieldsFromSource(true);
await PageObjects.discover.clickNewSearchButton();
await PageObjects.reporting.setTimepickerInDataRange();
await PageObjects.discover.saveSearch(
'my search - with fieldsFromSource data - expectReportCanBeCreated'
);
await PageObjects.reporting.openCsvReportingPanel();
await PageObjects.reporting.clickGenerateReportButton();
const url = await PageObjects.reporting.getReportURL(60000);
const res = await PageObjects.reporting.getResponse(url);
expect(res.status).to.equal(200);
expect(res.get('content-type')).to.equal('text/csv; charset=utf-8');
expectSnapshot(res.text).toMatch();
await setFieldsFromSource(false);
});
it('generates a report with no data', async () => {
await PageObjects.reporting.setTimepickerInNoDataRange();
await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated');
@ -98,6 +131,24 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
describe('Generate CSV: archived search', () => {
const setupPage = async () => {
const fromTime = 'Apr 27, 2019 @ 23:56:51.374';
const toTime = 'Aug 23, 2019 @ 16:18:51.821';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
};
const getReport = async () => {
await PageObjects.reporting.openCsvReportingPanel();
await PageObjects.reporting.clickGenerateReportButton();
const url = await PageObjects.reporting.getReportURL(60000);
const res = await PageObjects.reporting.getResponse(url);
expect(res.status).to.equal(200);
expect(res.get('content-type')).to.equal('text/csv; charset=utf-8');
return res;
};
before(async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
@ -111,41 +162,36 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
beforeEach(() => PageObjects.common.navigateToApp('discover'));
it('generates a report with data', async () => {
await setupPage();
await PageObjects.discover.loadSavedSearch('Ecommerce Data');
const fromTime = 'Apr 27, 2019 @ 23:56:51.374';
const toTime = 'Aug 23, 2019 @ 16:18:51.821';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
await PageObjects.reporting.openCsvReportingPanel();
await PageObjects.reporting.clickGenerateReportButton();
const url = await PageObjects.reporting.getReportURL(60000);
const res = await PageObjects.reporting.getResponse(url);
expect(res.status).to.equal(200);
expect(res.get('content-type')).to.equal('text/csv; charset=utf-8');
expectSnapshot(res.text).toMatch();
const { text } = await getReport();
expectSnapshot(text).toMatch();
});
it('generates a report with filtered data', async () => {
await setupPage();
await PageObjects.discover.loadSavedSearch('Ecommerce Data');
const fromTime = 'Apr 27, 2019 @ 23:56:51.374';
const toTime = 'Aug 23, 2019 @ 16:18:51.821';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
// filter and re-save
await filterBar.addFilter('currency', 'is', 'EUR');
await PageObjects.discover.saveSearch(`Ecommerce Data: EUR Filtered`);
await PageObjects.discover.saveSearch(`Ecommerce Data: EUR Filtered`); // renamed the search
await PageObjects.reporting.openCsvReportingPanel();
await PageObjects.reporting.clickGenerateReportButton();
const { text } = await getReport();
expectSnapshot(text).toMatch();
await PageObjects.discover.saveSearch(`Ecommerce Data`); // rename the search back for the next test
});
const url = await PageObjects.reporting.getReportURL(60000);
const res = await PageObjects.reporting.getResponse(url);
it('generates a report with discover:searchFieldsFromSource = true', async () => {
await setupPage();
await PageObjects.discover.loadSavedSearch('Ecommerce Data');
expect(res.status).to.equal(200);
expect(res.get('content-type')).to.equal('text/csv; charset=utf-8');
expectSnapshot(res.text).toMatch();
await setFieldsFromSource(true);
await browser.refresh();
const { text } = await getReport();
expectSnapshot(text).toMatch();
await setFieldsFromSource(false);
});
});
});

BIN
x-pack/test/functional/es_archives/reporting/ecommerce/data.json.gz
View file

Binary file not shown.

1015
x-pack/test/functional/es_archives/reporting/ecommerce/mappings.json
View file

File diff suppressed because it is too large Load diff

420
x-pack/test/functional/es_archives/reporting/ecommerce_kibana/data.json
View file

File diff suppressed because one or more lines are too long

51
x-pack/test/functional/es_archives/reporting/ecommerce_kibana/mappings.json
View file

@ -57,7 +57,6 @@
"map": "9134b47593116d7953f6adba096fc463",
"maps-telemetry": "5ef305b18111b77789afefbd36b66171",
"metrics-explorer-view": "3d1b76c39bfb2cc8296b024d73854724",
"migrationVersion": "4a1746014a75ade3a714e1db5763276f",
"ml-job": "3bb64c31915acf93fc724af137a0891b",
"ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9",
"monitoring-telemetry": "2669d5ec15e82391cf58df4294ee9c68",
@ -1745,56 +1744,6 @@
"dynamic": "false",
"type": "object"
},
"migrationVersion": {
"dynamic": "true",
"properties": {
"config": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"dashboard": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"index-pattern": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"search": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"visualization": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"ml-job": {
"properties": {
"datafeed_id": {

BIN
x-pack/test/functional/es_archives/reporting/hugedata/data.json.gz
View file

Binary file not shown.

2310
x-pack/test/functional/es_archives/reporting/hugedata/mappings.json
View file

File diff suppressed because it is too large Load diff

BIN
x-pack/test/functional/es_archives/reporting/scripted_small2/data.json.gz
View file

Binary file not shown.

2217
x-pack/test/functional/es_archives/reporting/scripted_small2/mappings.json
View file

File diff suppressed because it is too large Load diff

261
x-pack/test/reporting_api_integration/fixtures.ts
View file

@ -5,267 +5,6 @@
* 2.0.
*/
export const CSV_RESULT_TIMEBASED_UTC = `"@timestamp",clientip,extension
"Sep 20, 2015 @ 10:26:48.725","74.214.76.90",jpg
"Sep 20, 2015 @ 10:26:48.540","146.86.123.109",jpg
"Sep 20, 2015 @ 10:26:48.353","233.126.159.144",jpg
"Sep 20, 2015 @ 10:26:45.468","153.139.156.196",png
"Sep 20, 2015 @ 10:26:34.063","25.140.171.133",css
"Sep 20, 2015 @ 10:26:11.181","239.249.202.59",jpg
"Sep 20, 2015 @ 10:26:00.639","95.59.225.31",css
"Sep 20, 2015 @ 10:26:00.094","247.174.57.245",jpg
"Sep 20, 2015 @ 10:25:55.744","116.126.47.226",css
"Sep 20, 2015 @ 10:25:54.701","169.228.188.120",jpg
"Sep 20, 2015 @ 10:25:52.360","74.224.77.232",css
"Sep 20, 2015 @ 10:25:49.913","97.83.96.39",css
"Sep 20, 2015 @ 10:25:44.979","175.188.44.145",css
"Sep 20, 2015 @ 10:25:40.968","89.143.125.181",jpg
"Sep 20, 2015 @ 10:25:36.331","231.169.195.137",css
"Sep 20, 2015 @ 10:25:34.064","137.205.146.206",jpg
"Sep 20, 2015 @ 10:25:32.312","53.0.188.251",jpg
"Sep 20, 2015 @ 10:25:27.254","111.214.104.239",jpg
"Sep 20, 2015 @ 10:25:22.561","111.46.85.146",jpg
"Sep 20, 2015 @ 10:25:06.674","55.100.60.111",jpg
"Sep 20, 2015 @ 10:25:05.114","34.197.178.155",jpg
"Sep 20, 2015 @ 10:24:55.114","163.123.136.118",jpg
"Sep 20, 2015 @ 10:24:54.818","11.195.163.57",jpg
"Sep 20, 2015 @ 10:24:53.742","96.222.137.213",png
"Sep 20, 2015 @ 10:24:48.798","227.228.214.218",jpg
"Sep 20, 2015 @ 10:24:20.223","228.53.110.116",jpg
"Sep 20, 2015 @ 10:24:01.794","196.131.253.111",png
"Sep 20, 2015 @ 10:23:49.521","125.163.133.47",jpg
"Sep 20, 2015 @ 10:23:45.816","148.47.216.255",jpg
"Sep 20, 2015 @ 10:23:36.052","51.105.100.214",jpg
"Sep 20, 2015 @ 10:23:34.323","41.210.252.157",gif
"Sep 20, 2015 @ 10:23:27.213","248.163.75.193",png
"Sep 20, 2015 @ 10:23:14.866","48.43.210.167",png
"Sep 20, 2015 @ 10:23:10.578","33.95.78.209",css
"Sep 20, 2015 @ 10:23:07.001","96.40.73.208",css
"Sep 20, 2015 @ 10:23:02.876","174.32.230.63",jpg
"Sep 20, 2015 @ 10:23:00.019","140.233.207.177",jpg
"Sep 20, 2015 @ 10:22:47.447","37.127.124.65",jpg
"Sep 20, 2015 @ 10:22:45.803","130.171.208.139",png
"Sep 20, 2015 @ 10:22:45.590","39.250.210.253",jpg
"Sep 20, 2015 @ 10:22:43.997","248.239.221.43",css
"Sep 20, 2015 @ 10:22:36.107","232.64.207.109",gif
"Sep 20, 2015 @ 10:22:30.527","24.186.122.118",jpg
"Sep 20, 2015 @ 10:22:25.697","23.3.174.206",jpg
"Sep 20, 2015 @ 10:22:08.272","185.170.80.142",php
"Sep 20, 2015 @ 10:21:40.822","202.22.74.232",png
"Sep 20, 2015 @ 10:21:36.210","39.227.27.167",jpg
"Sep 20, 2015 @ 10:21:19.154","140.233.207.177",jpg
"Sep 20, 2015 @ 10:21:09.852","22.151.97.227",jpg
"Sep 20, 2015 @ 10:21:06.079","157.39.25.197",css
"Sep 20, 2015 @ 10:21:01.357","37.127.124.65",jpg
"Sep 20, 2015 @ 10:20:56.519","23.184.94.58",jpg
"Sep 20, 2015 @ 10:20:40.189","80.83.92.252",jpg
"Sep 20, 2015 @ 10:20:27.012","66.194.157.171",png
"Sep 20, 2015 @ 10:20:24.450","15.191.218.38",jpg
`;
export const CSV_RESULT_TIMEBASED_CUSTOM = `"@timestamp",clientip,extension
"Sep 20, 2015 @ 03:26:48.725","74.214.76.90",jpg
"Sep 20, 2015 @ 03:26:48.540","146.86.123.109",jpg
"Sep 20, 2015 @ 03:26:48.353","233.126.159.144",jpg
"Sep 20, 2015 @ 03:26:45.468","153.139.156.196",png
"Sep 20, 2015 @ 03:26:34.063","25.140.171.133",css
"Sep 20, 2015 @ 03:26:11.181","239.249.202.59",jpg
"Sep 20, 2015 @ 03:26:00.639","95.59.225.31",css
"Sep 20, 2015 @ 03:26:00.094","247.174.57.245",jpg
"Sep 20, 2015 @ 03:25:55.744","116.126.47.226",css
"Sep 20, 2015 @ 03:25:54.701","169.228.188.120",jpg
"Sep 20, 2015 @ 03:25:52.360","74.224.77.232",css
"Sep 20, 2015 @ 03:25:49.913","97.83.96.39",css
"Sep 20, 2015 @ 03:25:44.979","175.188.44.145",css
"Sep 20, 2015 @ 03:25:40.968","89.143.125.181",jpg
"Sep 20, 2015 @ 03:25:36.331","231.169.195.137",css
"Sep 20, 2015 @ 03:25:34.064","137.205.146.206",jpg
"Sep 20, 2015 @ 03:25:32.312","53.0.188.251",jpg
"Sep 20, 2015 @ 03:25:27.254","111.214.104.239",jpg
"Sep 20, 2015 @ 03:25:22.561","111.46.85.146",jpg
"Sep 20, 2015 @ 03:25:06.674","55.100.60.111",jpg
"Sep 20, 2015 @ 03:25:05.114","34.197.178.155",jpg
"Sep 20, 2015 @ 03:24:55.114","163.123.136.118",jpg
"Sep 20, 2015 @ 03:24:54.818","11.195.163.57",jpg
"Sep 20, 2015 @ 03:24:53.742","96.222.137.213",png
"Sep 20, 2015 @ 03:24:48.798","227.228.214.218",jpg
"Sep 20, 2015 @ 03:24:20.223","228.53.110.116",jpg
"Sep 20, 2015 @ 03:24:01.794","196.131.253.111",png
"Sep 20, 2015 @ 03:23:49.521","125.163.133.47",jpg
"Sep 20, 2015 @ 03:23:45.816","148.47.216.255",jpg
"Sep 20, 2015 @ 03:23:36.052","51.105.100.214",jpg
"Sep 20, 2015 @ 03:23:34.323","41.210.252.157",gif
"Sep 20, 2015 @ 03:23:27.213","248.163.75.193",png
"Sep 20, 2015 @ 03:23:14.866","48.43.210.167",png
"Sep 20, 2015 @ 03:23:10.578","33.95.78.209",css
"Sep 20, 2015 @ 03:23:07.001","96.40.73.208",css
"Sep 20, 2015 @ 03:23:02.876","174.32.230.63",jpg
"Sep 20, 2015 @ 03:23:00.019","140.233.207.177",jpg
"Sep 20, 2015 @ 03:22:47.447","37.127.124.65",jpg
"Sep 20, 2015 @ 03:22:45.803","130.171.208.139",png
"Sep 20, 2015 @ 03:22:45.590","39.250.210.253",jpg
"Sep 20, 2015 @ 03:22:43.997","248.239.221.43",css
"Sep 20, 2015 @ 03:22:36.107","232.64.207.109",gif
"Sep 20, 2015 @ 03:22:30.527","24.186.122.118",jpg
"Sep 20, 2015 @ 03:22:25.697","23.3.174.206",jpg
"Sep 20, 2015 @ 03:22:08.272","185.170.80.142",php
"Sep 20, 2015 @ 03:21:40.822","202.22.74.232",png
"Sep 20, 2015 @ 03:21:36.210","39.227.27.167",jpg
"Sep 20, 2015 @ 03:21:19.154","140.233.207.177",jpg
"Sep 20, 2015 @ 03:21:09.852","22.151.97.227",jpg
"Sep 20, 2015 @ 03:21:06.079","157.39.25.197",css
"Sep 20, 2015 @ 03:21:01.357","37.127.124.65",jpg
"Sep 20, 2015 @ 03:20:56.519","23.184.94.58",jpg
"Sep 20, 2015 @ 03:20:40.189","80.83.92.252",jpg
"Sep 20, 2015 @ 03:20:27.012","66.194.157.171",png
"Sep 20, 2015 @ 03:20:24.450","15.191.218.38",jpg
`;
export const CSV_RESULT_TIMELESS = `name,power
"Jonelle-Jane Marth","1.177"
"Suzie-May Rishel","1.824"
"Suzie-May Rishel","2.077"
"Rosana Casto","2.808"
"Stephen Cortez","4.986"
"Jonelle-Jane Marth","6.156"
"Jonelle-Jane Marth","7.097"
"Florinda Alejandro","10.373"
"Jonelle-Jane Marth","14.807"
"Suzie-May Rishel","19.738"
"Suzie-May Rishel","20.92"
"Florinda Alejandro","22.209"
`;
export const CSV_RESULT_SCRIPTED = `date,name,percent,value,year,"years_ago",gender
"Jan 1, 1980 @ 00:00:00.000",Fecki,0,92,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fecki,0,78,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fecky,"0.001","2,071","1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fekki,0,6,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felen,0,40,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felia,0,21,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felina,0,6,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felinda,"0.001","1,620","1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felinda,"0.001","1,886","1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felisa,0,5,"1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felita,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felkys,0,7,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felkys,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fell,0,6,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felle,0,22,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felma,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felynda,0,31,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fenita,0,219,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fenjamin,0,22,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fenjamin,0,27,"1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fenji,0,5,"1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Fennie,0,16,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fenny,0,5,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Ferenice,0,9,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Frijida,0,5,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Frita,0,14,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fritney,0,10,"1,980","39.00000000000000000000",F
`;
export const CSV_RESULT_SCRIPTED_REQUERY = `date,name,percent,value,year,"years_ago",gender
"Jan 1, 1980 @ 00:00:00.000",Felen,0,40,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felia,0,21,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felina,0,6,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felinda,"0.001","1,620","1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felinda,"0.001","1,886","1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felisa,0,5,"1,981","38.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felita,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felkys,0,7,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felkys,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Fell,0,6,"1,980","39.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felle,0,22,"1,980","39.00000000000000000000",F
"Jan 1, 1981 @ 00:00:00.000",Felma,0,8,"1,981","38.00000000000000000000",F
"Jan 1, 1980 @ 00:00:00.000",Felynda,0,31,"1,980","39.00000000000000000000",F
`;
export const CSV_RESULT_SCRIPTED_RESORTED = `date,year,name,value,"years_ago"
"Jan 1, 1981 @ 00:00:00.000","1,981",Farbara,"6,456","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Farbara,"8,026","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fecky,"1,930","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Fecky,"2,071","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Felinda,"1,886","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Feth,"3,685","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Feth,"4,246","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fetty,"1,763","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Fetty,"1,967","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Feverly,"1,987","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Feverly,"2,249","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fonnie,"2,330","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Fonnie,"2,748","39.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Frenda,"7,162","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Frenda,"8,335","39.00000000000000000000"
`;
export const CSV_RESULT_HUGE = `date,year,name,value,"years_ago"
"Jan 1, 1984 @ 00:00:00.000","1,984",Fobby,"2,791","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Frent,"3,416","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Frett,"2,679","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Filly,"3,366","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Frian,"34,468","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Fenjamin,"7,191","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Frandon,"5,863","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Fruce,"1,855","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Fryan,"7,236","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Frad,"2,482","35.00000000000000000000"
"Jan 1, 1984 @ 00:00:00.000","1,984",Fradley,"5,175","35.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Fryan,"7,114","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Fradley,"4,752","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Frian,"35,717","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Farbara,"4,434","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Fenjamin,"5,235","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Fruce,"1,914","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Fobby,"2,888","36.00000000000000000000"
"Jan 1, 1983 @ 00:00:00.000","1,983",Frett,"3,031","36.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Fonnie,"1,853","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Frandy,"2,082","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Fecky,"1,786","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Frandi,"2,056","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Fridget,"1,864","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Farbara,"5,081","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Feth,"2,818","37.00000000000000000000"
"Jan 1, 1982 @ 00:00:00.000","1,982",Frenda,"6,270","37.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fetty,"1,763","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fonnie,"2,330","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Farbara,"6,456","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Felinda,"1,886","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Frenda,"7,162","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Feth,"3,685","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Feverly,"1,987","38.00000000000000000000"
"Jan 1, 1981 @ 00:00:00.000","1,981",Fecky,"1,930","38.00000000000000000000"
"Jan 1, 1980 @ 00:00:00.000","1,980",Fonnie,"2,748","39.00000000000000000000"
`;
// 'UTC'
export const CSV_RESULT_NANOS = `date,message,"_id"
"Jan 1, 2015 @ 12:10:30.123456789","Hello 2",
"Jan 1, 2015 @ 12:10:30.000000000","Hello 1",
`;
// 'America/New_York'
export const CSV_RESULT_NANOS_CUSTOM = `date,message,"_id"
"Jan 1, 2015 @ 07:10:30.123456789","Hello 2",
"Jan 1, 2015 @ 07:10:30.000000000","Hello 1",
`;
export const CSV_RESULT_DOCVALUE = `"order_date",category,currency,"customer_id","order_id","day_of_week_i","order_date","products.created_on",sku
"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,26,569309,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0364103641"",""ZO0708807088""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,24,569311,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0024600246"",""ZO0660706607""]"
"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing"",""Men's Shoes""]",EUR,31,569312,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0425104251"",""ZO0107901079""]"
"Jun 26, 2019 @ 00:00:00.000","[""Men's Shoes""]",EUR,14,569336,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0512505125"",""ZO0384103841""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing""]",EUR,28,569337,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0634106341"",""ZO0066900669""]"
"Jun 26, 2019 @ 00:00:00.000","[""Men's Accessories"",""Men's Clothing""]",EUR,31,569338,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0702507025"",""ZO0528105281""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Shoes"",""Women's Clothing""]",EUR,27,569356,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0010500105"",""ZO0172201722""]"
"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing"",""Men's Shoes""]",EUR,19,569362,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0292402924"",""ZO0681006810""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Accessories"",""Women's Clothing""]",EUR,42,569370,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0358603586"",""ZO0641106411""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing"",""Women's Accessories""]",EUR,20,569371,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0225702257"",""ZO0186601866""]"
"Jun 26, 2019 @ 00:00:00.000","[""Women's Clothing"",""Women's Shoes""]",EUR,43,569375,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0347603476"",""ZO0668806688""]"
"Jun 26, 2019 @ 00:00:00.000","[""Men's Clothing""]",EUR,48,569387,3,"Jun 26, 2019 @ 00:00:00.000","[""Dec 15, 2016 @ 00:00:00.000"",""Dec 15, 2016 @ 00:00:00.000""]","[""ZO0593805938"",""ZO0125201252""]"
`;
// This concatenates lines of multi-line string into a single line.
// It is so long strings can be entered at short widths, making syntax highlighting easier on editors
function singleLine(literals: TemplateStringsArray): string {

2
x-pack/test/reporting_api_integration/reporting_and_security.config.ts
View file

@ -42,7 +42,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
`--server.maxPayloadBytes=1679958`,
`--server.port=${kbnTestConfig.getPort()}`,
`--xpack.reporting.capture.maxAttempts=1`,
`--xpack.reporting.csv.maxSizeBytes=2850`,
`--xpack.reporting.csv.maxSizeBytes=6000`,
`--xpack.reporting.queue.pollInterval=3000`,
`--xpack.security.session.idleTimeout=3600000`,
`--xpack.reporting.capture.networkPolicy.rules=${JSON.stringify(testPolicyRules)}`,

250
x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/csv_searchsource_immediate.snap generated Normal file
View file

@ -0,0 +1,250 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Reporting APIs CSV Generation from SearchSource Exports CSV with all fields when using defaults 1`] = `
"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,,Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"{
\\"\\"coordinates\\"\\": [
54.4,
24.5
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.39, 32.99, 10.34, 6.11\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"79.99, 59.99, 21.99, 11.99\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"173.96\\",\\"173.96\\",4,4,order,sultan
9gMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",EUR,Pia,Pia,\\"Pia Richards\\",\\"Pia Richards\\",FEMALE,45,Richards,Richards,,Saturday,5,\\"pia@richards-family.zzz\\",Cannes,Europe,FR,\\"{
\\"\\"coordinates\\"\\": [
7,
43.6
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"Alpes-Maritimes\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591503,\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"20.99, 20.99\\",\\"20.99, 20.99\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"10.7, 9.87\\",\\"20.99, 20.99\\",\\"14,761, 11,632\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"1, 1\\",\\"ZO0006400064, ZO0150601506\\",\\"0, 0\\",\\"20.99, 20.99\\",\\"20.99, 20.99\\",\\"0, 0\\",\\"ZO0006400064, ZO0150601506\\",\\"41.98\\",\\"41.98\\",2,2,order,pia
BgMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"_doc\\",\\"Women's Clothing\\",\\"Women's Clothing\\",EUR,Brigitte,Brigitte,\\"Brigitte Meyer\\",\\"Brigitte Meyer\\",FEMALE,12,Meyer,Meyer,,Saturday,5,\\"brigitte@meyer-family.zzz\\",\\"New York\\",\\"North America\\",US,\\"{
\\"\\"coordinates\\"\\": [
-74,
40.8
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"New York\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591709,\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"7.99, 32.99\\",\\"7.99, 32.99\\",\\"Women's Clothing, Women's Clothing\\",\\"Women's Clothing, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"3.6, 17.48\\",\\"7.99, 32.99\\",\\"20,734, 7,539\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"1, 1\\",\\"ZO0638206382, ZO0038800388\\",\\"0, 0\\",\\"7.99, 32.99\\",\\"7.99, 32.99\\",\\"0, 0\\",\\"ZO0638206382, ZO0038800388\\",\\"40.98\\",\\"40.98\\",2,2,order,brigitte
KQMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"_doc\\",\\"Men's Clothing\\",\\"Men's Clothing\\",EUR,Abd,Abd,\\"Abd Mccarthy\\",\\"Abd Mccarthy\\",MALE,52,Mccarthy,Mccarthy,,Saturday,5,\\"abd@mccarthy-family.zzz\\",Cairo,Africa,EG,\\"{
\\"\\"coordinates\\"\\": [
31.3,
30.1
],
\\"\\"type\\"\\": \\"\\"Point\\"\\"
}\\",\\"Cairo Governorate\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"Jul 12, 2019 @ 00:00:00.000\\",590937,\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"28.99, 12.99\\",\\"28.99, 12.99\\",\\"Men's Clothing, Men's Clothing\\",\\"Men's Clothing, Men's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"13.34, 6.11\\",\\"28.99, 12.99\\",\\"14,438, 23,607\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"1, 1\\",\\"ZO0297602976, ZO0565605656\\",\\"0, 0\\",\\"28.99, 12.99\\",\\"28.99, 12.99\\",\\"0, 0\\",\\"ZO0297602976, ZO0565605656\\",\\"41.98\\",\\"41.98\\",2,2,order,abd
"
`;
exports[`Reporting APIs CSV Generation from SearchSource Exports CSV with almost all fields when using fieldsFromSource 1`] = `
"\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,currency,\\"customer_first_name\\",\\"customer_full_name\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,geoip,manufacturer,\\"order_date\\",\\"order_id\\",products,\\"products.created_on\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al Boone\\",MALE,19,Boone,\\"-\\",Saturday,5,\\"sultan al@boone-family.zzz\\",\\"{\\"\\"city_name\\"\\":\\"\\"Abu Dhabi\\"\\",\\"\\"continent_name\\"\\":\\"\\"Asia\\"\\",\\"\\"country_iso_code\\"\\":\\"\\"AE\\"\\",\\"\\"location\\"\\":{\\"\\"lat\\"\\":24.5,\\"\\"lon\\"\\":54.4},\\"\\"region_name\\"\\":\\"\\"Abu Dhabi\\"\\"}\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"{\\"\\"_id\\"\\":\\"\\"sold_product_716724_23975\\"\\",\\"\\"base_price\\"\\":79.99,\\"\\"base_unit_price\\"\\":79.99,\\"\\"category\\"\\":\\"\\"Men's Shoes\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Angeldale\\"\\",\\"\\"min_price\\"\\":42.39,\\"\\"price\\"\\":79.99,\\"\\"product_id\\"\\":23975,\\"\\"product_name\\"\\":\\"\\"Winter boots - cognac\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0687606876\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":79.99,\\"\\"taxless_price\\"\\":79.99,\\"\\"unit_discount_amount\\"\\":0}, {\\"\\"_id\\"\\":\\"\\"sold_product_716724_6338\\"\\",\\"\\"base_price\\"\\":59.99,\\"\\"base_unit_price\\"\\":59.99,\\"\\"category\\"\\":\\"\\"Men's Clothing\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Oceanavigations\\"\\",\\"\\"min_price\\"\\":32.99,\\"\\"price\\"\\":59.99,\\"\\"product_id\\"\\":6338,\\"\\"product_name\\"\\":\\"\\"Trenchcoat - black\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0290502905\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":59.99,\\"\\"taxless_price\\"\\":59.99,\\"\\"unit_discount_amount\\"\\":0}, {\\"\\"_id\\"\\":\\"\\"sold_product_716724_14116\\"\\",\\"\\"base_price\\"\\":21.99,\\"\\"base_unit_price\\"\\":21.99,\\"\\"category\\"\\":\\"\\"Women's Accessories\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Microlutions\\"\\",\\"\\"min_price\\"\\":10.34,\\"\\"price\\"\\":21.99,\\"\\"product_id\\"\\":14116,\\"\\"product_name\\"\\":\\"\\"Watch - black\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0126701267\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":21.99,\\"\\"taxless_price\\"\\":21.99,\\"\\"unit_discount_amount\\"\\":0}, {\\"\\"_id\\"\\":\\"\\"sold_product_716724_15290\\"\\",\\"\\"base_price\\"\\":11.99,\\"\\"base_unit_price\\"\\":11.99,\\"\\"category\\"\\":\\"\\"Men's Accessories\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Oceanavigations\\"\\",\\"\\"min_price\\"\\":6.11,\\"\\"price\\"\\":11.99,\\"\\"product_id\\"\\":15290,\\"\\"product_name\\"\\":\\"\\"Hat - light grey multicolor\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0308503085\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":11.99,\\"\\"taxless_price\\"\\":11.99,\\"\\"unit_discount_amount\\"\\":0}\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",173.96,173.96,4,4,order,sultan
9gMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"_doc\\",\\"Women's Shoes, Women's Clothing\\",EUR,Pia,\\"Pia Richards\\",FEMALE,45,Richards,\\"-\\",Saturday,5,\\"pia@richards-family.zzz\\",\\"{\\"\\"city_name\\"\\":\\"\\"Cannes\\"\\",\\"\\"continent_name\\"\\":\\"\\"Europe\\"\\",\\"\\"country_iso_code\\"\\":\\"\\"FR\\"\\",\\"\\"location\\"\\":{\\"\\"lat\\"\\":43.6,\\"\\"lon\\"\\":7},\\"\\"region_name\\"\\":\\"\\"Alpes-Maritimes\\"\\"}\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591503,\\"{\\"\\"_id\\"\\":\\"\\"sold_product_591503_14761\\"\\",\\"\\"base_price\\"\\":20.99,\\"\\"base_unit_price\\"\\":20.99,\\"\\"category\\"\\":\\"\\"Women's Shoes\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Tigress Enterprises\\"\\",\\"\\"min_price\\"\\":10.7,\\"\\"price\\"\\":20.99,\\"\\"product_id\\"\\":14761,\\"\\"product_name\\"\\":\\"\\"Classic heels - blue\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0006400064\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":20.99,\\"\\"taxless_price\\"\\":20.99,\\"\\"unit_discount_amount\\"\\":0}, {\\"\\"_id\\"\\":\\"\\"sold_product_591503_11632\\"\\",\\"\\"base_price\\"\\":20.99,\\"\\"base_unit_price\\"\\":20.99,\\"\\"category\\"\\":\\"\\"Women's Clothing\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Pyramidustries\\"\\",\\"\\"min_price\\"\\":9.87,\\"\\"price\\"\\":20.99,\\"\\"product_id\\"\\":11632,\\"\\"product_name\\"\\":\\"\\"Summer dress - coral/pink\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0150601506\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":20.99,\\"\\"taxless_price\\"\\":20.99,\\"\\"unit_discount_amount\\"\\":0}\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\",41.98,41.98,2,2,order,pia
BgMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"_doc\\",\\"Women's Clothing\\",EUR,Brigitte,\\"Brigitte Meyer\\",FEMALE,12,Meyer,\\"-\\",Saturday,5,\\"brigitte@meyer-family.zzz\\",\\"{\\"\\"city_name\\"\\":\\"\\"New York\\"\\",\\"\\"continent_name\\"\\":\\"\\"North America\\"\\",\\"\\"country_iso_code\\"\\":\\"\\"US\\"\\",\\"\\"location\\"\\":{\\"\\"lat\\"\\":40.8,\\"\\"lon\\"\\":-74},\\"\\"region_name\\"\\":\\"\\"New York\\"\\"}\\",\\"Spherecords, Tigress Enterprises\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591709,\\"{\\"\\"_id\\"\\":\\"\\"sold_product_591709_20734\\"\\",\\"\\"base_price\\"\\":7.99,\\"\\"base_unit_price\\"\\":7.99,\\"\\"category\\"\\":\\"\\"Women's Clothing\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Spherecords\\"\\",\\"\\"min_price\\"\\":3.6,\\"\\"price\\"\\":7.99,\\"\\"product_id\\"\\":20734,\\"\\"product_name\\"\\":\\"\\"Basic T-shirt - dark blue\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0638206382\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":7.99,\\"\\"taxless_price\\"\\":7.99,\\"\\"unit_discount_amount\\"\\":0}, {\\"\\"_id\\"\\":\\"\\"sold_product_591709_7539\\"\\",\\"\\"base_price\\"\\":32.99,\\"\\"base_unit_price\\"\\":32.99,\\"\\"category\\"\\":\\"\\"Women's Clothing\\"\\",\\"\\"created_on\\"\\":\\"\\"2016-12-31T00:00:00+00:00\\"\\",\\"\\"discount_amount\\"\\":0,\\"\\"discount_percentage\\"\\":0,\\"\\"manufacturer\\"\\":\\"\\"Tigress Enterprises\\"\\",\\"\\"min_price\\"\\":17.48,\\"\\"price\\"\\":32.99,\\"\\"product_id\\"\\":7539,\\"\\"product_name\\"\\":\\"\\"Summer dress - scarab\\"\\",\\"\\"quantity\\"\\":1,\\"\\"sku\\"\\":\\"\\"ZO0038800388\\"\\",\\"\\"tax_amount\\"\\":0,\\"\\"taxful_price\\"\\":32.99,\\"\\"taxless_price\\"\\":32.99,\\"\\"unit_discount_amount\\"\\":0}\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\",40.98,40.98,2,2,order,brigitte
"
`;
exports[`Reporting APIs CSV Generation from SearchSource Logs the error explanation if the search query returns an error 1`] = `"{\\"statusCode\\":500,\\"error\\":\\"Internal Server Error\\",\\"message\\":\\"An internal server error occurred.\\"}"`;
exports[`Reporting APIs CSV Generation from SearchSource date formatting Formatted date_nanos data, UTC timezone 1`] = `
"date,message
\\"Jan 1, 2015 @ 12:10:30.123456789\\",\\"Hello 2\\"
\\"Jan 1, 2015 @ 12:10:30.000000000\\",\\"Hello 1\\"
"
`;
exports[`Reporting APIs CSV Generation from SearchSource date formatting Formatted date_nanos data, custom timezone (New York) 1`] = `
"date,message
\\"Jan 1, 2015 @ 07:10:30.123456789\\",\\"Hello 2\\"
\\"Jan 1, 2015 @ 07:10:30.000000000\\",\\"Hello 1\\"
"
`;
exports[`Reporting APIs CSV Generation from SearchSource date formatting With filters and timebased data, default to UTC 1`] = `
"\\"@timestamp\\",clientip,extension
\\"Sep 20, 2015 @ 10:26:48.725\\",\\"74.214.76.90\\",jpg
\\"Sep 20, 2015 @ 10:26:48.540\\",\\"146.86.123.109\\",jpg
\\"Sep 20, 2015 @ 10:26:48.353\\",\\"233.126.159.144\\",jpg
\\"Sep 20, 2015 @ 10:26:45.468\\",\\"153.139.156.196\\",png
\\"Sep 20, 2015 @ 10:26:34.063\\",\\"25.140.171.133\\",css
\\"Sep 20, 2015 @ 10:26:11.181\\",\\"239.249.202.59\\",jpg
\\"Sep 20, 2015 @ 10:26:00.639\\",\\"95.59.225.31\\",css
\\"Sep 20, 2015 @ 10:26:00.094\\",\\"247.174.57.245\\",jpg
\\"Sep 20, 2015 @ 10:25:55.744\\",\\"116.126.47.226\\",css
\\"Sep 20, 2015 @ 10:25:54.701\\",\\"169.228.188.120\\",jpg
\\"Sep 20, 2015 @ 10:25:52.360\\",\\"74.224.77.232\\",css
\\"Sep 20, 2015 @ 10:25:49.913\\",\\"97.83.96.39\\",css
\\"Sep 20, 2015 @ 10:25:44.979\\",\\"175.188.44.145\\",css
\\"Sep 20, 2015 @ 10:25:40.968\\",\\"89.143.125.181\\",jpg
\\"Sep 20, 2015 @ 10:25:36.331\\",\\"231.169.195.137\\",css
\\"Sep 20, 2015 @ 10:25:34.064\\",\\"137.205.146.206\\",jpg
\\"Sep 20, 2015 @ 10:25:32.312\\",\\"53.0.188.251\\",jpg
\\"Sep 20, 2015 @ 10:25:27.254\\",\\"111.214.104.239\\",jpg
\\"Sep 20, 2015 @ 10:25:22.561\\",\\"111.46.85.146\\",jpg
\\"Sep 20, 2015 @ 10:25:06.674\\",\\"55.100.60.111\\",jpg
\\"Sep 20, 2015 @ 10:25:05.114\\",\\"34.197.178.155\\",jpg
\\"Sep 20, 2015 @ 10:24:55.114\\",\\"163.123.136.118\\",jpg
\\"Sep 20, 2015 @ 10:24:54.818\\",\\"11.195.163.57\\",jpg
\\"Sep 20, 2015 @ 10:24:53.742\\",\\"96.222.137.213\\",png
\\"Sep 20, 2015 @ 10:24:48.798\\",\\"227.228.214.218\\",jpg
\\"Sep 20, 2015 @ 10:24:20.223\\",\\"228.53.110.116\\",jpg
\\"Sep 20, 2015 @ 10:24:01.794\\",\\"196.131.253.111\\",png
\\"Sep 20, 2015 @ 10:23:49.521\\",\\"125.163.133.47\\",jpg
\\"Sep 20, 2015 @ 10:23:45.816\\",\\"148.47.216.255\\",jpg
\\"Sep 20, 2015 @ 10:23:36.052\\",\\"51.105.100.214\\",jpg
\\"Sep 20, 2015 @ 10:23:34.323\\",\\"41.210.252.157\\",gif
\\"Sep 20, 2015 @ 10:23:27.213\\",\\"248.163.75.193\\",png
\\"Sep 20, 2015 @ 10:23:14.866\\",\\"48.43.210.167\\",png
\\"Sep 20, 2015 @ 10:23:10.578\\",\\"33.95.78.209\\",css
\\"Sep 20, 2015 @ 10:23:07.001\\",\\"96.40.73.208\\",css
\\"Sep 20, 2015 @ 10:23:02.876\\",\\"174.32.230.63\\",jpg
\\"Sep 20, 2015 @ 10:23:00.019\\",\\"140.233.207.177\\",jpg
\\"Sep 20, 2015 @ 10:22:47.447\\",\\"37.127.124.65\\",jpg
\\"Sep 20, 2015 @ 10:22:45.803\\",\\"130.171.208.139\\",png
\\"Sep 20, 2015 @ 10:22:45.590\\",\\"39.250.210.253\\",jpg
\\"Sep 20, 2015 @ 10:22:43.997\\",\\"248.239.221.43\\",css
\\"Sep 20, 2015 @ 10:22:36.107\\",\\"232.64.207.109\\",gif
\\"Sep 20, 2015 @ 10:22:30.527\\",\\"24.186.122.118\\",jpg
\\"Sep 20, 2015 @ 10:22:25.697\\",\\"23.3.174.206\\",jpg
\\"Sep 20, 2015 @ 10:22:08.272\\",\\"185.170.80.142\\",php
\\"Sep 20, 2015 @ 10:21:40.822\\",\\"202.22.74.232\\",png
\\"Sep 20, 2015 @ 10:21:36.210\\",\\"39.227.27.167\\",jpg
\\"Sep 20, 2015 @ 10:21:19.154\\",\\"140.233.207.177\\",jpg
\\"Sep 20, 2015 @ 10:21:09.852\\",\\"22.151.97.227\\",jpg
\\"Sep 20, 2015 @ 10:21:06.079\\",\\"157.39.25.197\\",css
\\"Sep 20, 2015 @ 10:21:01.357\\",\\"37.127.124.65\\",jpg
\\"Sep 20, 2015 @ 10:20:56.519\\",\\"23.184.94.58\\",jpg
\\"Sep 20, 2015 @ 10:20:40.189\\",\\"80.83.92.252\\",jpg
\\"Sep 20, 2015 @ 10:20:27.012\\",\\"66.194.157.171\\",png
\\"Sep 20, 2015 @ 10:20:24.450\\",\\"15.191.218.38\\",jpg
\\"Sep 20, 2015 @ 10:19:45.764\\",\\"199.113.69.162\\",jpg
\\"Sep 20, 2015 @ 10:19:43.754\\",\\"171.243.18.67\\",gif
\\"Sep 20, 2015 @ 10:19:41.208\\",\\"126.87.234.213\\",jpg
\\"Sep 20, 2015 @ 10:19:40.307\\",\\"78.216.173.242\\",css
"
`;
exports[`Reporting APIs CSV Generation from SearchSource date formatting With filters and timebased data, non-default timezone 1`] = `
"\\"@timestamp\\",clientip,extension
\\"Sep 20, 2015 @ 03:26:48.725\\",\\"74.214.76.90\\",jpg
\\"Sep 20, 2015 @ 03:26:48.540\\",\\"146.86.123.109\\",jpg
\\"Sep 20, 2015 @ 03:26:48.353\\",\\"233.126.159.144\\",jpg
\\"Sep 20, 2015 @ 03:26:45.468\\",\\"153.139.156.196\\",png
\\"Sep 20, 2015 @ 03:26:34.063\\",\\"25.140.171.133\\",css
\\"Sep 20, 2015 @ 03:26:11.181\\",\\"239.249.202.59\\",jpg
\\"Sep 20, 2015 @ 03:26:00.639\\",\\"95.59.225.31\\",css
\\"Sep 20, 2015 @ 03:26:00.094\\",\\"247.174.57.245\\",jpg
\\"Sep 20, 2015 @ 03:25:55.744\\",\\"116.126.47.226\\",css
\\"Sep 20, 2015 @ 03:25:54.701\\",\\"169.228.188.120\\",jpg
\\"Sep 20, 2015 @ 03:25:52.360\\",\\"74.224.77.232\\",css
\\"Sep 20, 2015 @ 03:25:49.913\\",\\"97.83.96.39\\",css
\\"Sep 20, 2015 @ 03:25:44.979\\",\\"175.188.44.145\\",css
\\"Sep 20, 2015 @ 03:25:40.968\\",\\"89.143.125.181\\",jpg
\\"Sep 20, 2015 @ 03:25:36.331\\",\\"231.169.195.137\\",css
\\"Sep 20, 2015 @ 03:25:34.064\\",\\"137.205.146.206\\",jpg
\\"Sep 20, 2015 @ 03:25:32.312\\",\\"53.0.188.251\\",jpg
\\"Sep 20, 2015 @ 03:25:27.254\\",\\"111.214.104.239\\",jpg
\\"Sep 20, 2015 @ 03:25:22.561\\",\\"111.46.85.146\\",jpg
\\"Sep 20, 2015 @ 03:25:06.674\\",\\"55.100.60.111\\",jpg
\\"Sep 20, 2015 @ 03:25:05.114\\",\\"34.197.178.155\\",jpg
\\"Sep 20, 2015 @ 03:24:55.114\\",\\"163.123.136.118\\",jpg
\\"Sep 20, 2015 @ 03:24:54.818\\",\\"11.195.163.57\\",jpg
\\"Sep 20, 2015 @ 03:24:53.742\\",\\"96.222.137.213\\",png
\\"Sep 20, 2015 @ 03:24:48.798\\",\\"227.228.214.218\\",jpg
\\"Sep 20, 2015 @ 03:24:20.223\\",\\"228.53.110.116\\",jpg
\\"Sep 20, 2015 @ 03:24:01.794\\",\\"196.131.253.111\\",png
\\"Sep 20, 2015 @ 03:23:49.521\\",\\"125.163.133.47\\",jpg
\\"Sep 20, 2015 @ 03:23:45.816\\",\\"148.47.216.255\\",jpg
\\"Sep 20, 2015 @ 03:23:36.052\\",\\"51.105.100.214\\",jpg
\\"Sep 20, 2015 @ 03:23:34.323\\",\\"41.210.252.157\\",gif
\\"Sep 20, 2015 @ 03:23:27.213\\",\\"248.163.75.193\\",png
\\"Sep 20, 2015 @ 03:23:14.866\\",\\"48.43.210.167\\",png
\\"Sep 20, 2015 @ 03:23:10.578\\",\\"33.95.78.209\\",css
\\"Sep 20, 2015 @ 03:23:07.001\\",\\"96.40.73.208\\",css
\\"Sep 20, 2015 @ 03:23:02.876\\",\\"174.32.230.63\\",jpg
\\"Sep 20, 2015 @ 03:23:00.019\\",\\"140.233.207.177\\",jpg
\\"Sep 20, 2015 @ 03:22:47.447\\",\\"37.127.124.65\\",jpg
\\"Sep 20, 2015 @ 03:22:45.803\\",\\"130.171.208.139\\",png
\\"Sep 20, 2015 @ 03:22:45.590\\",\\"39.250.210.253\\",jpg
\\"Sep 20, 2015 @ 03:22:43.997\\",\\"248.239.221.43\\",css
\\"Sep 20, 2015 @ 03:22:36.107\\",\\"232.64.207.109\\",gif
\\"Sep 20, 2015 @ 03:22:30.527\\",\\"24.186.122.118\\",jpg
\\"Sep 20, 2015 @ 03:22:25.697\\",\\"23.3.174.206\\",jpg
\\"Sep 20, 2015 @ 03:22:08.272\\",\\"185.170.80.142\\",php
\\"Sep 20, 2015 @ 03:21:40.822\\",\\"202.22.74.232\\",png
\\"Sep 20, 2015 @ 03:21:36.210\\",\\"39.227.27.167\\",jpg
\\"Sep 20, 2015 @ 03:21:19.154\\",\\"140.233.207.177\\",jpg
\\"Sep 20, 2015 @ 03:21:09.852\\",\\"22.151.97.227\\",jpg
\\"Sep 20, 2015 @ 03:21:06.079\\",\\"157.39.25.197\\",css
\\"Sep 20, 2015 @ 03:21:01.357\\",\\"37.127.124.65\\",jpg
\\"Sep 20, 2015 @ 03:20:56.519\\",\\"23.184.94.58\\",jpg
\\"Sep 20, 2015 @ 03:20:40.189\\",\\"80.83.92.252\\",jpg
\\"Sep 20, 2015 @ 03:20:27.012\\",\\"66.194.157.171\\",png
\\"Sep 20, 2015 @ 03:20:24.450\\",\\"15.191.218.38\\",jpg
\\"Sep 20, 2015 @ 03:19:45.764\\",\\"199.113.69.162\\",jpg
\\"Sep 20, 2015 @ 03:19:43.754\\",\\"171.243.18.67\\",gif
\\"Sep 20, 2015 @ 03:19:41.208\\",\\"126.87.234.213\\",jpg
\\"Sep 20, 2015 @ 03:19:40.307\\",\\"78.216.173.242\\",css
"
`;
exports[`Reporting APIs CSV Generation from SearchSource non-timebased Handle _id and _index columns 1`] = `
"date,message,\\"_id\\",\\"_index\\"
\\"Jan 1, 2015 @ 12:10:30.123456789\\",\\"Hello 2\\",2,nanos
\\"Jan 1, 2015 @ 12:10:30.000000000\\",\\"Hello 1\\",1,nanos
"
`;
exports[`Reporting APIs CSV Generation from SearchSource non-timebased With filters and non-timebased data 1`] = `
"name,power
\\"Jonelle-Jane Marth\\",1
\\"Suzie-May Rishel\\",1
\\"Suzie-May Rishel\\",2
\\"Rosana Casto\\",2
\\"Stephen Cortez\\",4
\\"Jonelle-Jane Marth\\",6
\\"Jonelle-Jane Marth\\",7
\\"Florinda Alejandro\\",10
\\"Jonelle-Jane Marth\\",14
\\"Suzie-May Rishel\\",19
\\"Suzie-May Rishel\\",20
\\"Florinda Alejandro\\",22
"
`;
exports[`Reporting APIs CSV Generation from SearchSource validation Searches large amount of data, stops at Max Size Reached 1`] = `
"\\"order_date\\",category,currency,\\"customer_id\\",\\"order_id\\",\\"day_of_week_i\\",\\"products.created_on\\",sku
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,19,716724,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,45,591503,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0006400064, ZO0150601506\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591709,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638206382, ZO0038800388\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,52,590937,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0297602976, ZO0565605656\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,29,590976,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0561405614, ZO0281602816\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,41,591636,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0385003850, ZO0408604086\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes\\",EUR,30,591539,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0505605056, ZO0513605136\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,41,591598,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0276702767, ZO0291702917\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,44,590927,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0046600466, ZO0050800508\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,48,590970,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0455604556, ZO0680806808\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,46,591299,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0229002290, ZO0674406744\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,36,591133,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0529905299, ZO0617006170\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,13,591175,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0299402994, ZO0433504335\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,21,591297,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0257502575, ZO0451704517\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,14,591149,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0584905849, ZO0578405784\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,27,591754,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0335803358, ZO0325903259\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Shoes\\",EUR,42,591803,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0645906459, ZO0324303243\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,46,592082,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0034400344, ZO0492904929\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Accessories\\",EUR,27,591283,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0239302393, ZO0198501985\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,4,591148,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0290302903, ZO0513705137\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Accessories, Men's Clothing\\",EUR,51,591417,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0464504645, ZO0621006210\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,14,591562,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0544305443, ZO0108001080\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing, Women's Accessories\\",EUR,5,590996,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0638106381, ZO0096900969\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes\\",EUR,27,591317,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0366203662, ZO0139501395\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,38,591362,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0541805418, ZO0594105941\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing\\",EUR,30,591411,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0693506935, ZO0532405324\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing, Men's Shoes\\",EUR,38,722629,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0424204242, ZO0403504035, ZO0506705067, ZO0395603956\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,16,591041,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0418704187, ZO0557105571\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,6,591074,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0268602686, ZO0484704847\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,7,591349,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0474804748, ZO0560705607\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Accessories, Women's Clothing\\",EUR,44,591374,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0206002060, ZO0268302683\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",EUR,12,591230,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0226902269, ZO0660106601\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",EUR,17,591717,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0248002480, ZO0646706467\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes\\",EUR,42,591768,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0005800058, ZO0133901339\\"
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",EUR,21,591810,5,\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"ZO0587405874, ZO0590305903\\"
"
`;

411
x-pack/test/reporting_api_integration/reporting_and_security/csv_saved_search.ts
View file

@ -1,411 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import supertest from 'supertest';
import * as fixtures from '../fixtures';
import { FtrProviderContext } from '../ftr_provider_context';
interface GenerateOpts {
timerange?: {
timezone: string;
min?: number | string | Date;
max?: number | string | Date;
};
state: any;
}
// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertestSvc = getService('supertest');
const reportingAPI = getService('reportingAPI');
const generateAPI = {
getCsvFromSavedSearch: async (
id: string,
{ timerange, state }: GenerateOpts,
isImmediate = true
) => {
return await supertestSvc
.post(`/api/reporting/v1/generate/${isImmediate ? 'immediate/' : ''}csv/saved-object/${id}`)
.set('kbn-xsrf', 'xxx')
.send({ timerange, state });
},
};
describe('Generation from Saved Search ID', () => {
after(async () => {
await reportingAPI.deleteAllReports();
});
describe('Saved Search Features', () => {
it('With filters and timebased data, explicit UTC format', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/logs');
await esArchiver.load('logstash_functional');
const res = (await generateAPI.getCsvFromSavedSearch(
'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
{
timerange: {
timezone: 'UTC',
min: '2015-09-19T10:00:00.000Z',
max: '2015-09-21T10:00:00.000Z',
},
state: {},
}
)) as supertest.Response;
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_UTC);
await esArchiver.unload('reporting/logs');
await esArchiver.unload('logstash_functional');
});
it('With filters and timebased data, default to UTC', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/logs');
await esArchiver.load('logstash_functional');
const res = (await generateAPI.getCsvFromSavedSearch(
'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
{
// @ts-expect-error: timerange.timezone is missing from post params
timerange: {
min: '2015-09-19T10:00:00.000Z',
max: '2015-09-21T10:00:00.000Z',
},
state: {},
}
)) as supertest.Response;
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_UTC);
await esArchiver.unload('reporting/logs');
await esArchiver.unload('logstash_functional');
});
it('With filters and timebased data, custom timezone', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/logs');
await esArchiver.load('logstash_functional');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
{
timerange: {
timezone: 'America/Phoenix',
min: '2015-09-19T10:00:00.000Z',
max: '2015-09-21T10:00:00.000Z',
},
state: {},
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_TIMEBASED_CUSTOM);
await esArchiver.unload('reporting/logs');
await esArchiver.unload('logstash_functional');
});
it('With filters and non-timebased data', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/sales');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:71e3ee20-3f99-11e9-b8ee-6b9604f2f877',
{
state: {},
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_TIMELESS);
await esArchiver.unload('reporting/sales');
});
it('With scripted fields and field formatters', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/scripted_small2');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:a6d51430-ace2-11ea-815f-39e12f89a8c2',
{
timerange: {
timezone: 'UTC',
min: '1979-01-01T10:00:00Z',
max: '1981-01-01T10:00:00Z',
},
state: {},
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED);
await esArchiver.unload('reporting/scripted_small2');
});
it('Formatted date_nanos data, UTC timezone', async () => {
await esArchiver.load('reporting/nanos');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:e4035040-a295-11e9-a900-ef10e0ac769e',
{
state: {},
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_NANOS);
await esArchiver.unload('reporting/nanos');
});
it('Formatted date_nanos data, custom time zone', async () => {
await esArchiver.load('reporting/nanos');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:e4035040-a295-11e9-a900-ef10e0ac769e',
{
state: {},
timerange: { timezone: 'America/New_York' },
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_NANOS_CUSTOM);
await esArchiver.unload('reporting/nanos');
});
});
describe('API Features', () => {
it('Return a 404', async () => {
const { body } = (await generateAPI.getCsvFromSavedSearch('search:gobbledygook', {
timerange: { timezone: 'UTC', min: 63097200000, max: 126255599999 },
state: {},
})) as supertest.Response;
const expectedBody = {
error: 'Not Found',
message: 'Saved object [search/gobbledygook] not found',
statusCode: 404,
};
expect(body).to.eql(expectedBody);
});
it('Return 400 if time range param is needed but missing', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/logs');
await esArchiver.load('logstash_functional');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7',
{ state: {} }
)) as supertest.Response;
expect(resStatus).to.eql(400);
expect(resType).to.eql('application/json');
const { message: errorMessage } = JSON.parse(resText);
expect(errorMessage).to.eql(
'Time range params are required for index pattern [logstash-*], using time field [@timestamp]'
);
await esArchiver.unload('reporting/logs');
await esArchiver.unload('logstash_functional');
});
it('Stops at Max Size Reached', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/hugedata');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:f34bf440-5014-11e9-bce7-4dabcb8bef24',
{
timerange: {
timezone: 'UTC',
min: '1960-01-01T10:00:00Z',
max: '1999-01-01T10:00:00Z',
},
state: {},
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_HUGE);
await esArchiver.unload('reporting/hugedata');
});
});
describe('Merge user state into the query', () => {
it('for query', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/scripted_small2');
const params = {
searchId: 'search:a6d51430-ace2-11ea-815f-39e12f89a8c2',
postPayload: {
timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore
state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fel*' } }] } } ] } } ] } } }, // prettier-ignore
},
isImmediate: true,
};
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
params.searchId,
params.postPayload,
params.isImmediate
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED_REQUERY);
await esArchiver.unload('reporting/scripted_small2');
});
it('for sort', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/hugedata');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
'search:f34bf440-5014-11e9-bce7-4dabcb8bef24',
{
timerange: {
timezone: 'UTC',
min: '1979-01-01T10:00:00Z',
max: '1981-01-01T10:00:00Z',
},
state: { sort: [{ name: { order: 'asc', unmapped_type: 'boolean' } }] },
}
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_SCRIPTED_RESORTED);
await esArchiver.unload('reporting/hugedata');
});
it('for docvalue_fields', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
const params = {
searchId: 'search:6091ead0-1c6d-11ea-a100-8589bb9d7c6b',
postPayload: {
timerange: {
min: '2019-05-28T00:00:00Z',
max: '2019-06-26T00:00:00Z',
timezone: 'UTC',
},
state: {
sort: [
{ order_date: { order: 'desc', unmapped_type: 'boolean' } },
{ order_id: { order: 'asc', unmapped_type: 'boolean' } },
],
docvalue_fields: [
{ field: 'customer_birth_date', format: 'date_time' },
{ field: 'order_date', format: 'date_time' },
{ field: 'products.created_on', format: 'date_time' },
],
query: {
bool: {
must: [],
filter: [
{ match_all: {} },
{ match_all: {} },
{
range: {
order_date: {
gte: '2019-05-28T00:00:00.000Z',
lte: '2019-06-26T00:00:00.000Z',
format: 'strict_date_optional_time',
},
},
},
],
should: [],
must_not: [],
},
},
},
},
isImmediate: true,
};
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCsvFromSavedSearch(
params.searchId,
params.postPayload,
params.isImmediate
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expect(resText).to.eql(fixtures.CSV_RESULT_DOCVALUE);
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
});
});
}

512
x-pack/test/reporting_api_integration/reporting_and_security/csv_searchsource_immediate.ts Normal file
View file

@ -0,0 +1,512 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import supertest from 'supertest';
import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types';
import { FtrProviderContext } from '../ftr_provider_context';
const getMockJobParams = (obj: Partial<JobParamsDownloadCSV>): JobParamsDownloadCSV => ({
title: `Mock CSV Title`,
...(obj as any),
});
// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');
const esArchiver = getService('esArchiver');
const supertestSvc = getService('supertest');
const reportingAPI = getService('reportingAPI');
const generateAPI = {
getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => {
return await supertestSvc
.post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send(job);
},
};
describe('CSV Generation from SearchSource', () => {
before(async () => {
await kibanaServer.uiSettings.update({
'csv:quoteValues': false,
'dateFormat:tz': 'UTC',
defaultIndex: 'logstash-*',
});
});
after(async () => {
await reportingAPI.deleteAllReports();
});
it('Exports CSV with almost all fields when using fieldsFromSource', async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
index: '5193f870-d861-11e9-a311-0fa548c5f953',
sort: [{ order_date: 'desc' }],
fieldsFromSource: [
'_id',
'_index',
'_score',
'_source',
'_type',
'category',
'category.keyword',
'currency',
'customer_birth_date',
'customer_first_name',
'customer_first_name.keyword',
'customer_full_name',
'customer_full_name.keyword',
'customer_gender',
'customer_id',
'customer_last_name',
'customer_last_name.keyword',
'customer_phone',
'day_of_week',
'day_of_week_i',
'email',
'geoip.city_name',
'geoip.continent_name',
'geoip.country_iso_code',
'geoip.location',
'geoip.region_name',
'manufacturer',
'manufacturer.keyword',
'order_date',
'order_id',
'products._id',
'products._id.keyword',
'products.base_price',
'products.base_unit_price',
'products.category',
'products.category.keyword',
'products.created_on',
'products.discount_amount',
'products.discount_percentage',
'products.manufacturer',
'products.manufacturer.keyword',
'products.min_price',
'products.price',
'products.product_id',
'products.product_name',
'products.product_name.keyword',
'products.quantity',
'products.sku',
'products.tax_amount',
'products.taxful_price',
'products.taxless_price',
'products.unit_discount_amount',
'sku',
'taxful_total_price',
'taxless_total_price',
'total_quantity',
'total_unique_products',
'type',
'user',
],
filter: [],
parent: {
query: { language: 'kuery', query: '' },
filter: [],
parent: {
filter: [
{
meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} },
range: {
order_date: {
gte: '2019-03-23T03:06:17.785Z',
lte: '2019-10-04T02:33:16.708Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
},
browserTimezone: 'UTC',
title: 'testfooyu78yt90-',
})
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
it('Exports CSV with all fields when using defaults', async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
index: '5193f870-d861-11e9-a311-0fa548c5f953',
sort: [{ order_date: 'desc' }],
fields: ['*'],
filter: [],
parent: {
query: { language: 'kuery', query: '' },
filter: [],
parent: {
filter: [
{
meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} },
range: {
order_date: {
gte: '2019-03-23T03:06:17.785Z',
lte: '2019-10-04T02:33:16.708Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
},
browserTimezone: 'UTC',
title: 'testfooyu78yt90-',
})
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
it('Logs the error explanation if the search query returns an error', async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
const { status: resStatus, text: resText } = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
index: '5193f870-d861-11e9-a311-0fa548c5f953',
sort: [{ order_date: 'desc' }],
fields: ['order_date', 'products'], // products is a non-leaf field
filter: [],
parent: {
query: { language: 'kuery', query: '' },
filter: [],
parent: {
filter: [
{
meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} },
range: {
order_date: {
gte: '2019-03-23T03:06:17.785Z',
lte: '2019-10-04T02:33:16.708Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
},
browserTimezone: 'UTC',
title: 'testfooyu78yt90-',
})
)) as supertest.Response;
expect(resStatus).to.eql(500);
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
describe('date formatting', () => {
before(async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/logs');
await esArchiver.load('logstash_functional');
});
after(async () => {
await esArchiver.unload('reporting/logs');
await esArchiver.unload('logstash_functional');
});
it('With filters and timebased data, default to UTC', async () => {
const res = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
fields: ['@timestamp', 'clientip', 'extension'],
filter: [
{
range: {
'@timestamp': {
gte: '2015-09-20T10:19:40.307Z',
lt: '2015-09-20T10:26:56.221Z',
},
},
},
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2015-01-12T07:00:55.654Z',
lte: '2016-01-29T21:08:10.881Z',
},
},
},
],
index: 'logstash-*',
query: { language: 'kuery', query: '' },
sort: [{ '@timestamp': 'desc' }],
},
})
)) as supertest.Response;
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
});
it('With filters and timebased data, non-default timezone', async () => {
const res = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
browserTimezone: 'America/Phoenix',
searchSource: {
fields: ['@timestamp', 'clientip', 'extension'],
filter: [
{
range: {
'@timestamp': {
gte: '2015-09-20T10:19:40.307Z',
lt: '2015-09-20T10:26:56.221Z',
},
},
},
{
range: {
'@timestamp': {
format: 'strict_date_optional_time',
gte: '2015-01-12T07:00:55.654Z',
lte: '2016-01-29T21:08:10.881Z',
},
},
},
],
index: 'logstash-*',
query: { language: 'kuery', query: '' },
sort: [{ '@timestamp': 'desc' }],
},
})
)) as supertest.Response;
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
});
it('Formatted date_nanos data, UTC timezone', async () => {
await esArchiver.load('reporting/nanos');
const res = await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
version: true,
index: '907bc200-a294-11e9-a900-ef10e0ac769e',
sort: [{ date: 'desc' }],
fields: ['date', 'message'],
filter: [],
},
})
);
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/nanos');
});
it('Formatted date_nanos data, custom timezone (New York)', async () => {
await esArchiver.load('reporting/nanos');
const res = await generateAPI.getCSVFromSearchSource(
getMockJobParams({
browserTimezone: 'America/New_York',
searchSource: {
query: { query: '', language: 'kuery' },
version: true,
index: '907bc200-a294-11e9-a900-ef10e0ac769e',
sort: [{ date: 'desc' }],
fields: ['date', 'message'],
filter: [],
},
})
);
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/nanos');
});
});
describe('non-timebased', () => {
it('Handle _id and _index columns', async () => {
await esArchiver.load('reporting/nanos');
const res = await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
version: true,
index: '907bc200-a294-11e9-a900-ef10e0ac769e',
sort: [{ date: 'desc' }],
fields: ['date', 'message', '_id', '_index'],
filter: [],
},
})
);
const { status: resStatus, text: resText, type: resType } = res;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/nanos');
});
it('With filters and non-timebased data', async () => {
// load test data that contains a saved search and documents
await esArchiver.load('reporting/sales');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
query: { query: '', language: 'kuery' },
version: true,
index: 'timeless-sales',
sort: [{ power: 'asc' }],
fields: ['name', 'power'],
filter: [
{
range: { power: { gte: 1, lt: null } },
},
],
},
})
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/sales');
});
});
describe('validation', () => {
it('Return a 404', async () => {
const { body } = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
index: 'gobbledygook',
},
})
)) as supertest.Response;
const expectedBody = {
error: 'Not Found',
message: 'Saved object [index-pattern/gobbledygook] not found',
statusCode: 404,
};
expect(body).to.eql(expectedBody);
});
it(`Searches large amount of data, stops at Max Size Reached`, async () => {
await esArchiver.load('reporting/ecommerce');
await esArchiver.load('reporting/ecommerce_kibana');
const {
status: resStatus,
text: resText,
type: resType,
} = (await generateAPI.getCSVFromSearchSource(
getMockJobParams({
searchSource: {
version: true,
query: { query: '', language: 'kuery' },
index: '5193f870-d861-11e9-a311-0fa548c5f953',
sort: [{ order_date: 'desc' }],
fields: [
'order_date',
'category',
'currency',
'customer_id',
'order_id',
'day_of_week_i',
'products.created_on',
'sku',
],
filter: [],
parent: {
query: { language: 'kuery', query: '' },
filter: [],
parent: {
filter: [
{
meta: { index: '5193f870-d861-11e9-a311-0fa548c5f953', params: {} },
range: {
order_date: {
gte: '2019-03-23T03:06:17.785Z',
lte: '2019-10-04T02:33:16.708Z',
format: 'strict_date_optional_time',
},
},
},
],
},
},
},
browserTimezone: 'UTC',
title: 'Ecommerce Data',
})
)) as supertest.Response;
expect(resStatus).to.eql(200);
expect(resType).to.eql('text/csv');
expectSnapshot(resText).toMatch();
await esArchiver.unload('reporting/ecommerce');
await esArchiver.unload('reporting/ecommerce_kibana');
});
});
});
}

2
x-pack/test/reporting_api_integration/reporting_and_security/index.ts
View file

@ -14,7 +14,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./bwc_existing_indexes'));
loadTestFile(require.resolve('./bwc_generation_urls'));
loadTestFile(require.resolve('./csv_job_params'));
loadTestFile(require.resolve('./csv_saved_search'));
loadTestFile(require.resolve('./csv_searchsource_immediate'));
loadTestFile(require.resolve('./network_policy'));
loadTestFile(require.resolve('./spaces'));
loadTestFile(require.resolve('./usage'));