Geo point field formatter (#121821)

* Geo point field formatter

* handle all possible types

* clean up tests

* tslint

* set default formatter

* make kind more descriptive

* add space to WKT output, make WKT text default output

* update reporting snapshots

* snapshot update

* add migration for format:defaultTypeMap

* update snapshots for 'CSV Generation from SearchSource'

* more snapshot updates

* fix jest test

src/core/server/ui_settings/saved_objects/migrations.test.ts

@@ -211,3 +211,51 @@ describe('ui_settings 8.0.0 migrations', () => {
     });
   });
 });
+
+describe('ui_settings 8.1.0 migrations', () => {
+  const migration = migrations['8.1.0'];
+
+  test('returns doc on empty object', () => {
+    expect(migration({} as SavedObjectUnsanitizedDoc)).toEqual({
+      references: [],
+    });
+  });
+
+  test('adds geo_point type to default map', () => {
+    const initialDefaultTypeMap = {
+      ip: { id: 'ip', params: {} },
+      date: { id: 'date', params: {} },
+      date_nanos: { id: 'date_nanos', params: {}, es: true },
+      number: { id: 'number', params: {} },
+      boolean: { id: 'boolean', params: {} },
+      histogram: { id: 'histogram', params: {} },
+      _source: { id: '_source', params: {} },
+      _default_: { id: 'string', params: {} },
+    };
+
+    const doc = {
+      type: 'config',
+      id: '8.0.0',
+      attributes: {
+        buildNum: 9007199254740991,
+        'format:defaultTypeMap': JSON.stringify(initialDefaultTypeMap),
+      },
+      references: [],
+      updated_at: '2020-06-09T20:18:20.349Z',
+      migrationVersion: {},
+    };
+    const migrated = migration(doc);
+    expect(migrated.attributes.buildNum).toBe(9007199254740991);
+    expect(JSON.parse(migrated.attributes['format:defaultTypeMap'])).toEqual({
+      ip: { id: 'ip', params: {} },
+      date: { id: 'date', params: {} },
+      date_nanos: { id: 'date_nanos', params: {}, es: true },
+      number: { id: 'number', params: {} },
+      boolean: { id: 'boolean', params: {} },
+      histogram: { id: 'histogram', params: {} },
+      _source: { id: '_source', params: {} },
+      _default_: { id: 'string', params: {} },
+      geo_point: { id: 'geo_point', params: { transform: 'wkt' } },
+    });
+  });
+});

src/core/server/ui_settings/saved_objects/migrations.ts

@@ -102,4 +102,28 @@ export const migrations = {
     }),
     references: doc.references || [],
   }),
+  '8.1.0': (doc: SavedObjectUnsanitizedDoc<any>): SavedObjectSanitizedDoc<any> => ({
+    ...doc,
+    ...(doc.attributes && {
+      attributes: Object.keys(doc.attributes).reduce((acc, key) => {
+        if (key === 'format:defaultTypeMap') {
+          const initial = JSON.parse(doc.attributes[key]);
+          const updated = {
+            ...initial,
+            geo_point: { id: 'geo_point', params: { transform: 'wkt' } },
+          };
+          return {
+            ...acc,
+            'format:defaultTypeMap': JSON.stringify(updated, null, 2),
+          };
+        } else {
+          return {
+            ...acc,
+            [key]: doc.attributes[key],
+          };
+        }
+      }, {}),
+    }),
+    references: doc.references || [],
+  }),
 };

src/plugins/data_view_field_editor/public/components/field_format_editor/editors/geo_point/constants.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const formatId = 'geo_point';

src/plugins/data_view_field_editor/public/components/field_format_editor/editors/geo_point/geo_point.tsx

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import React, { Fragment } from 'react';
+
+import { EuiFormRow, EuiSelect } from '@elastic/eui';
+
+import { FormattedMessage } from '@kbn/i18n-react';
+import { DefaultFormatEditor, defaultState } from '../default/default';
+
+import { FormatEditorSamples } from '../../samples';
+import { formatId } from './constants';
+import { GeoPointFormat } from '../../../../../../field_formats/common';
+
+interface GeoPointFormatEditorFormatParams {
+  transform: string;
+}
+
+export class GeoPointFormatEditor extends DefaultFormatEditor<GeoPointFormatEditorFormatParams> {
+  static formatId = formatId;
+  state = {
+    ...defaultState,
+    sampleInputs: [
+      {
+        coordinates: [125.6, 10.1],
+        type: 'Point',
+      },
+    ],
+  };
+
+  render() {
+    const { formatParams, format } = this.props;
+    const { error, samples } = this.state;
+
+    return (
+      <Fragment>
+        <EuiFormRow
+          label={
+            <FormattedMessage
+              id="indexPatternFieldEditor.geoPoint.transformLabel"
+              defaultMessage="Transform"
+            />
+          }
+          isInvalid={!!error}
+          error={error}
+        >
+          <EuiSelect
+            data-test-subj="geoPointEditorTransform"
+            defaultValue={formatParams.transform}
+            options={((format.type as typeof GeoPointFormat).transformOptions || []).map(
+              (option) => {
+                return {
+                  value: option.kind as string,
+                  text: option.text,
+                };
+              }
+            )}
+            onChange={(e) => {
+              this.onChange({ transform: e.target.value });
+            }}
+            isInvalid={!!error}
+          />
+        </EuiFormRow>
+        <FormatEditorSamples samples={samples} />
+      </Fragment>
+    );
+  }
+}

src/plugins/data_view_field_editor/public/components/field_format_editor/editors/geo_point/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FieldFormatEditorFactory } from '../types';
+import { formatId } from './constants';
+
+export type { GeoPointFormatEditor } from './geo_point';
+export const geoPointFormatEditorFactory: FieldFormatEditorFactory = () =>
+  import('./geo_point').then((m) => m.GeoPointFormatEditor);
+geoPointFormatEditorFactory.formatId = formatId;

src/plugins/data_view_field_editor/public/components/field_format_editor/editors/index.ts

@@ -15,6 +15,7 @@ export { ColorFormatEditor, colorFormatEditorFactory } from './color';
 export { DateFormatEditor, dateFormatEditorFactory } from './date';
 export { DateNanosFormatEditor, dateNanosFormatEditorFactory } from './date_nanos';
 export { DurationFormatEditor, durationFormatEditorFactory } from './duration';
+export { GeoPointFormatEditor, geoPointFormatEditorFactory } from './geo_point';
 export { NumberFormatEditor, numberFormatEditorFactory } from './number';
 export { PercentFormatEditor, percentFormatEditorFactory } from './percent';
 export { StaticLookupFormatEditor, staticLookupFormatEditorFactory } from './static_lookup';

src/plugins/data_view_field_editor/public/components/field_format_editor/types.ts

@@ -8,7 +8,7 @@
 
 import { ReactText } from 'react';
 
-export type SampleInput = ReactText | ReactText[] | Record<string, ReactText[]>;
+export type SampleInput = ReactText | ReactText[] | Record<string, ReactText[]> | object;
 
 export interface Sample {
   input: SampleInput;

src/plugins/data_view_field_editor/public/service/format_editor_service.ts

@@ -14,6 +14,7 @@ import {
   dateFormatEditorFactory,
   dateNanosFormatEditorFactory,
   durationFormatEditorFactory,
+  geoPointFormatEditorFactory,
   numberFormatEditorFactory,
   percentFormatEditorFactory,
   staticLookupFormatEditorFactory,
@@ -43,6 +44,7 @@ export class FormatEditorService {
       dateFormatEditorFactory,
       dateNanosFormatEditorFactory,
       durationFormatEditorFactory,
+      geoPointFormatEditorFactory,
       numberFormatEditorFactory,
       percentFormatEditorFactory,
       staticLookupFormatEditorFactory,

src/plugins/field_formats/common/constants/base_formatters.ts

@@ -13,6 +13,7 @@ import {
   BytesFormat,
   ColorFormat,
   DurationFormat,
+  GeoPointFormat,
   IpFormat,
   NumberFormat,
   PercentFormat,
@@ -30,6 +31,7 @@ export const baseFormatters: FieldFormatInstanceType[] = [
   BytesFormat,
   ColorFormat,
   DurationFormat,
+  GeoPointFormat,
   IpFormat,
   NumberFormat,
   PercentFormat,

src/plugins/field_formats/common/converters/geo_point.test.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { GeoPointFormat } from './geo_point';
+
+describe('GeoPoint Format', () => {
+  describe('output format', () => {
+    test('"lat_lon_string"', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert({ type: 'Point', coordinates: [125.6, 10.1] })).toBe(
+        '10.1,125.6'
+      );
+    });
+
+    test('"WKT"', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'wkt',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert({ type: 'Point', coordinates: [125.6, 10.1] })).toBe(
+        'POINT (125.6 10.1)'
+      );
+    });
+  });
+
+  describe('inputs', () => {
+    test('Geopoint expressed as an GeoJson geometry', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert({ type: 'Point', coordinates: [125.6, 10.1] })).toBe(
+        '10.1,125.6'
+      );
+    });
+
+    test('Geopoint expressed as an object, with lat and lon keys', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert({ lat: 10.1, lon: 125.6 })).toBe('10.1,125.6');
+    });
+
+    test('Geopoint expressed as a string with the format: "lat,lon"', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert('10.1,125.6')).toBe('10.1,125.6');
+    });
+
+    test('Geopoint expressed as a Well-Known Text POINT with the format: "POINT (lon lat)"', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert('POINT (125.6 10.1)')).toBe('10.1,125.6');
+    });
+
+    test('non-geopoint', () => {
+      const geoPointFormat = new GeoPointFormat(
+        {
+          transform: 'lat_lon_string',
+        },
+        jest.fn()
+      );
+      expect(geoPointFormat.convert('notgeopoint')).toBe('notgeopoint');
+    });
+  });
+});

src/plugins/field_formats/common/converters/geo_point.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Point } from 'geojson';
+import { i18n } from '@kbn/i18n';
+import { KBN_FIELD_TYPES } from '@kbn/field-types';
+import { FieldFormat } from '../field_format';
+import { FIELD_FORMAT_IDS, TextContextTypeConvert } from '../types';
+import { asPrettyString } from '../utils';
+
+const TRANSFORM_OPTIONS = [
+  {
+    kind: 'none',
+    text: i18n.translate('fieldFormats.geoPoint.transformOptions.none', {
+      defaultMessage: '- None -',
+    }),
+  },
+  {
+    kind: 'lat_lon_string',
+    text: i18n.translate('fieldFormats.geoPoint.transformOptions.latLonString', {
+      defaultMessage: 'string with the format: "lat,lon"',
+    }),
+  },
+  {
+    kind: 'wkt',
+    text: i18n.translate('fieldFormats.geoPoint.transformOptions.wkt', {
+      defaultMessage: 'Well-Known Text',
+    }),
+  },
+];
+
+/*
+ * Convert value to GeoJson point
+ *
+ * When value is read from fields API, its a GeoJSON Point geometry
+ * When value is ready from source, its as provided during ingest which supports multiple formats
+ */
+function toPoint(val: Point | { lat: number; lon: number } | string): Point | null {
+  let lat: number = NaN;
+  let lon: number = NaN;
+
+  if (typeof val === 'object' && 'lat' in val && 'lon' in val) {
+    lat = val.lat;
+    lon = val.lon;
+  } else if (typeof val === 'string') {
+    if (val.startsWith('POINT (')) {
+      const pointArg = val.slice('POINT ('.length, val.length);
+      const split = pointArg.split(' ');
+      if (split.length === 2) {
+        lat = parseFloat(split[1]);
+        lon = parseFloat(split[0]);
+      }
+    } else if (val.includes(',')) {
+      const split = val.split(',');
+      lat = parseFloat(split[0]);
+      lon = parseFloat(split[1]);
+    }
+  }
+
+  return Number.isNaN(lat) || Number.isNaN(lon)
+    ? null
+    : {
+        type: 'Point',
+        coordinates: [lon, lat],
+      };
+}
+
+function isPoint(val: Point | { lat: number; lon: number } | string): boolean {
+  return (
+    typeof val === 'object' &&
+    'type' in val &&
+    val.type === 'Point' &&
+    'coordinates' in val &&
+    val.coordinates.length === 2
+  );
+}
+
+/** @public */
+export class GeoPointFormat extends FieldFormat {
+  static id = FIELD_FORMAT_IDS.GEO_POINT;
+  static title = i18n.translate('fieldFormats.geoPoint.title', {
+    defaultMessage: 'Geo point',
+  });
+  static fieldType = [KBN_FIELD_TYPES.GEO_POINT];
+  static transformOptions = TRANSFORM_OPTIONS;
+
+  getParamDefaults() {
+    return {
+      transform: 'none',
+    };
+  }
+
+  textConvert: TextContextTypeConvert = (val: Point | { lat: number; lon: number } | string) => {
+    if (!val) {
+      return '';
+    }
+
+    const point: Point | null = isPoint(val) ? (val as Point) : toPoint(val);
+    if (!point) {
+      return asPrettyString(val);
+    }
+
+    switch (this.param('transform')) {
+      case 'lat_lon_string':
+        return `${point.coordinates[1]},${point.coordinates[0]}`;
+      case 'wkt':
+        return `POINT (${point.coordinates[0]} ${point.coordinates[1]})`;
+      default:
+        return asPrettyString(val);
+    }
+  };
+}

src/plugins/field_formats/common/converters/index.ts

@@ -10,6 +10,7 @@ export { UrlFormat } from './url';
 export { BytesFormat } from './bytes';
 export { RelativeDateFormat } from './relative_date';
 export { DurationFormat } from './duration';
+export { GeoPointFormat } from './geo_point';
 export { IpFormat } from './ip';
 export { NumberFormat } from './number';
 export { PercentFormat } from './percent';

src/plugins/field_formats/common/index.ts

@@ -21,6 +21,7 @@ export {
   BytesFormat,
   ColorFormat,
   DurationFormat,
+  GeoPointFormat,
   IpFormat,
   NumberFormat,
   PercentFormat,

src/plugins/field_formats/common/types.ts

@@ -61,6 +61,7 @@ export enum FIELD_FORMAT_IDS {
   DATE = 'date',
   DATE_NANOS = 'date_nanos',
   DURATION = 'duration',
+  GEO_POINT = 'geo_point',
   IP = 'ip',
   NUMBER = 'number',
   PERCENT = 'percent',

src/plugins/field_formats/server/ui_settings.ts

@@ -42,6 +42,7 @@ export function getUiSettings(): Record<string, UiSettingsParams<unknown>> {
   "ip": { "id": "ip", "params": {} },
   "date": { "id": "date", "params": {} },
   "date_nanos": { "id": "date_nanos", "params": {}, "es": true },
+  "geo_point": { "id": "geo_point", "params": { "transform": "wkt" } },
   "number": { "id": "number", "params": {} },
   "boolean": { "id": "boolean", "params": {} },
   "histogram": { "id": "histogram", "params": {} },
@@ -71,6 +72,12 @@ export function getUiSettings(): Record<string, UiSettingsParams<unknown>> {
           params: schema.object({}),
           es: schema.boolean(),
         }),
+        geo_point: schema.object({
+          id: schema.string(),
+          params: schema.object({
+            transform: schema.string(),
+          }),
+        }),
         number: schema.object({
           id: schema.string(),
           params: schema.object({}),

x-pack/test/functional/apps/discover/reporting.ts

@@ -111,7 +111,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
         // match file length, the beginning and the end of the csv file contents
         const { text: csvFile } = await getReport();
-        expect(csvFile.length).to.be(5093456);
+        expect(csvFile.length).to.be(4845681);
         expectSnapshot(csvFile.slice(0, 5000)).toMatch();
         expectSnapshot(csvFile.slice(-5000)).toMatch();
       });

x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/download_csv_dashboard.snap

@@ -168,41 +168,11 @@ exports[`Reporting APIs CSV Generation from SearchSource non-timebased With filt
 
 exports[`Reporting APIs CSV Generation from SearchSource unquoted values Exports CSV with all fields when using defaults 1`] = `
 "_id,_index,_score,_type,category,category.keyword,currency,customer_first_name,customer_first_name.keyword,customer_full_name,customer_full_name.keyword,customer_gender,customer_id,customer_last_name,customer_last_name.keyword,customer_phone,day_of_week,day_of_week_i,email,geoip.city_name,geoip.continent_name,geoip.country_iso_code,geoip.location,geoip.region_name,manufacturer,manufacturer.keyword,order_date,order_id,products._id,products._id.keyword,products.base_price,products.base_unit_price,products.category,products.category.keyword,products.created_on,products.discount_amount,products.discount_percentage,products.manufacturer,products.manufacturer.keyword,products.min_price,products.price,products.product_id,products.product_name,products.product_name.keyword,products.quantity,products.sku,products.tax_amount,products.taxful_price,products.taxless_price,products.unit_discount_amount,sku,taxful_total_price,taxless_total_price,total_quantity,total_unique_products,type,user
-9AMtOW0BH63Xcmy432DJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Boris,Boris,Boris Bradley,Boris Bradley,MALE,36,Bradley,Bradley,(empty),Wednesday,2,boris@bradley-family.zzz,-,Europe,GB,{
-  \\"coordinates\\": [
-    -0.1,
-    51.5
-  ],
-  \\"type\\": \\"Point\\"
-},-,Microlutions, Elitelligence,Microlutions, Elitelligence,Jun 25, 2019 @ 00:00:00.000,568397,sold_product_568397_24419, sold_product_568397_20207,sold_product_568397_24419, sold_product_568397_20207,33, 28.984,33, 28.984,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Microlutions, Elitelligence,Microlutions, Elitelligence,17.484, 13.922,33, 28.984,24,419, 20,207,Cargo trousers - oliv, Trousers - black,Cargo trousers - oliv, Trousers - black,1, 1,ZO0112101121, ZO0530405304,0, 0,33, 28.984,33, 28.984,0, 0,ZO0112101121, ZO0530405304,61.969,61.969,2,2,order,boris
-9QMtOW0BH63Xcmy432DJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Oliver,Oliver,Oliver Hubbard,Oliver Hubbard,MALE,7,Hubbard,Hubbard,(empty),Wednesday,2,oliver@hubbard-family.zzz,-,Europe,GB,{
-  \\"coordinates\\": [
-    -0.1,
-    51.5
-  ],
-  \\"type\\": \\"Point\\"
-},-,Spritechnologies, Microlutions,Spritechnologies, Microlutions,Jun 25, 2019 @ 00:00:00.000,568044,sold_product_568044_12799, sold_product_568044_18008,sold_product_568044_12799, sold_product_568044_18008,14.992, 16.984,14.992, 16.984,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Spritechnologies, Microlutions,Spritechnologies, Microlutions,6.898, 8.828,14.992, 16.984,12,799, 18,008,Undershirt - dark grey multicolor, Long sleeved top - purple,Undershirt - dark grey multicolor, Long sleeved top - purple,1, 1,ZO0630406304, ZO0120201202,0, 0,14.992, 16.984,14.992, 16.984,0, 0,ZO0630406304, ZO0120201202,31.984,31.984,2,2,order,oliver
-OAMtOW0BH63Xcmy432HJ,ecommerce,-,-,Women's Accessories,Women's Accessories,EUR,Betty,Betty,Betty Reese,Betty Reese,FEMALE,44,Reese,Reese,(empty),Wednesday,2,betty@reese-family.zzz,New York,North America,US,{
-  \\"coordinates\\": [
-    -74,
-    40.7
-  ],
-  \\"type\\": \\"Point\\"
-},New York,Pyramidustries,Pyramidustries,Jun 25, 2019 @ 00:00:00.000,568229,sold_product_568229_24991, sold_product_568229_12039,sold_product_568229_24991, sold_product_568229_12039,11.992, 10.992,11.992, 10.992,Women's Accessories, Women's Accessories,Women's Accessories, Women's Accessories,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Pyramidustries, Pyramidustries,Pyramidustries, Pyramidustries,6.352, 5.82,11.992, 10.992,24,991, 12,039,Scarf - rose/white, Scarf - nude/black/turquoise,Scarf - rose/white, Scarf - nude/black/turquoise,1, 1,ZO0192201922, ZO0192801928,0, 0,11.992, 10.992,11.992, 10.992,0, 0,ZO0192201922, ZO0192801928,22.984,22.984,2,2,order,betty
-OQMtOW0BH63Xcmy432HJ,ecommerce,-,-,Men's Clothing, Men's Accessories,Men's Clothing, Men's Accessories,EUR,Recip,Recip,Recip Salazar,Recip Salazar,MALE,10,Salazar,Salazar,(empty),Wednesday,2,recip@salazar-family.zzz,Istanbul,Asia,TR,{
-  \\"coordinates\\": [
-    29,
-    41
-  ],
-  \\"type\\": \\"Point\\"
-},Istanbul,Elitelligence,Elitelligence,Jun 25, 2019 @ 00:00:00.000,568292,sold_product_568292_23627, sold_product_568292_11149,sold_product_568292_23627, sold_product_568292_11149,24.984, 10.992,24.984, 10.992,Men's Clothing, Men's Accessories,Men's Clothing, Men's Accessories,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Elitelligence, Elitelligence,Elitelligence, Elitelligence,12.492, 5.059,24.984, 10.992,23,627, 11,149,Slim fit jeans - grey, Sunglasses - black,Slim fit jeans - grey, Sunglasses - black,1, 1,ZO0534205342, ZO0599605996,0, 0,24.984, 10.992,24.984, 10.992,0, 0,ZO0534205342, ZO0599605996,35.969,35.969,2,2,order,recip
-jwMtOW0BH63Xcmy432HJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Jackson,Jackson,Jackson Harper,Jackson Harper,MALE,13,Harper,Harper,(empty),Wednesday,2,jackson@harper-family.zzz,Los Angeles,North America,US,{
-  \\"coordinates\\": [
-    -118.2,
-    34.1
-  ],
-  \\"type\\": \\"Point\\"
-},California,Low Tide Media, Oceanavigations,Low Tide Media, Oceanavigations,Jun 25, 2019 @ 00:00:00.000,568386,sold_product_568386_11959, sold_product_568386_2774,sold_product_568386_11959, sold_product_568386_2774,24.984, 85,24.984, 85,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Low Tide Media, Oceanavigations,Low Tide Media, Oceanavigations,12.742, 45.875,24.984, 85,11,959, 2,774,SLIM FIT - Formal shirt - lila, Classic coat - black,SLIM FIT - Formal shirt - lila, Classic coat - black,1, 1,ZO0422404224, ZO0291702917,0, 0,24.984, 85,24.984, 85,0, 0,ZO0422404224, ZO0291702917,110,110,2,2,order,jackson
+9AMtOW0BH63Xcmy432DJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Boris,Boris,Boris Bradley,Boris Bradley,MALE,36,Bradley,Bradley,(empty),Wednesday,2,boris@bradley-family.zzz,-,Europe,GB,POINT (-0.1 51.5),-,Microlutions, Elitelligence,Microlutions, Elitelligence,Jun 25, 2019 @ 00:00:00.000,568397,sold_product_568397_24419, sold_product_568397_20207,sold_product_568397_24419, sold_product_568397_20207,33, 28.984,33, 28.984,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Microlutions, Elitelligence,Microlutions, Elitelligence,17.484, 13.922,33, 28.984,24,419, 20,207,Cargo trousers - oliv, Trousers - black,Cargo trousers - oliv, Trousers - black,1, 1,ZO0112101121, ZO0530405304,0, 0,33, 28.984,33, 28.984,0, 0,ZO0112101121, ZO0530405304,61.969,61.969,2,2,order,boris
+9QMtOW0BH63Xcmy432DJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Oliver,Oliver,Oliver Hubbard,Oliver Hubbard,MALE,7,Hubbard,Hubbard,(empty),Wednesday,2,oliver@hubbard-family.zzz,-,Europe,GB,POINT (-0.1 51.5),-,Spritechnologies, Microlutions,Spritechnologies, Microlutions,Jun 25, 2019 @ 00:00:00.000,568044,sold_product_568044_12799, sold_product_568044_18008,sold_product_568044_12799, sold_product_568044_18008,14.992, 16.984,14.992, 16.984,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Spritechnologies, Microlutions,Spritechnologies, Microlutions,6.898, 8.828,14.992, 16.984,12,799, 18,008,Undershirt - dark grey multicolor, Long sleeved top - purple,Undershirt - dark grey multicolor, Long sleeved top - purple,1, 1,ZO0630406304, ZO0120201202,0, 0,14.992, 16.984,14.992, 16.984,0, 0,ZO0630406304, ZO0120201202,31.984,31.984,2,2,order,oliver
+OAMtOW0BH63Xcmy432HJ,ecommerce,-,-,Women's Accessories,Women's Accessories,EUR,Betty,Betty,Betty Reese,Betty Reese,FEMALE,44,Reese,Reese,(empty),Wednesday,2,betty@reese-family.zzz,New York,North America,US,POINT (-74 40.7),New York,Pyramidustries,Pyramidustries,Jun 25, 2019 @ 00:00:00.000,568229,sold_product_568229_24991, sold_product_568229_12039,sold_product_568229_24991, sold_product_568229_12039,11.992, 10.992,11.992, 10.992,Women's Accessories, Women's Accessories,Women's Accessories, Women's Accessories,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Pyramidustries, Pyramidustries,Pyramidustries, Pyramidustries,6.352, 5.82,11.992, 10.992,24,991, 12,039,Scarf - rose/white, Scarf - nude/black/turquoise,Scarf - rose/white, Scarf - nude/black/turquoise,1, 1,ZO0192201922, ZO0192801928,0, 0,11.992, 10.992,11.992, 10.992,0, 0,ZO0192201922, ZO0192801928,22.984,22.984,2,2,order,betty
+OQMtOW0BH63Xcmy432HJ,ecommerce,-,-,Men's Clothing, Men's Accessories,Men's Clothing, Men's Accessories,EUR,Recip,Recip,Recip Salazar,Recip Salazar,MALE,10,Salazar,Salazar,(empty),Wednesday,2,recip@salazar-family.zzz,Istanbul,Asia,TR,POINT (29 41),Istanbul,Elitelligence,Elitelligence,Jun 25, 2019 @ 00:00:00.000,568292,sold_product_568292_23627, sold_product_568292_11149,sold_product_568292_23627, sold_product_568292_11149,24.984, 10.992,24.984, 10.992,Men's Clothing, Men's Accessories,Men's Clothing, Men's Accessories,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Elitelligence, Elitelligence,Elitelligence, Elitelligence,12.492, 5.059,24.984, 10.992,23,627, 11,149,Slim fit jeans - grey, Sunglasses - black,Slim fit jeans - grey, Sunglasses - black,1, 1,ZO0534205342, ZO0599605996,0, 0,24.984, 10.992,24.984, 10.992,0, 0,ZO0534205342, ZO0599605996,35.969,35.969,2,2,order,recip
+jwMtOW0BH63Xcmy432HJ,ecommerce,-,-,Men's Clothing,Men's Clothing,EUR,Jackson,Jackson,Jackson Harper,Jackson Harper,MALE,13,Harper,Harper,(empty),Wednesday,2,jackson@harper-family.zzz,Los Angeles,North America,US,POINT (-118.2 34.1),California,Low Tide Media, Oceanavigations,Low Tide Media, Oceanavigations,Jun 25, 2019 @ 00:00:00.000,568386,sold_product_568386_11959, sold_product_568386_2774,sold_product_568386_11959, sold_product_568386_2774,24.984, 85,24.984, 85,Men's Clothing, Men's Clothing,Men's Clothing, Men's Clothing,Dec 14, 2016 @ 00:00:00.000, Dec 14, 2016 @ 00:00:00.000,0, 0,0, 0,Low Tide Media, Oceanavigations,Low Tide Media, Oceanavigations,12.742, 45.875,24.984, 85,11,959, 2,774,SLIM FIT - Formal shirt - lila, Classic coat - black,SLIM FIT - Formal shirt - lila, Classic coat - black,1, 1,ZO0422404224, ZO0291702917,0, 0,24.984, 85,24.984, 85,0, 0,ZO0422404224, ZO0291702917,110,110,2,2,order,jackson
 "
 `;
 
@@ -217,33 +187,9 @@ OQMtOW0BH63Xcmy432HJ,ecommerce,-,-,Men's Clothing, Men's Accessories,EUR,Recip,R
 
 exports[`Reporting APIs CSV Generation from SearchSource validation Searches large amount of data, stops at Max Size Reached 1`] = `
 "\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
-3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,\\"(empty)\\",Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"{
-  \\"\\"coordinates\\"\\": [
-    54.4,
-    24.5
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.375, 33, 10.344, 6.109\\",\\"80, 60, 21.984, 11.992\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",174,174,4,4,order,sultan
-9gMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",EUR,Pia,Pia,\\"Pia Richards\\",\\"Pia Richards\\",FEMALE,45,Richards,Richards,\\"(empty)\\",Saturday,5,\\"pia@richards-family.zzz\\",Cannes,Europe,FR,\\"{
-  \\"\\"coordinates\\"\\": [
-    7,
-    43.6
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"Alpes-Maritimes\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591503,\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"20.984, 20.984\\",\\"20.984, 20.984\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"10.703, 9.867\\",\\"20.984, 20.984\\",\\"14,761, 11,632\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"1, 1\\",\\"ZO0006400064, ZO0150601506\\",\\"0, 0\\",\\"20.984, 20.984\\",\\"20.984, 20.984\\",\\"0, 0\\",\\"ZO0006400064, ZO0150601506\\",\\"41.969\\",\\"41.969\\",2,2,order,pia
-BgMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Clothing\\",\\"Women's Clothing\\",EUR,Brigitte,Brigitte,\\"Brigitte Meyer\\",\\"Brigitte Meyer\\",FEMALE,12,Meyer,Meyer,\\"(empty)\\",Saturday,5,\\"brigitte@meyer-family.zzz\\",\\"New York\\",\\"North America\\",US,\\"{
-  \\"\\"coordinates\\"\\": [
-    -74,
-    40.8
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"New York\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591709,\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"7.988, 33\\",\\"7.988, 33\\",\\"Women's Clothing, Women's Clothing\\",\\"Women's Clothing, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"3.6, 17.484\\",\\"7.988, 33\\",\\"20,734, 7,539\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"1, 1\\",\\"ZO0638206382, ZO0038800388\\",\\"0, 0\\",\\"7.988, 33\\",\\"7.988, 33\\",\\"0, 0\\",\\"ZO0638206382, ZO0038800388\\",\\"40.969\\",\\"40.969\\",2,2,order,brigitte
-KQMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing\\",\\"Men's Clothing\\",EUR,Abd,Abd,\\"Abd Mccarthy\\",\\"Abd Mccarthy\\",MALE,52,Mccarthy,Mccarthy,\\"(empty)\\",Saturday,5,\\"abd@mccarthy-family.zzz\\",Cairo,Africa,EG,\\"{
-  \\"\\"coordinates\\"\\": [
-    31.3,
-    30.1
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"Cairo Governorate\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"Jul 12, 2019 @ 00:00:00.000\\",590937,\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"28.984, 12.992\\",\\"28.984, 12.992\\",\\"Men's Clothing, Men's Clothing\\",\\"Men's Clothing, Men's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"13.344, 6.109\\",\\"28.984, 12.992\\",\\"14,438, 23,607\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"1, 1\\",\\"ZO0297602976, ZO0565605656\\",\\"0, 0\\",\\"28.984, 12.992\\",\\"28.984, 12.992\\",\\"0, 0\\",\\"ZO0297602976, ZO0565605656\\",\\"41.969\\",\\"41.969\\",2,2,order,abd
+3AMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,\\"(empty)\\",Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"POINT (54.4 24.5)\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.375, 33, 10.344, 6.109\\",\\"80, 60, 21.984, 11.992\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",174,174,4,4,order,sultan
+9gMtOW0BH63Xcmy432DJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",EUR,Pia,Pia,\\"Pia Richards\\",\\"Pia Richards\\",FEMALE,45,Richards,Richards,\\"(empty)\\",Saturday,5,\\"pia@richards-family.zzz\\",Cannes,Europe,FR,\\"POINT (7 43.6)\\",\\"Alpes-Maritimes\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591503,\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"sold_product_591503_14761, sold_product_591503_11632\\",\\"20.984, 20.984\\",\\"20.984, 20.984\\",\\"Women's Shoes, Women's Clothing\\",\\"Women's Shoes, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Tigress Enterprises, Pyramidustries\\",\\"Tigress Enterprises, Pyramidustries\\",\\"10.703, 9.867\\",\\"20.984, 20.984\\",\\"14,761, 11,632\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"Classic heels - blue, Summer dress - coral/pink\\",\\"1, 1\\",\\"ZO0006400064, ZO0150601506\\",\\"0, 0\\",\\"20.984, 20.984\\",\\"20.984, 20.984\\",\\"0, 0\\",\\"ZO0006400064, ZO0150601506\\",\\"41.969\\",\\"41.969\\",2,2,order,pia
+BgMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Clothing\\",\\"Women's Clothing\\",EUR,Brigitte,Brigitte,\\"Brigitte Meyer\\",\\"Brigitte Meyer\\",FEMALE,12,Meyer,Meyer,\\"(empty)\\",Saturday,5,\\"brigitte@meyer-family.zzz\\",\\"New York\\",\\"North America\\",US,\\"POINT (-74 40.8)\\",\\"New York\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"Jul 12, 2019 @ 00:00:00.000\\",591709,\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"sold_product_591709_20734, sold_product_591709_7539\\",\\"7.988, 33\\",\\"7.988, 33\\",\\"Women's Clothing, Women's Clothing\\",\\"Women's Clothing, Women's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spherecords, Tigress Enterprises\\",\\"Spherecords, Tigress Enterprises\\",\\"3.6, 17.484\\",\\"7.988, 33\\",\\"20,734, 7,539\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"Basic T-shirt - dark blue, Summer dress - scarab\\",\\"1, 1\\",\\"ZO0638206382, ZO0038800388\\",\\"0, 0\\",\\"7.988, 33\\",\\"7.988, 33\\",\\"0, 0\\",\\"ZO0638206382, ZO0038800388\\",\\"40.969\\",\\"40.969\\",2,2,order,brigitte
+KQMtOW0BH63Xcmy432LJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing\\",\\"Men's Clothing\\",EUR,Abd,Abd,\\"Abd Mccarthy\\",\\"Abd Mccarthy\\",MALE,52,Mccarthy,Mccarthy,\\"(empty)\\",Saturday,5,\\"abd@mccarthy-family.zzz\\",Cairo,Africa,EG,\\"POINT (31.3 30.1)\\",\\"Cairo Governorate\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"Jul 12, 2019 @ 00:00:00.000\\",590937,\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"sold_product_590937_14438, sold_product_590937_23607\\",\\"28.984, 12.992\\",\\"28.984, 12.992\\",\\"Men's Clothing, Men's Clothing\\",\\"Men's Clothing, Men's Clothing\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Oceanavigations, Elitelligence\\",\\"Oceanavigations, Elitelligence\\",\\"13.344, 6.109\\",\\"28.984, 12.992\\",\\"14,438, 23,607\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"Jumper - dark grey multicolor, Print T-shirt - black\\",\\"1, 1\\",\\"ZO0297602976, ZO0565605656\\",\\"0, 0\\",\\"28.984, 12.992\\",\\"28.984, 12.992\\",\\"0, 0\\",\\"ZO0297602976, ZO0565605656\\",\\"41.969\\",\\"41.969\\",2,2,order,abd
 "
 `;

x-pack/test/reporting_api_integration/reporting_and_security/__snapshots__/generate_csv_discover.snap

@@ -2,33 +2,9 @@
 
 exports[`Reporting APIs Generate CSV from SearchSource exported CSV file matches snapshot 1`] = `
 "\\"_id\\",\\"_index\\",\\"_score\\",\\"_type\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
-NwMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Accessories, Men's Clothing\\",\\"Men's Accessories, Men's Clothing\\",EUR,Mostafa,Mostafa,\\"Mostafa Lambert\\",\\"Mostafa Lambert\\",MALE,9,Lambert,Lambert,\\"(empty)\\",Tuesday,1,\\"mostafa@lambert-family.zzz\\",Cairo,Africa,EG,\\"{
-  \\"\\"coordinates\\"\\": [
-    31.3,
-    30.1
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"Cairo Governorate\\",\\"Oceanavigations, Low Tide Media\\",\\"Oceanavigations, Low Tide Media\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567868,\\"sold_product_567868_15827, sold_product_567868_6221\\",\\"sold_product_567868_15827, sold_product_567868_6221\\",\\"20.984, 28.984\\",\\"20.984, 28.984\\",\\"Men's Accessories, Men's Clothing\\",\\"Men's Accessories, Men's Clothing\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Oceanavigations, Low Tide Media\\",\\"Oceanavigations, Low Tide Media\\",\\"9.867, 15.07\\",\\"20.984, 28.984\\",\\"15,827, 6,221\\",\\"Belt - black/brown, Shirt - dark blue\\",\\"Belt - black/brown, Shirt - dark blue\\",\\"1, 1\\",\\"ZO0310403104, ZO0416604166\\",\\"0, 0\\",\\"20.984, 28.984\\",\\"20.984, 28.984\\",\\"0, 0\\",\\"ZO0310403104, ZO0416604166\\",\\"49.969\\",\\"49.969\\",2,2,order,mostafa
-SgMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Shoes\\",\\"Women's Shoes\\",EUR,Selena,Selena,\\"Selena Lewis\\",\\"Selena Lewis\\",FEMALE,42,Lewis,Lewis,\\"(empty)\\",Tuesday,1,\\"selena@lewis-family.zzz\\",Marrakesh,Africa,MA,\\"{
-  \\"\\"coordinates\\"\\": [
-    -8,
-    31.6
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"Marrakech-Tensift-Al Haouz\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567446,\\"sold_product_567446_12751, sold_product_567446_12494\\",\\"sold_product_567446_12751, sold_product_567446_12494\\",\\"65, 24.984\\",\\"65, 24.984\\",\\"Women's Shoes, Women's Shoes\\",\\"Women's Shoes, Women's Shoes\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Gnomehouse, Tigress Enterprises\\",\\"31.844, 11.25\\",\\"65, 24.984\\",\\"12,751, 12,494\\",\\"Lace-ups - black, Classic heels - cognac/beige\\",\\"Lace-ups - black, Classic heels - cognac/beige\\",\\"1, 1\\",\\"ZO0322803228, ZO0002700027\\",\\"0, 0\\",\\"65, 24.984\\",\\"65, 24.984\\",\\"0, 0\\",\\"ZO0322803228, ZO0002700027\\",90,90,2,2,order,selena
-bwMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing, Men's Shoes\\",\\"Men's Clothing, Men's Shoes\\",EUR,Oliver,Oliver,\\"Oliver Martin\\",\\"Oliver Martin\\",MALE,7,Martin,Martin,\\"(empty)\\",Tuesday,1,\\"oliver@martin-family.zzz\\",\\"-\\",Europe,GB,\\"{
-  \\"\\"coordinates\\"\\": [
-    -0.1,
-    51.5
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",\\"-\\",\\"Spritechnologies, Elitelligence\\",\\"Spritechnologies, Elitelligence\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567340,\\"sold_product_567340_3840, sold_product_567340_14835\\",\\"sold_product_567340_3840, sold_product_567340_14835\\",\\"16.984, 42\\",\\"16.984, 42\\",\\"Men's Clothing, Men's Shoes\\",\\"Men's Clothing, Men's Shoes\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spritechnologies, Elitelligence\\",\\"Spritechnologies, Elitelligence\\",\\"7.82, 21.406\\",\\"16.984, 42\\",\\"3,840, 14,835\\",\\"Sports shirt - dark grey multicolor, High-top trainers - grey\\",\\"Sports shirt - dark grey multicolor, High-top trainers - grey\\",\\"1, 1\\",\\"ZO0615606156, ZO0514905149\\",\\"0, 0\\",\\"16.984, 42\\",\\"16.984, 42\\",\\"0, 0\\",\\"ZO0615606156, ZO0514905149\\",\\"58.969\\",\\"58.969\\",2,2,order,oliver
-5AMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing\\",\\"Men's Clothing\\",EUR,Kamal,Kamal,\\"Kamal Salazar\\",\\"Kamal Salazar\\",MALE,39,Salazar,Salazar,\\"(empty)\\",Tuesday,1,\\"kamal@salazar-family.zzz\\",Istanbul,Asia,TR,\\"{
-  \\"\\"coordinates\\"\\": [
-    29,
-    41
-  ],
-  \\"\\"type\\"\\": \\"\\"Point\\"\\"
-}\\",Istanbul,\\"Spherecords, Spritechnologies\\",\\"Spherecords, Spritechnologies\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567736,\\"sold_product_567736_24718, sold_product_567736_24306\\",\\"sold_product_567736_24718, sold_product_567736_24306\\",\\"11.992, 75\\",\\"11.992, 75\\",\\"Men's Clothing, Men's Clothing\\",\\"Men's Clothing, Men's Clothing\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spherecords, Spritechnologies\\",\\"Spherecords, Spritechnologies\\",\\"6.109, 36.75\\",\\"11.992, 75\\",\\"24,718, 24,306\\",\\"Pyjama bottoms - light grey multicolor, Waterproof trousers - scarlet\\",\\"Pyjama bottoms - light grey multicolor, Waterproof trousers - scarlet\\",\\"1, 1\\",\\"ZO0663706637, ZO0620906209\\",\\"0, 0\\",\\"11.992, 75\\",\\"11.992, 75\\",\\"0, 0\\",\\"ZO0663706637, ZO0620906209\\",87,87,2,2,order,kamal
+NwMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Accessories, Men's Clothing\\",\\"Men's Accessories, Men's Clothing\\",EUR,Mostafa,Mostafa,\\"Mostafa Lambert\\",\\"Mostafa Lambert\\",MALE,9,Lambert,Lambert,\\"(empty)\\",Tuesday,1,\\"mostafa@lambert-family.zzz\\",Cairo,Africa,EG,\\"POINT (31.3 30.1)\\",\\"Cairo Governorate\\",\\"Oceanavigations, Low Tide Media\\",\\"Oceanavigations, Low Tide Media\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567868,\\"sold_product_567868_15827, sold_product_567868_6221\\",\\"sold_product_567868_15827, sold_product_567868_6221\\",\\"20.984, 28.984\\",\\"20.984, 28.984\\",\\"Men's Accessories, Men's Clothing\\",\\"Men's Accessories, Men's Clothing\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Oceanavigations, Low Tide Media\\",\\"Oceanavigations, Low Tide Media\\",\\"9.867, 15.07\\",\\"20.984, 28.984\\",\\"15,827, 6,221\\",\\"Belt - black/brown, Shirt - dark blue\\",\\"Belt - black/brown, Shirt - dark blue\\",\\"1, 1\\",\\"ZO0310403104, ZO0416604166\\",\\"0, 0\\",\\"20.984, 28.984\\",\\"20.984, 28.984\\",\\"0, 0\\",\\"ZO0310403104, ZO0416604166\\",\\"49.969\\",\\"49.969\\",2,2,order,mostafa
+SgMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Women's Shoes\\",\\"Women's Shoes\\",EUR,Selena,Selena,\\"Selena Lewis\\",\\"Selena Lewis\\",FEMALE,42,Lewis,Lewis,\\"(empty)\\",Tuesday,1,\\"selena@lewis-family.zzz\\",Marrakesh,Africa,MA,\\"POINT (-8 31.6)\\",\\"Marrakech-Tensift-Al Haouz\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567446,\\"sold_product_567446_12751, sold_product_567446_12494\\",\\"sold_product_567446_12751, sold_product_567446_12494\\",\\"65, 24.984\\",\\"65, 24.984\\",\\"Women's Shoes, Women's Shoes\\",\\"Women's Shoes, Women's Shoes\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Gnomehouse, Tigress Enterprises\\",\\"Gnomehouse, Tigress Enterprises\\",\\"31.844, 11.25\\",\\"65, 24.984\\",\\"12,751, 12,494\\",\\"Lace-ups - black, Classic heels - cognac/beige\\",\\"Lace-ups - black, Classic heels - cognac/beige\\",\\"1, 1\\",\\"ZO0322803228, ZO0002700027\\",\\"0, 0\\",\\"65, 24.984\\",\\"65, 24.984\\",\\"0, 0\\",\\"ZO0322803228, ZO0002700027\\",90,90,2,2,order,selena
+bwMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing, Men's Shoes\\",\\"Men's Clothing, Men's Shoes\\",EUR,Oliver,Oliver,\\"Oliver Martin\\",\\"Oliver Martin\\",MALE,7,Martin,Martin,\\"(empty)\\",Tuesday,1,\\"oliver@martin-family.zzz\\",\\"-\\",Europe,GB,\\"POINT (-0.1 51.5)\\",\\"-\\",\\"Spritechnologies, Elitelligence\\",\\"Spritechnologies, Elitelligence\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567340,\\"sold_product_567340_3840, sold_product_567340_14835\\",\\"sold_product_567340_3840, sold_product_567340_14835\\",\\"16.984, 42\\",\\"16.984, 42\\",\\"Men's Clothing, Men's Shoes\\",\\"Men's Clothing, Men's Shoes\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spritechnologies, Elitelligence\\",\\"Spritechnologies, Elitelligence\\",\\"7.82, 21.406\\",\\"16.984, 42\\",\\"3,840, 14,835\\",\\"Sports shirt - dark grey multicolor, High-top trainers - grey\\",\\"Sports shirt - dark grey multicolor, High-top trainers - grey\\",\\"1, 1\\",\\"ZO0615606156, ZO0514905149\\",\\"0, 0\\",\\"16.984, 42\\",\\"16.984, 42\\",\\"0, 0\\",\\"ZO0615606156, ZO0514905149\\",\\"58.969\\",\\"58.969\\",2,2,order,oliver
+5AMtOW0BH63Xcmy432HJ,ecommerce,\\"-\\",\\"-\\",\\"Men's Clothing\\",\\"Men's Clothing\\",EUR,Kamal,Kamal,\\"Kamal Salazar\\",\\"Kamal Salazar\\",MALE,39,Salazar,Salazar,\\"(empty)\\",Tuesday,1,\\"kamal@salazar-family.zzz\\",Istanbul,Asia,TR,\\"POINT (29 41)\\",Istanbul,\\"Spherecords, Spritechnologies\\",\\"Spherecords, Spritechnologies\\",\\"Jun 24, 2019 @ 00:00:00.000\\",567736,\\"sold_product_567736_24718, sold_product_567736_24306\\",\\"sold_product_567736_24718, sold_product_567736_24306\\",\\"11.992, 75\\",\\"11.992, 75\\",\\"Men's Clothing, Men's Clothing\\",\\"Men's Clothing, Men's Clothing\\",\\"Dec 13, 2016 @ 00:00:00.000, Dec 13, 2016 @ 00:00:00.000\\",\\"0, 0\\",\\"0, 0\\",\\"Spherecords, Spritechnologies\\",\\"Spherecords, Spritechnologies\\",\\"6.109, 36.75\\",\\"11.992, 75\\",\\"24,718, 24,306\\",\\"Pyjama bottoms - light grey multicolor, Waterproof trousers - scarlet\\",\\"Pyjama bottoms - light grey multicolor, Waterproof trousers - scarlet\\",\\"1, 1\\",\\"ZO0663706637, ZO0620906209\\",\\"0, 0\\",\\"11.992, 75\\",\\"11.992, 75\\",\\"0, 0\\",\\"ZO0663706637, ZO0620906209\\",87,87,2,2,order,kamal
 "
 `;