mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[ES|QL] Fix CSV report time range when exporting from Discover (#216792)
- Closes https://github.com/elastic/kibana/issues/216605 ## Summary This PR makes sure to use the absolute time range when generating a CSV report in ES|QL mode. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed
This commit is contained in:
Julia Rechkunova
GitHub
parent
8b848120d1
commit
6a0c173b1a
3 changed files with 155 additions and 50 deletions
|
|
@ -114,7 +114,17 @@ export const getShareAppMenuItem = ({
|
|||
},
|
||||
sharingData: {
|
||||
isTextBased: isEsqlMode,
|
||||
locatorParams: [{ id: locator.id, params }],
|
||||
locatorParams: [
|
||||
{
|
||||
id: locator.id,
|
||||
params: isEsqlMode
|
||||
? {
|
||||
...params,
|
||||
timeRange: timefilter.getAbsoluteTime(), // Will be used when generating CSV on server. See `filtersFromLocator`.
|
||||
}
|
||||
: params,
|
||||
},
|
||||
],
|
||||
...searchSourceSharingData,
|
||||
// CSV reports can be generated without a saved search so we provide a fallback title
|
||||
title:
|
||||
|
|
|
|||
|
|
@ -916,6 +916,25 @@ exports[`discover Discover CSV Export Generate CSV: new search generate a report
|
|||
"
|
||||
`;
|
||||
|
||||
exports[`discover Discover CSV Export Generate CSV: new search generate a report using ES|QL for relative time range as absolute dates and time params 1`] = `
|
||||
"name,numberValue
|
||||
\\"test-487\\",486
|
||||
\\"test-488\\",487
|
||||
\\"test-489\\",488
|
||||
\\"test-490\\",489
|
||||
\\"test-491\\",490
|
||||
\\"test-492\\",491
|
||||
\\"test-493\\",492
|
||||
\\"test-494\\",493
|
||||
\\"test-495\\",494
|
||||
\\"test-496\\",495
|
||||
\\"test-497\\",496
|
||||
\\"test-498\\",497
|
||||
\\"test-499\\",498
|
||||
\\"test-500\\",499
|
||||
"
|
||||
`;
|
||||
|
||||
exports[`discover Discover CSV Export Generate CSV: new search generates a large export 1`] = `
|
||||
"\\"_id\\",\\"_ignored\\",\\"_index\\",\\"_score\\",category,\\"category.keyword\\",currency,\\"customer_first_name\\",\\"customer_first_name.keyword\\",\\"customer_full_name\\",\\"customer_full_name.keyword\\",\\"customer_gender\\",\\"customer_id\\",\\"customer_last_name\\",\\"customer_last_name.keyword\\",\\"customer_phone\\",\\"day_of_week\\",\\"day_of_week_i\\",email,\\"geoip.city_name\\",\\"geoip.continent_name\\",\\"geoip.country_iso_code\\",\\"geoip.location\\",\\"geoip.region_name\\",manufacturer,\\"manufacturer.keyword\\",\\"order_date\\",\\"order_id\\",\\"products._id\\",\\"products._id.keyword\\",\\"products.base_price\\",\\"products.base_unit_price\\",\\"products.category\\",\\"products.category.keyword\\",\\"products.created_on\\",\\"products.discount_amount\\",\\"products.discount_percentage\\",\\"products.manufacturer\\",\\"products.manufacturer.keyword\\",\\"products.min_price\\",\\"products.price\\",\\"products.product_id\\",\\"products.product_name\\",\\"products.product_name.keyword\\",\\"products.quantity\\",\\"products.sku\\",\\"products.tax_amount\\",\\"products.taxful_price\\",\\"products.taxless_price\\",\\"products.unit_discount_amount\\",sku,\\"taxful_total_price\\",\\"taxless_total_price\\",\\"total_quantity\\",\\"total_unique_products\\",type,user
|
||||
3AMtOW0BH63Xcmy432DJ,\\"-\\",ecommerce,\\"-\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",EUR,\\"Sultan Al\\",\\"Sultan Al\\",\\"Sultan Al Boone\\",\\"Sultan Al Boone\\",MALE,19,Boone,Boone,\\"(empty)\\",Saturday,5,\\"sultan al@boone-family.zzz\\",\\"Abu Dhabi\\",Asia,AE,\\"POINT (54.4 24.5)\\",\\"Abu Dhabi\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Angeldale, Oceanavigations, Microlutions\\",\\"Jul 12, 2019 @ 00:00:00.000\\",716724,\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"sold_product_716724_23975, sold_product_716724_6338, sold_product_716724_14116, sold_product_716724_15290\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000, Dec 31, 2016 @ 00:00:00.000\\",\\"0, 0, 0, 0\\",\\"0, 0, 0, 0\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"Angeldale, Oceanavigations, Microlutions, Oceanavigations\\",\\"42.375, 33, 10.344, 6.109\\",\\"80, 60, 21.984, 11.992\\",\\"23,975, 6,338, 14,116, 15,290\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"Winter boots - cognac, Trenchcoat - black, Watch - black, Hat - light grey multicolor\\",\\"1, 1, 1, 1\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",\\"0, 0, 0, 0\\",\\"80, 60, 21.984, 11.992\\",\\"80, 60, 21.984, 11.992\\",\\"0, 0, 0, 0\\",\\"ZO0687606876, ZO0290502905, ZO0126701267, ZO0308503085\\",174,174,4,4,order,sultan
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import moment from 'moment';
|
||||
import moment, { DurationInputArg2 } from 'moment';
|
||||
import { Key } from 'selenium-webdriver';
|
||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||
|
||||
|
|
@ -31,6 +31,72 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
const testSubjects = getService('testSubjects');
|
||||
const toasts = getService('toasts');
|
||||
|
||||
const deleteIndex = async (index: string) => {
|
||||
try {
|
||||
await es.indices.delete({ index });
|
||||
} catch (err) {
|
||||
// ignore 404 error
|
||||
}
|
||||
};
|
||||
|
||||
const createDocs = async ({
|
||||
index,
|
||||
endDate,
|
||||
docCount,
|
||||
dateSubstractUnit,
|
||||
addNumberField,
|
||||
}: {
|
||||
index: string;
|
||||
endDate: string;
|
||||
docCount: number;
|
||||
dateSubstractUnit?: DurationInputArg2;
|
||||
addNumberField?: boolean;
|
||||
}) => {
|
||||
interface TestDoc {
|
||||
timestamp: string;
|
||||
name: string;
|
||||
updated_at?: string;
|
||||
numberValue?: number;
|
||||
}
|
||||
|
||||
const docs = Array<TestDoc>(docCount);
|
||||
|
||||
for (let i = 0; i <= docs.length - 1; i++) {
|
||||
const name = `test-${i + 1}`;
|
||||
const timestamp = moment
|
||||
.utc(endDate)
|
||||
.subtract(docCount - i, dateSubstractUnit ?? 'days')
|
||||
.format();
|
||||
|
||||
const commonFields: Pick<TestDoc, 'timestamp' | 'name' | 'numberValue'> = {
|
||||
timestamp,
|
||||
name,
|
||||
};
|
||||
|
||||
if (addNumberField) {
|
||||
commonFields.numberValue = i;
|
||||
}
|
||||
|
||||
if (i === 0) {
|
||||
// only the oldest document has a value for updated_at
|
||||
docs[i] = {
|
||||
...commonFields,
|
||||
updated_at: moment.utc(endDate).format(),
|
||||
};
|
||||
} else {
|
||||
// updated_at field does not exist in first 500 documents
|
||||
docs[i] = commonFields;
|
||||
}
|
||||
}
|
||||
|
||||
const res = await es.bulk({
|
||||
index,
|
||||
operations: docs.map((d) => `{"index": {}}\n${JSON.stringify(d)}\n`),
|
||||
});
|
||||
|
||||
log.info(`Indexed ${res.items.length} test data docs into ${index}.`);
|
||||
};
|
||||
|
||||
const getReport = async ({ timeout } = { timeout: 60 * 1000 }) => {
|
||||
// close any open notification toasts
|
||||
await toasts.dismissAll();
|
||||
|
|
@ -46,6 +112,15 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
return res;
|
||||
};
|
||||
|
||||
const getReportPostUrl = async () => {
|
||||
// click 'Copy POST URL'
|
||||
await share.clickShareTopNavButton();
|
||||
await reporting.openExportTab();
|
||||
const copyButton = await testSubjects.find('shareReportingCopyURL');
|
||||
|
||||
return decodeURIComponent((await copyButton.getAttribute('data-share-url')) ?? '');
|
||||
};
|
||||
|
||||
describe('Discover CSV Export', () => {
|
||||
describe('Check Available', () => {
|
||||
before(async () => {
|
||||
|
|
@ -189,60 +264,61 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
const csvFile = res.text;
|
||||
expectSnapshot(csvFile).toMatch();
|
||||
});
|
||||
|
||||
it('generate a report using ES|QL for relative time range as absolute dates and time params', async () => {
|
||||
const RECENT_DATA_INDEX_NAME = 'test_recent_data';
|
||||
const RECENT_DOC_COUNT = 500;
|
||||
const RECENT_DOC_END_DATE = moment().toISOString();
|
||||
|
||||
await deleteIndex(RECENT_DATA_INDEX_NAME);
|
||||
await createDocs({
|
||||
index: RECENT_DATA_INDEX_NAME,
|
||||
endDate: RECENT_DOC_END_DATE,
|
||||
docCount: RECENT_DOC_COUNT,
|
||||
dateSubstractUnit: 'minutes',
|
||||
addNumberField: true,
|
||||
});
|
||||
|
||||
await timePicker.setCommonlyUsedTime('Last_15 minutes');
|
||||
await discover.selectTextBaseLang();
|
||||
await header.waitUntilLoadingHasFinished();
|
||||
await discover.waitUntilSearchingHasFinished();
|
||||
|
||||
const testQuery = `from ${RECENT_DATA_INDEX_NAME} | sort timestamp | WHERE timestamp >= ?_tstart AND timestamp <= ?_tend | KEEP name, numberValue`;
|
||||
await monacoEditor.setCodeEditorValue(testQuery);
|
||||
await testSubjects.click('querySubmitButton');
|
||||
await header.waitUntilLoadingHasFinished();
|
||||
await discover.waitUntilSearchingHasFinished();
|
||||
|
||||
const reportPostUrl = await getReportPostUrl();
|
||||
expect(reportPostUrl).to.contain(`timeRange:(from:'2`); // not `from:now-15m`
|
||||
expect(reportPostUrl).to.contain(`filters:!()`);
|
||||
expect(reportPostUrl).to.contain(`query:(esql:'${testQuery}')`);
|
||||
|
||||
const res = await getReport();
|
||||
expect(res.status).to.equal(200);
|
||||
expect(res.get('content-type')).to.equal('text/csv; charset=utf-8');
|
||||
|
||||
const csvFile = res.text;
|
||||
expectSnapshot(csvFile).toMatch();
|
||||
|
||||
await deleteIndex(RECENT_DATA_INDEX_NAME);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Generate CSV: sparse data', () => {
|
||||
const TEST_INDEX_NAME = 'sparse_data';
|
||||
const TEST_DOC_COUNT = 510;
|
||||
|
||||
const reset = async () => {
|
||||
try {
|
||||
await es.indices.delete({ index: TEST_INDEX_NAME });
|
||||
} catch (err) {
|
||||
// ignore 404 error
|
||||
}
|
||||
};
|
||||
|
||||
const createDocs = async () => {
|
||||
interface TestDoc {
|
||||
timestamp: string;
|
||||
name: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
const docs = Array<TestDoc>(TEST_DOC_COUNT);
|
||||
|
||||
for (let i = 0; i <= docs.length - 1; i++) {
|
||||
const name = `test-${i + 1}`;
|
||||
const timestamp = moment
|
||||
.utc('2006-08-14T00:00:00')
|
||||
.subtract(TEST_DOC_COUNT - i, 'days')
|
||||
.format();
|
||||
|
||||
if (i === 0) {
|
||||
// only the oldest document has a value for updated_at
|
||||
docs[i] = {
|
||||
timestamp,
|
||||
name,
|
||||
updated_at: moment.utc('2006-08-14T00:00:00').format(),
|
||||
};
|
||||
} else {
|
||||
// updated_at field does not exist in first 500 documents
|
||||
docs[i] = { timestamp, name };
|
||||
}
|
||||
}
|
||||
|
||||
const res = await es.bulk({
|
||||
index: TEST_INDEX_NAME,
|
||||
operations: docs.map((d) => `{"index": {}}\n${JSON.stringify(d)}\n`),
|
||||
});
|
||||
|
||||
log.info(`Indexed ${res.items.length} test data docs.`);
|
||||
};
|
||||
const TEST_DOC_END_DATE = '2006-08-14T00:00:00';
|
||||
|
||||
before(async () => {
|
||||
await reset();
|
||||
await createDocs();
|
||||
await deleteIndex(TEST_INDEX_NAME);
|
||||
await createDocs({
|
||||
index: TEST_INDEX_NAME,
|
||||
endDate: TEST_DOC_END_DATE,
|
||||
docCount: TEST_DOC_COUNT,
|
||||
dateSubstractUnit: 'days',
|
||||
});
|
||||
await reportingAPI.initLogs();
|
||||
await common.navigateToApp('discover');
|
||||
await discover.loadSavedSearch('Sparse Columns');
|
||||
|
|
@ -250,7 +326,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
|||
|
||||
after(async () => {
|
||||
await reportingAPI.teardownLogs();
|
||||
await reset();
|
||||
await deleteIndex(TEST_INDEX_NAME);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue