[Discover] Show ignored field values (#115040)

* WIP replacing indexPattern.flattenHit by tabify

* Fix jest tests

* Read metaFields from index pattern

* Remove old test code

* remove unnecessary changes

* Remove flattenHitWrapper APIs

* Fix imports

* Fix missing metaFields

* Add all meta fields to allowlist

* Improve inline comments

* Move flattenHit test to new implementation

* Add deprecation comment to implementation

* WIP - Show ignored field values

* Disable filters in doc_table

* remove redundant comments

* No, it wasn't

* start warning message

* Enable ignored values in CSV reports

* Add help tooltip

* Better styling with warning plus collapsible button

* Disable filtering within table for ignored values

* Fix jest tests

* Fix types in tests

* Add more tests and documentation

* Remove comment

* Move dangerouslySetInnerHTML into helper method

* Extract document formatting into common utility

* Remove HTML source field formatter

* Move formatHit to Discover

* Change wording of ignored warning

* Add cache for formatted hits

* Remove dead type

* Fix row_formatter for objects

* Improve mobile layout

* Fix jest tests

* Fix typo

* Remove additional span again

* Change mock to revert test

* Improve tests

* More jest tests

* Fix typo

* Change wording

* Remove dead comment

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Tim Roes 2021-10-19 16:43:23 +02:00 committed by GitHub
parent 9dcf5bf1b7
commit e8663d4ea4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 857 additions and 401 deletions

View file

@ -41,6 +41,11 @@ function create(id: string) {
});
}
const meta = {
_index: 'index-name',
_id: '1',
};
describe('tabify_docs', () => {
describe('flattenHit', () => {
let indexPattern: DataView;
@ -70,6 +75,50 @@ describe('tabify_docs', () => {
expect(Object.keys(response)).toEqual(expectedOrder);
expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
});
it('does merge values from ignored_field_values and fields correctly', () => {
const flatten = flattenHit(
{
...meta,
fields: { 'extension.keyword': ['foo'], extension: ['foo', 'ignored'] },
ignored_field_values: {
'extension.keyword': ['ignored'],
fully_ignored: ['some', 'value'],
},
},
indexPattern,
{ includeIgnoredValues: true }
);
expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
expect(flatten).toHaveProperty('extension', ['foo', 'ignored']);
expect(flatten).toHaveProperty('fully_ignored', ['some', 'value']);
});
it('does not merge values from ignored_field_values into _source', () => {
const flatten = flattenHit(
{
...meta,
_source: { 'extension.keyword': ['foo', 'ignored'] },
ignored_field_values: { 'extension.keyword': ['ignored'] },
},
indexPattern,
{ includeIgnoredValues: true, source: true }
);
expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
});
it('does merge ignored_field_values when no _source was present, even when parameter was on', () => {
const flatten = flattenHit(
{
...meta,
fields: { 'extension.keyword': ['foo'] },
ignored_field_values: { 'extension.keyword': ['ignored'] },
},
indexPattern,
{ includeIgnoredValues: true, source: true }
);
expect(flatten).toHaveProperty(['extension.keyword'], ['foo', 'ignored']);
});
});
describe('tabifyDocs', () => {

View file

@ -55,8 +55,18 @@ export interface TabifyDocsOptions {
* merged into the flattened document.
*/
source?: boolean;
/**
* If set to `true` values that have been ignored in ES (ignored_field_values)
* will be merged into the flattened document. This will only have an effect if
* the `hit` has been retrieved using the `fields` option.
*/
includeIgnoredValues?: boolean;
}
// This is an overwrite of the SearchHit type to add the ignored_field_values.
// Can be removed once the estypes.SearchHit knows about ignored_field_values
type Hit<T = unknown> = estypes.SearchHit<T> & { ignored_field_values?: Record<string, unknown[]> };
/**
* Flattens an individual hit (from an ES response) into an object. This will
* create flattened field names, like `user.name`.
@ -65,11 +75,7 @@ export interface TabifyDocsOptions {
* @param indexPattern The index pattern for the requested index if available.
* @param params Parameters how to flatten the hit
*/
export function flattenHit(
hit: estypes.SearchHit,
indexPattern?: IndexPattern,
params?: TabifyDocsOptions
) {
export function flattenHit(hit: Hit, indexPattern?: IndexPattern, params?: TabifyDocsOptions) {
const flat = {} as Record<string, any>;
function flatten(obj: Record<string, any>, keyPrefix: string = '') {
@ -109,6 +115,28 @@ export function flattenHit(
flatten(hit.fields || {});
if (params?.source !== false && hit._source) {
flatten(hit._source as Record<string, any>);
} else if (params?.includeIgnoredValues && hit.ignored_field_values) {
// If enabled merge the ignored_field_values into the flattened hit. This will
// merge values that are not actually indexed by ES (i.e. ignored), e.g. because
// they were above the `ignore_above` limit or malformed for specific types.
// This API will only contain the values that were actually ignored, i.e. for the same
// field there might exist another value in the `fields` response, why this logic
// merged them both together. We do not merge this (even if enabled) in case source has been
// merged, since we would otherwise duplicate values, since ignore_field_values and _source
// contain the same values.
Object.entries(hit.ignored_field_values).forEach(([fieldName, fieldValue]) => {
if (flat[fieldName]) {
// If there was already a value from the fields API, make sure we're merging both together
if (Array.isArray(flat[fieldName])) {
flat[fieldName] = [...flat[fieldName], ...fieldValue];
} else {
flat[fieldName] = [flat[fieldName], ...fieldValue];
}
} else {
// If no previous value was assigned we can simply use the value from `ignored_field_values` as it is
flat[fieldName] = fieldValue;
}
});
}
// Merge all valid meta fields into the flattened object

View file

@ -17,7 +17,6 @@ import { DuplicateField } from '../../../kibana_utils/common';
import { IIndexPattern, IFieldType } from '../../common';
import { DataViewField, IIndexPatternFieldList, fieldList } from '../fields';
import { formatHitProvider } from './format_hit';
import { flattenHitWrapper } from './flatten_hit';
import {
FieldFormatsStartCommon,
@ -45,8 +44,6 @@ interface SavedObjectBody {
type?: string;
}
type FormatFieldFn = (hit: Record<string, any>, fieldName: string) => any;
export class DataView implements IIndexPattern {
public id?: string;
public title: string = '';
@ -67,11 +64,6 @@ export class DataView implements IIndexPattern {
* Type is used to identify rollup index patterns
*/
public type: string | undefined;
public formatHit: {
(hit: Record<string, any>, type?: string): any;
formatField: FormatFieldFn;
};
public formatField: FormatFieldFn;
/**
* @deprecated Use `flattenHit` utility method exported from data plugin instead.
*/
@ -103,11 +95,6 @@ export class DataView implements IIndexPattern {
this.fields = fieldList([], this.shortDotsEnable);
this.flattenHit = flattenHitWrapper(this, metaFields);
this.formatHit = formatHitProvider(
this,
fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING)
);
this.formatField = this.formatHit.formatField;
// set values
this.id = spec.id;

View file

@ -1,74 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import _ from 'lodash';
import { DataView } from './data_view';
import { FieldFormatsContentType } from '../../../field_formats/common';
const formattedCache = new WeakMap();
const partialFormattedCache = new WeakMap();
// Takes a hit, merges it with any stored/scripted fields, and with the metaFields
// returns a formatted version
export function formatHitProvider(dataView: DataView, defaultFormat: any) {
function convert(
hit: Record<string, any>,
val: any,
fieldName: string,
type: FieldFormatsContentType = 'html'
) {
const field = dataView.fields.getByName(fieldName);
const format = field ? dataView.getFormatterForField(field) : defaultFormat;
return format.convert(val, type, { field, hit, indexPattern: dataView });
}
function formatHit(hit: Record<string, any>, type: string = 'html') {
const cached = formattedCache.get(hit);
if (cached) {
return cached;
}
// use and update the partial cache, but don't rewrite it.
// _source is stored in partialFormattedCache but not formattedCache
const partials = partialFormattedCache.get(hit) || {};
partialFormattedCache.set(hit, partials);
const cache: Record<string, any> = {};
formattedCache.set(hit, cache);
_.forOwn(dataView.flattenHit(hit), function (val: any, fieldName?: string) {
// sync the formatted and partial cache
if (!fieldName) {
return;
}
const formatted =
partials[fieldName] == null ? convert(hit, val, fieldName) : partials[fieldName];
cache[fieldName] = partials[fieldName] = formatted;
});
return cache;
}
formatHit.formatField = function (hit: Record<string, any>, fieldName: string) {
let partials = partialFormattedCache.get(hit);
if (partials && partials[fieldName] != null) {
return partials[fieldName];
}
if (!partials) {
partials = {};
partialFormattedCache.set(hit, partials);
}
const val = fieldName === '_source' ? hit._source : dataView.flattenHit(hit)[fieldName];
return convert(hit, val, fieldName);
};
return formatHit;
}

View file

@ -8,6 +8,5 @@
export * from './_pattern_cache';
export * from './flatten_hit';
export * from './format_hit';
export * from './data_view';
export * from './data_views';

View file

@ -13,7 +13,7 @@ export {
ILLEGAL_CHARACTERS,
validateDataView,
} from '../common/lib';
export { formatHitProvider, onRedirectNoIndexPattern } from './data_views';
export { onRedirectNoIndexPattern } from './data_views';
export { IndexPatternField, IIndexPatternFieldList, TypeMeta } from '../common';

View file

@ -8,6 +8,7 @@
"data",
"embeddable",
"inspector",
"fieldFormats",
"kibanaLegacy",
"urlForwarding",
"navigation",
@ -16,7 +17,7 @@
"indexPatternFieldEditor"
],
"optionalPlugins": ["home", "share", "usageCollection", "spaces"],
"requiredBundles": ["kibanaUtils", "home", "kibanaReact", "fieldFormats", "dataViews"],
"requiredBundles": ["kibanaUtils", "home", "kibanaReact", "dataViews"],
"extraPublicDirs": ["common"],
"owner": {
"name": "Data Discovery",

View file

@ -6,8 +6,7 @@
* Side Public License, v 1.
*/
import type { estypes } from '@elastic/elasticsearch';
import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
import { IIndexPatternFieldList } from '../../../data/common';
import { IndexPattern } from '../../../data/common';
const fields = [
@ -28,6 +27,7 @@ const fields = [
{
name: 'message',
type: 'string',
displayName: 'message',
scripted: false,
filterable: false,
aggregatable: false,
@ -35,6 +35,7 @@ const fields = [
{
name: 'extension',
type: 'string',
displayName: 'extension',
scripted: false,
filterable: true,
aggregatable: true,
@ -42,6 +43,7 @@ const fields = [
{
name: 'bytes',
type: 'number',
displayName: 'bytesDisplayName',
scripted: false,
filterable: true,
aggregatable: true,
@ -49,12 +51,14 @@ const fields = [
{
name: 'scripted',
type: 'number',
displayName: 'scripted',
scripted: true,
filterable: false,
},
{
name: 'object.value',
type: 'number',
displayName: 'object.value',
scripted: false,
filterable: true,
aggregatable: true,
@ -73,23 +77,15 @@ const indexPattern = {
id: 'the-index-pattern-id',
title: 'the-index-pattern-title',
metaFields: ['_index', '_score'],
formatField: jest.fn(),
flattenHit: undefined,
formatHit: jest.fn((hit) => (hit.fields ? hit.fields : hit._source)),
fields,
getComputedFields: () => ({ docvalueFields: [], scriptFields: {}, storedFields: ['*'] }),
getSourceFiltering: () => ({}),
getFieldByName: jest.fn(() => ({})),
timeFieldName: '',
docvalueFields: [],
getFormatterForField: () => ({ convert: () => 'formatted' }),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
} as unknown as IndexPattern;
indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
return fieldName === '_source'
? hit._source
: flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
};
export const indexPatternMock = indexPattern;

View file

@ -6,9 +6,8 @@
* Side Public License, v 1.
*/
import { flattenHit, IIndexPatternFieldList } from '../../../data/common';
import { IIndexPatternFieldList } from '../../../data/common';
import { IndexPattern } from '../../../data/common';
import type { estypes } from '@elastic/elasticsearch';
const fields = [
{
@ -64,23 +63,16 @@ const indexPattern = {
id: 'index-pattern-with-timefield-id',
title: 'index-pattern-with-timefield',
metaFields: ['_index', '_score'],
flattenHit: undefined,
formatHit: jest.fn((hit) => hit._source),
fields,
getComputedFields: () => ({}),
getSourceFiltering: () => ({}),
getFieldByName: (name: string) => fields.getByName(name),
timeFieldName: 'timestamp',
getFormatterForField: () => ({ convert: () => 'formatted' }),
getFormatterForField: () => ({ convert: (value: unknown) => value }),
isTimeNanosBased: () => false,
popularizeField: () => {},
} as unknown as IndexPattern;
indexPattern.isTimeBased = () => !!indexPattern.timeFieldName;
indexPattern.formatField = (hit: Record<string, unknown>, fieldName: string) => {
return fieldName === '_source'
? hit._source
: flattenHit(hit as unknown as estypes.SearchHit, indexPattern)[fieldName];
};
export const indexPatternWithTimefieldMock = indexPattern;

View file

@ -13,6 +13,7 @@ import {
CONTEXT_STEP_SETTING,
DEFAULT_COLUMNS_SETTING,
DOC_HIDE_TIME_COLUMN_SETTING,
MAX_DOC_FIELDS_DISPLAYED,
SAMPLE_SIZE_SETTING,
SORT_DEFAULT_ORDER_SETTING,
} from '../../common';
@ -43,9 +44,13 @@ export const discoverServiceMock = {
save: true,
},
},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
filterManager: dataPlugin.query.filterManager,
uiSettings: {
get: (key: string) => {
get: jest.fn((key: string) => {
if (key === 'fields:popularLimit') {
return 5;
} else if (key === DEFAULT_COLUMNS_SETTING) {
@ -62,8 +67,10 @@ export const discoverServiceMock = {
return false;
} else if (key === SAMPLE_SIZE_SETTING) {
return 250;
} else if (key === MAX_DOC_FIELDS_DISPLAYED) {
return 50;
}
},
}),
isDefault: (key: string) => {
return true;
},

View file

@ -62,6 +62,10 @@ describe('ContextApp test', () => {
navigation: mockNavigationPlugin,
core: { notifications: { toasts: [] } },
history: () => {},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
filterManager: mockFilterManager,
uiSettings: uiSettingsMock,
} as unknown as DiscoverServices);

View file

@ -10,6 +10,7 @@ import React, { Fragment, useCallback, useMemo, useState } from 'react';
import classNames from 'classnames';
import { i18n } from '@kbn/i18n';
import { EuiButtonEmpty, EuiIcon } from '@elastic/eui';
import { formatFieldValue } from '../../../../../helpers/format_value';
import { flattenHit } from '../../../../../../../../data/common';
import { DocViewer } from '../../../../../components/doc_viewer/doc_viewer';
import { FilterManager, IndexPattern } from '../../../../../../../../data/public';
@ -58,7 +59,10 @@ export const TableRow = ({
});
const anchorDocTableRowSubj = row.isAnchor ? ' docTableAnchorRow' : '';
const flattenedRow = useMemo(() => flattenHit(row, indexPattern), [indexPattern, row]);
const flattenedRow = useMemo(
() => flattenHit(row, indexPattern, { includeIgnoredValues: true }),
[indexPattern, row]
);
const mapping = useMemo(() => indexPattern.fields.getByName, [indexPattern]);
// toggle display of the rows details, a full list of the fields from each row
@ -68,13 +72,24 @@ export const TableRow = ({
* Fill an element with the value of a field
*/
const displayField = (fieldName: string) => {
const formattedField = indexPattern.formatField(row, fieldName);
// If we're formatting the _source column, don't use the regular field formatter,
// but our Discover mechanism to format a hit in a better human-readable way.
if (fieldName === '_source') {
return formatRow(row, indexPattern, fieldsToShow);
}
// field formatters take care of escaping
// eslint-disable-next-line react/no-danger
const fieldElement = <span dangerouslySetInnerHTML={{ __html: formattedField }} />;
const formattedField = formatFieldValue(
flattenedRow[fieldName],
row,
indexPattern,
mapping(fieldName)
);
return <div className="truncate-by-height">{fieldElement}</div>;
return (
// formatFieldValue always returns sanitized HTML
// eslint-disable-next-line react/no-danger
<div className="truncate-by-height" dangerouslySetInnerHTML={{ __html: formattedField }} />
);
};
const inlineFilter = useCallback(
(column: string, type: '+' | '-') => {
@ -141,10 +156,9 @@ export const TableRow = ({
);
} else {
columns.forEach(function (column: string) {
// when useNewFieldsApi is true, addressing to the fields property is safe
if (useNewFieldsApi && !mapping(column) && !row.fields![column]) {
if (useNewFieldsApi && !mapping(column) && row.fields && !row.fields[column]) {
const innerColumns = Object.fromEntries(
Object.entries(row.fields!).filter(([key]) => {
Object.entries(row.fields).filter(([key]) => {
return key.indexOf(`${column}.`) === 0;
})
);
@ -161,7 +175,13 @@ export const TableRow = ({
/>
);
} else {
const isFilterable = Boolean(mapping(column)?.filterable && filter);
// Check whether the field is defined as filterable in the mapping and does
// NOT have ignored values in it to determine whether we want to allow filtering.
// We should improve this and show a helpful tooltip why the filter buttons are not
// there/disabled when there are ignored values.
const isFilterable = Boolean(
mapping(column)?.filterable && filter && !row._ignored?.includes(column)
);
rowCells.push(
<TableCell
key={column}

View file

@ -17,6 +17,7 @@ import { stubbedSavedObjectIndexPattern } from '../../../../../../../../data/com
describe('Row formatter', () => {
const hit = {
_id: 'a',
_index: 'foo',
_type: 'doc',
_score: 1,
_source: {
@ -39,7 +40,7 @@ describe('Row formatter', () => {
spec: { id, type, version, timeFieldName, fields: JSON.parse(fields), title },
fieldFormats: fieldFormatsMock,
shortDotsEnable: false,
metaFields: [],
metaFields: ['_id', '_type', '_score'],
});
};
@ -47,26 +48,15 @@ describe('Row formatter', () => {
const fieldsToShow = indexPattern.fields.getAll().map((fld) => fld.name);
// Realistic response with alphabetical insertion order
const formatHitReturnValue = {
also: 'with \\&quot;quotes\\&quot; or &#39;single qoutes&#39;',
foo: 'bar',
number: '42',
hello: '&lt;h1&gt;World&lt;/h1&gt;',
_id: 'a',
_type: 'doc',
_score: 1,
};
const formatHitMock = jest.fn().mockReturnValue(formatHitReturnValue);
beforeEach(() => {
// @ts-expect-error
indexPattern.formatHit = formatHitMock;
setServices({
uiSettings: {
get: () => 100,
},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
} as unknown as DiscoverServices);
});
@ -77,32 +67,32 @@ describe('Row formatter', () => {
Array [
Array [
"also",
"with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;",
"with \\"quotes\\" or 'single quotes'",
],
Array [
"foo",
"bar",
],
Array [
"number",
"42",
"hello",
"<h1>World</h1>",
],
Array [
"hello",
"&lt;h1&gt;World&lt;/h1&gt;",
"number",
42,
],
Array [
"_id",
"a",
],
Array [
"_type",
"doc",
],
Array [
"_score",
1,
],
Array [
"_type",
"doc",
],
]
}
/>
@ -114,6 +104,10 @@ describe('Row formatter', () => {
uiSettings: {
get: () => 1,
},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
} as unknown as DiscoverServices);
expect(formatRow(hit, indexPattern, [])).toMatchInlineSnapshot(`
<TemplateComponent
@ -121,7 +115,31 @@ describe('Row formatter', () => {
Array [
Array [
"also",
"with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;",
"with \\"quotes\\" or 'single quotes'",
],
Array [
"foo",
"bar",
],
Array [
"hello",
"<h1>World</h1>",
],
Array [
"number",
42,
],
Array [
"_id",
"a",
],
Array [
"_score",
1,
],
Array [
"_type",
"doc",
],
]
}
@ -130,18 +148,18 @@ describe('Row formatter', () => {
});
it('formats document with highlighted fields first', () => {
expect(formatRow({ ...hit, highlight: { number: '42' } }, indexPattern, fieldsToShow))
expect(formatRow({ ...hit, highlight: { number: ['42'] } }, indexPattern, fieldsToShow))
.toMatchInlineSnapshot(`
<TemplateComponent
defPairs={
Array [
Array [
"number",
"42",
42,
],
Array [
"also",
"with \\\\&quot;quotes\\\\&quot; or &#39;single qoutes&#39;",
"with \\"quotes\\" or 'single quotes'",
],
Array [
"foo",
@ -149,20 +167,20 @@ describe('Row formatter', () => {
],
Array [
"hello",
"&lt;h1&gt;World&lt;/h1&gt;",
"<h1>World</h1>",
],
Array [
"_id",
"a",
],
Array [
"_type",
"doc",
],
Array [
"_score",
1,
],
Array [
"_type",
"doc",
],
]
}
/>

View file

@ -6,15 +6,17 @@
* Side Public License, v 1.
*/
import { estypes } from '@elastic/elasticsearch';
import React, { Fragment } from 'react';
import type { IndexPattern } from 'src/plugins/data/common';
import { MAX_DOC_FIELDS_DISPLAYED } from '../../../../../../../common';
import { getServices } from '../../../../../../kibana_services';
import { formatHit } from '../../../../../helpers/format_hit';
import './row_formatter.scss';
interface Props {
defPairs: Array<[string, unknown]>;
defPairs: Array<[string, string]>;
}
const TemplateComponent = ({ defPairs }: Props) => {
return (
@ -24,8 +26,8 @@ const TemplateComponent = ({ defPairs }: Props) => {
<dt>{pair[0]}:</dt>
<dd
className="rowFormatter__value"
// We can dangerously set HTML here because this content is guaranteed to have been run through a valid field formatter first.
dangerouslySetInnerHTML={{ __html: `${pair[1]}` }} // eslint-disable-line react/no-danger
// eslint-disable-next-line react/no-danger
dangerouslySetInnerHTML={{ __html: pair[1] }}
/>{' '}
</Fragment>
))}
@ -34,30 +36,12 @@ const TemplateComponent = ({ defPairs }: Props) => {
};
export const formatRow = (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
hit: Record<string, any>,
hit: estypes.SearchHit,
indexPattern: IndexPattern,
fieldsToShow: string[]
) => {
const highlights = hit?.highlight ?? {};
// Keys are sorted in the hits object
const formatted = indexPattern.formatHit(hit);
const fields = indexPattern.fields;
const highlightPairs: Array<[string, unknown]> = [];
const sourcePairs: Array<[string, unknown]> = [];
Object.entries(formatted).forEach(([key, val]) => {
const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined;
const pairs = highlights[key] ? highlightPairs : sourcePairs;
if (displayKey) {
if (fieldsToShow.includes(displayKey)) {
pairs.push([displayKey, val]);
}
} else {
pairs.push([key, val]);
}
});
const maxEntries = getServices().uiSettings.get(MAX_DOC_FIELDS_DISPLAYED);
return <TemplateComponent defPairs={[...highlightPairs, ...sourcePairs].slice(0, maxEntries)} />;
const pairs = formatHit(hit, indexPattern, fieldsToShow);
return <TemplateComponent defPairs={pairs} />;
};
export const formatTopLevelObject = (
@ -68,8 +52,8 @@ export const formatTopLevelObject = (
indexPattern: IndexPattern
) => {
const highlights = row.highlight ?? {};
const highlightPairs: Array<[string, unknown]> = [];
const sourcePairs: Array<[string, unknown]> = [];
const highlightPairs: Array<[string, string]> = [];
const sourcePairs: Array<[string, string]> = [];
const sorted = Object.entries(fields).sort(([keyA], [keyB]) => keyA.localeCompare(keyB));
sorted.forEach(([key, values]) => {
const field = indexPattern.getFieldByName(key);

View file

@ -20,6 +20,11 @@ import { DiscoverDocuments } from './discover_documents';
import { ElasticSearchHit } from '../../../../doc_views/doc_views_types';
import { indexPatternMock } from '../../../../../__mocks__/index_pattern';
jest.mock('../../../../../kibana_services', () => ({
...jest.requireActual('../../../../../kibana_services'),
getServices: () => jest.requireActual('../../../../../__mocks__/services').discoverServiceMock,
}));
setHeaderActionMenuMounter(jest.fn());
function getProps(fetchStatus: FetchStatus, hits: ElasticSearchHit[]) {

View file

@ -33,6 +33,19 @@ import { RequestAdapter } from '../../../../../../../inspector';
import { Chart } from '../chart/point_series';
import { DiscoverSidebar } from '../sidebar/discover_sidebar';
jest.mock('../../../../../kibana_services', () => ({
...jest.requireActual('../../../../../kibana_services'),
getServices: () => ({
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
uiSettings: {
get: jest.fn((key: string) => key === 'discover:maxDocFieldsDisplayed' && 50),
},
}),
}));
setHeaderActionMenuMounter(jest.fn());
function getProps(indexPattern: IndexPattern, wasSidebarClosed?: boolean): DiscoverLayoutProps {

View file

@ -115,42 +115,7 @@ exports[`Discover IndexPattern Management renders correctly 1`] = `
"deserialize": [MockFunction],
"getByFieldType": [MockFunction],
"getDefaultConfig": [MockFunction],
"getDefaultInstance": [MockFunction] {
"calls": Array [
Array [
"string",
],
Array [
"string",
],
Array [
"string",
],
],
"results": Array [
Object {
"type": "return",
"value": Object {
"convert": [MockFunction],
"getConverterFor": [MockFunction],
},
},
Object {
"type": "return",
"value": Object {
"convert": [MockFunction],
"getConverterFor": [MockFunction],
},
},
Object {
"type": "return",
"value": Object {
"convert": [MockFunction],
"getConverterFor": [MockFunction],
},
},
],
},
"getDefaultInstance": [MockFunction],
"getDefaultInstanceCacheResolver": [MockFunction],
"getDefaultInstancePlain": [MockFunction],
"getDefaultType": [MockFunction],
@ -651,8 +616,6 @@ exports[`Discover IndexPattern Management renders correctly 1`] = `
},
],
"flattenHit": [Function],
"formatField": [Function],
"formatHit": [Function],
"getFieldAttrs": [Function],
"getOriginalSavedObjectBody": [Function],
"id": "logstash-*",

View file

@ -13,7 +13,7 @@ import { flattenHit } from '../../../../../../../../data/common';
function getFieldValues(hits, field, indexPattern) {
const name = field.name;
return map(hits, function (hit) {
return flattenHit(hit, indexPattern)[name];
return flattenHit(hit, indexPattern, { includeIgnoredValues: true })[name];
});
}

View file

@ -22,7 +22,7 @@ export function calcFieldCounts(
return {};
}
for (const hit of rows) {
const fields = Object.keys(flattenHit(hit, indexPattern));
const fields = Object.keys(flattenHit(hit, indexPattern, { includeIgnoredValues: true }));
for (const fieldName of fields) {
counts[fieldName] = (counts[fieldName] || 0) + 1;
}

View file

@ -19,6 +19,11 @@ import { DiscoverServices } from '../../../build_services';
import { ElasticSearchHit } from '../../doc_views/doc_views_types';
import { getDocId } from './discover_grid_document_selection';
jest.mock('../../../kibana_services', () => ({
...jest.requireActual('../../../kibana_services'),
getServices: () => jest.requireActual('../../../__mocks__/services').discoverServiceMock,
}));
function getProps() {
const servicesMock = {
uiSettings: uiSettingsMock,

View file

@ -271,7 +271,11 @@ export const DiscoverGrid = ({
getRenderCellValueFn(
indexPattern,
displayedRows,
displayedRows ? displayedRows.map((hit) => flattenHit(hit, indexPattern)) : [],
displayedRows
? displayedRows.map((hit) =>
flattenHit(hit, indexPattern, { includeIgnoredValues: true })
)
: [],
useNewFieldsApi,
fieldsToShow,
services.uiSettings.get(MAX_DOC_FIELDS_DISPLAYED)

View file

@ -25,6 +25,9 @@ jest.mock('../../../kibana_services', () => ({
uiSettings: {
get: jest.fn(),
},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => (value ? value : '-') })),
},
}),
}));
@ -102,7 +105,7 @@ describe('Discover grid cell rendering', function () {
rowsSource,
rowsSource.map(flatten),
false,
[],
['extension', 'bytes'],
100
);
const component = shallow(
@ -133,7 +136,7 @@ describe('Discover grid cell rendering', function () {
}
/>
<EuiDescriptionListTitle>
bytes
bytesDisplayName
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
@ -143,6 +146,28 @@ describe('Discover grid cell rendering', function () {
}
}
/>
<EuiDescriptionListTitle>
_index
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": "test",
}
}
/>
<EuiDescriptionListTitle>
_score
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": 1,
}
}
/>
</EuiDescriptionList>
`);
});
@ -196,7 +221,7 @@ describe('Discover grid cell rendering', function () {
rowsFields,
rowsFields.map(flatten),
true,
[],
['extension', 'bytes'],
100
);
const component = shallow(
@ -229,7 +254,7 @@ describe('Discover grid cell rendering', function () {
}
/>
<EuiDescriptionListTitle>
bytes
bytesDisplayName
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
@ -241,6 +266,28 @@ describe('Discover grid cell rendering', function () {
}
}
/>
<EuiDescriptionListTitle>
_index
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": "test",
}
}
/>
<EuiDescriptionListTitle>
_score
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": 1,
}
}
/>
</EuiDescriptionList>
`);
});
@ -251,7 +298,7 @@ describe('Discover grid cell rendering', function () {
rowsFields,
rowsFields.map(flatten),
true,
[],
['extension', 'bytes'],
// this is the number of rendered items
1
);
@ -284,6 +331,41 @@ describe('Discover grid cell rendering', function () {
}
}
/>
<EuiDescriptionListTitle>
bytesDisplayName
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": Array [
100,
],
}
}
/>
<EuiDescriptionListTitle>
_index
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": "test",
}
}
/>
<EuiDescriptionListTitle>
_score
</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": 1,
}
}
/>
</EuiDescriptionList>
`);
});
@ -342,7 +424,7 @@ describe('Discover grid cell rendering', function () {
rowsFieldsWithTopLevelObject,
rowsFieldsWithTopLevelObject.map(flatten),
true,
[],
['object.value', 'extension', 'bytes'],
100
);
const component = shallow(
@ -368,7 +450,7 @@ describe('Discover grid cell rendering', function () {
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={
Object {
"__html": "formatted",
"__html": "100",
}
}
/>
@ -383,7 +465,7 @@ describe('Discover grid cell rendering', function () {
rowsFieldsWithTopLevelObject,
rowsFieldsWithTopLevelObject.map(flatten),
true,
[],
['extension', 'bytes', 'object.value'],
100
);
const component = shallow(

View file

@ -22,6 +22,8 @@ import { DiscoverGridContext } from './discover_grid_context';
import { JsonCodeEditor } from '../json_code_editor/json_code_editor';
import { defaultMonacoEditorWidth } from './constants';
import { EsHitRecord } from '../../types';
import { formatFieldValue } from '../../helpers/format_value';
import { formatHit } from '../../helpers/format_hit';
export const getRenderCellValueFn =
(
@ -145,39 +147,19 @@ export const getRenderCellValueFn =
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return <JsonCodeEditor json={row as any} width={defaultMonacoEditorWidth} />;
}
const formatted = indexPattern.formatHit(row);
// Put the most important fields first
const highlights: Record<string, unknown> = (row.highlight as Record<string, unknown>) ?? {};
const highlightPairs: Array<[string, string]> = [];
const sourcePairs: Array<[string, string]> = [];
Object.entries(formatted).forEach(([key, val]) => {
const pairs = highlights[key] ? highlightPairs : sourcePairs;
const displayKey = indexPattern.fields.getByName
? indexPattern.fields.getByName(key)?.displayName
: undefined;
if (displayKey) {
if (fieldsToShow.includes(displayKey)) {
pairs.push([displayKey, val as string]);
}
} else {
pairs.push([key, val as string]);
}
});
const pairs = formatHit(row, indexPattern, fieldsToShow);
return (
<EuiDescriptionList type="inline" compressed className="dscDiscoverGrid__descriptionList">
{[...highlightPairs, ...sourcePairs]
.slice(0, maxDocFieldsDisplayed)
.map(([key, value]) => (
<Fragment key={key}>
<EuiDescriptionListTitle>{key}</EuiDescriptionListTitle>
<EuiDescriptionListDescription
dangerouslySetInnerHTML={{ __html: value }}
className="dscDiscoverGrid__descriptionListDescription"
/>
</Fragment>
))}
{pairs.map(([key, value]) => (
<Fragment key={key}>
<EuiDescriptionListTitle>{key}</EuiDescriptionListTitle>
<EuiDescriptionListDescription
className="dscDiscoverGrid__descriptionListDescription"
dangerouslySetInnerHTML={{ __html: value }}
/>
</Fragment>
))}
</EuiDescriptionList>
);
}
@ -191,12 +173,13 @@ export const getRenderCellValueFn =
return <span>{JSON.stringify(rowFlattened[columnId])}</span>;
}
const valueFormatted = indexPattern.formatField(row, columnId);
if (typeof valueFormatted === 'undefined') {
return <span>-</span>;
}
return (
// eslint-disable-next-line react/no-danger
<span dangerouslySetInnerHTML={{ __html: indexPattern.formatField(row, columnId) }} />
<span
// formatFieldValue guarantees sanitized values
// eslint-disable-next-line react/no-danger
dangerouslySetInnerHTML={{
__html: formatFieldValue(rowFlattened[columnId], row, indexPattern, field),
}}
/>
);
};

View file

@ -27,6 +27,10 @@ import { getServices } from '../../../kibana_services';
}
},
},
fieldFormats: {
getDefaultInstance: jest.fn(() => ({ convert: (value: unknown) => value })),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
},
}));
const indexPattern = {
@ -65,8 +69,7 @@ const indexPattern = {
],
},
metaFields: ['_index', '_score'],
flattenHit: jest.fn(),
formatHit: jest.fn((hit) => hit._source),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
} as unknown as IndexPattern;
indexPattern.fields.getByName = (name: string) => {
@ -359,32 +362,7 @@ describe('DocViewTable at Discover Doc with Fields API', () => {
],
},
metaFields: ['_index', '_type', '_score', '_id'],
flattenHit: jest.fn((hit) => {
const result = {} as Record<string, unknown>;
Object.keys(hit).forEach((key) => {
if (key !== 'fields') {
result[key] = hit[key];
} else {
Object.keys(hit.fields).forEach((field) => {
result[field] = hit.fields[field];
});
}
});
return result;
}),
formatHit: jest.fn((hit) => {
const result = {} as Record<string, unknown>;
Object.keys(hit).forEach((key) => {
if (key !== 'fields') {
result[key] = hit[key];
} else {
Object.keys(hit.fields).forEach((field) => {
result[field] = hit.fields[field];
});
}
});
return result;
}),
getFormatterForField: jest.fn(() => ({ convert: (value: unknown) => value })),
} as unknown as IndexPattern;
indexPatterneCommerce.fields.getByName = (name: string) => {

View file

@ -20,6 +20,8 @@ import {
} from '../../doc_views/doc_views_types';
import { ACTIONS_COLUMN, MAIN_COLUMNS } from './table_columns';
import { getFieldsToShow } from '../../helpers/get_fields_to_show';
import { getIgnoredReason, IgnoredReason } from '../../helpers/get_ignored_reason';
import { formatFieldValue } from '../../helpers/format_value';
export interface DocViewerTableProps {
columns?: string[];
@ -46,6 +48,7 @@ export interface FieldRecord {
};
value: {
formattedValue: string;
ignored?: IgnoredReason;
};
}
@ -64,8 +67,6 @@ export const DocViewerTable = ({
[indexPattern?.fields]
);
const formattedHit = useMemo(() => indexPattern?.formatHit(hit, 'html'), [hit, indexPattern]);
const tableColumns = useMemo(() => {
return filter ? [ACTIONS_COLUMN, ...MAIN_COLUMNS] : MAIN_COLUMNS;
}, [filter]);
@ -96,7 +97,7 @@ export const DocViewerTable = ({
return null;
}
const flattened = flattenHit(hit, indexPattern, { source: true });
const flattened = flattenHit(hit, indexPattern, { source: true, includeIgnoredValues: true });
const fieldsToShow = getFieldsToShow(Object.keys(flattened), indexPattern, showMultiFields);
const items: FieldRecord[] = Object.keys(flattened)
@ -115,6 +116,8 @@ export const DocViewerTable = ({
const displayName = fieldMapping?.displayName ?? field;
const fieldType = isNestedFieldParent(field, indexPattern) ? 'nested' : fieldMapping?.type;
const ignored = getIgnoredReason(fieldMapping ?? field, hit._ignored);
return {
action: {
onToggleColumn,
@ -130,7 +133,8 @@ export const DocViewerTable = ({
scripted: Boolean(fieldMapping?.scripted),
},
value: {
formattedValue: formattedHit[field],
formattedValue: formatFieldValue(flattened[field], hit, indexPattern, fieldMapping),
ignored,
},
};
});

View file

@ -21,6 +21,7 @@ interface TableActionsProps {
fieldMapping?: IndexPatternField;
onFilter: DocViewFilterFn;
onToggleColumn: (field: string) => void;
ignoredValue: boolean;
}
export const TableActions = ({
@ -30,15 +31,16 @@ export const TableActions = ({
flattenedField,
onToggleColumn,
onFilter,
ignoredValue,
}: TableActionsProps) => {
return (
<div className="kbnDocViewer__buttons">
<DocViewTableRowBtnFilterAdd
disabled={!fieldMapping || !fieldMapping.filterable}
disabled={!fieldMapping || !fieldMapping.filterable || ignoredValue}
onClick={() => onFilter(fieldMapping, flattenedField, '+')}
/>
<DocViewTableRowBtnFilterRemove
disabled={!fieldMapping || !fieldMapping.filterable}
disabled={!fieldMapping || !fieldMapping.filterable || ignoredValue}
onClick={() => onFilter(fieldMapping, flattenedField, '-')}
/>
<DocViewTableRowBtnToggleColumn

View file

@ -6,19 +6,98 @@
* Side Public License, v 1.
*/
import { css } from '@emotion/react';
import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiTextColor, EuiToolTip } from '@elastic/eui';
import classNames from 'classnames';
import React, { Fragment, useState } from 'react';
import { i18n } from '@kbn/i18n';
import { IgnoredReason } from '../../helpers/get_ignored_reason';
import { FieldRecord } from './table';
import { DocViewTableRowBtnCollapse } from './table_row_btn_collapse';
const COLLAPSE_LINE_LENGTH = 350;
type TableFieldValueProps = FieldRecord['value'] & Pick<FieldRecord['field'], 'field'>;
interface IgnoreWarningProps {
reason: IgnoredReason;
rawValue: unknown;
}
export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps) => {
const IgnoreWarning: React.FC<IgnoreWarningProps> = React.memo(({ rawValue, reason }) => {
const multiValue = Array.isArray(rawValue) && rawValue.length > 1;
const getToolTipContent = (): string => {
switch (reason) {
case IgnoredReason.IGNORE_ABOVE:
return multiValue
? i18n.translate('discover.docView.table.ignored.multiAboveTooltip', {
defaultMessage: `One or more values in this field are too long and can't be searched or filtered.`,
})
: i18n.translate('discover.docView.table.ignored.singleAboveTooltip', {
defaultMessage: `The value in this field is too long and can't be searched or filtered.`,
});
case IgnoredReason.MALFORMED:
return multiValue
? i18n.translate('discover.docView.table.ignored.multiMalformedTooltip', {
defaultMessage: `This field has one or more malformed values that can't be searched or filtered.`,
})
: i18n.translate('discover.docView.table.ignored.singleMalformedTooltip', {
defaultMessage: `The value in this field is malformed and can't be searched or filtered.`,
});
case IgnoredReason.UNKNOWN:
return multiValue
? i18n.translate('discover.docView.table.ignored.multiUnknownTooltip', {
defaultMessage: `One or more values in this field were ignored by Elasticsearch and can't be searched or filtered.`,
})
: i18n.translate('discover.docView.table.ignored.singleUnknownTooltip', {
defaultMessage: `The value in this field was ignored by Elasticsearch and can't be searched or filtered.`,
});
}
};
return (
<EuiToolTip content={getToolTipContent()}>
<EuiFlexGroup
gutterSize="xs"
responsive={false}
alignItems="center"
css={css`
cursor: help;
`}
>
<EuiFlexItem grow={false}>
<EuiIcon type="alert" color="warning" />
</EuiFlexItem>
<EuiFlexItem>
<EuiTextColor color="warning">
{multiValue
? i18n.translate('discover.docViews.table.ignored.multiValueLabel', {
defaultMessage: 'Contains ignored values',
})
: i18n.translate('discover.docViews.table.ignored.singleValueLabel', {
defaultMessage: 'Ignored value',
})}
</EuiTextColor>
</EuiFlexItem>
</EuiFlexGroup>
</EuiToolTip>
);
});
type TableFieldValueProps = Pick<FieldRecord['field'], 'field'> & {
formattedValue: FieldRecord['value']['formattedValue'];
rawValue: unknown;
ignoreReason?: IgnoredReason;
};
export const TableFieldValue = ({
formattedValue,
field,
rawValue,
ignoreReason,
}: TableFieldValueProps) => {
const [fieldOpen, setFieldOpen] = useState(false);
const value = String(formattedValue);
const value = String(rawValue);
const isCollapsible = value.length > COLLAPSE_LINE_LENGTH;
const isCollapsed = isCollapsible && !fieldOpen;
@ -32,18 +111,26 @@ export const TableFieldValue = ({ formattedValue, field }: TableFieldValueProps)
return (
<Fragment>
{isCollapsible && (
<DocViewTableRowBtnCollapse onClick={onToggleCollapse} isCollapsed={isCollapsed} />
{(isCollapsible || ignoreReason) && (
<EuiFlexGroup gutterSize="s">
{isCollapsible && (
<EuiFlexItem grow={false}>
<DocViewTableRowBtnCollapse onClick={onToggleCollapse} isCollapsed={isCollapsed} />
</EuiFlexItem>
)}
{ignoreReason && (
<EuiFlexItem grow={false}>
<IgnoreWarning reason={ignoreReason} rawValue={rawValue} />
</EuiFlexItem>
)}
</EuiFlexGroup>
)}
<div
className={valueClassName}
data-test-subj={`tableDocViewRow-${field}-value`}
/*
* Justification for dangerouslySetInnerHTML:
* We just use values encoded by our field formatters
*/
// Value returned from formatFieldValue is always sanitized
// eslint-disable-next-line react/no-danger
dangerouslySetInnerHTML={{ __html: value }}
dangerouslySetInnerHTML={{ __html: formattedValue }}
/>
</Fragment>
);

View file

@ -31,7 +31,7 @@ export const ACTIONS_COLUMN: EuiBasicTableColumn<FieldRecord> = {
),
render: (
{ flattenedField, isActive, onFilter, onToggleColumn }: FieldRecord['action'],
{ field: { field, fieldMapping } }: FieldRecord
{ field: { field, fieldMapping }, value: { ignored } }: FieldRecord
) => {
return (
<TableActions
@ -41,6 +41,7 @@ export const ACTIONS_COLUMN: EuiBasicTableColumn<FieldRecord> = {
flattenedField={flattenedField}
onFilter={onFilter!}
onToggleColumn={onToggleColumn}
ignoredValue={!!ignored}
/>
);
},
@ -82,8 +83,18 @@ export const MAIN_COLUMNS: Array<EuiBasicTableColumn<FieldRecord>> = [
</strong>
</EuiText>
),
render: ({ formattedValue }: FieldRecord['value'], { field: { field } }: FieldRecord) => {
return <TableFieldValue field={field} formattedValue={formattedValue} />;
render: (
{ formattedValue, ignored }: FieldRecord['value'],
{ field: { field }, action: { flattenedField } }: FieldRecord
) => {
return (
<TableFieldValue
field={field}
formattedValue={formattedValue}
rawValue={flattenedField}
ignoreReason={ignored}
/>
);
},
},
];

View file

@ -20,7 +20,7 @@ export function DocViewTableRowBtnFilterAdd({ onClick, disabled = false }: Props
const tooltipContent = disabled ? (
<FormattedMessage
id="discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip"
defaultMessage="Unindexed fields can not be searched"
defaultMessage="Unindexed fields or ignored values cannot be searched"
/>
) : (
<FormattedMessage

View file

@ -20,7 +20,7 @@ export function DocViewTableRowBtnFilterRemove({ onClick, disabled = false }: Pr
const tooltipContent = disabled ? (
<FormattedMessage
id="discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip"
defaultMessage="Unindexed fields can not be searched"
defaultMessage="Unindexed fields or ignored values cannot be searched"
/>
) : (
<FormattedMessage

View file

@ -0,0 +1,96 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { estypes } from '@elastic/elasticsearch';
import { indexPatternMock as dataViewMock } from '../../__mocks__/index_pattern';
import { formatHit } from './format_hit';
import { discoverServiceMock } from '../../__mocks__/services';
import { MAX_DOC_FIELDS_DISPLAYED } from '../../../common';
jest.mock('../../kibana_services', () => ({
getServices: () => jest.requireActual('../../__mocks__/services').discoverServiceMock,
}));
describe('formatHit', () => {
let hit: estypes.SearchHit;
beforeEach(() => {
hit = {
_id: '1',
_index: 'logs',
fields: {
message: ['foobar'],
extension: ['png'],
'object.value': [42, 13],
bytes: [123],
},
};
(dataViewMock.getFormatterForField as jest.Mock).mockReturnValue({
convert: (value: unknown) => `formatted:${value}`,
});
});
afterEach(() => {
(discoverServiceMock.uiSettings.get as jest.Mock).mockReset();
});
it('formats a document as expected', () => {
const formatted = formatHit(hit, dataViewMock, ['message', 'extension', 'object.value']);
expect(formatted).toEqual([
['extension', 'formatted:png'],
['message', 'formatted:foobar'],
['object.value', 'formatted:42,13'],
['_index', 'formatted:logs'],
['_score', undefined],
]);
});
it('orders highlighted fields first', () => {
const formatted = formatHit({ ...hit, highlight: { message: ['%%'] } }, dataViewMock, [
'message',
'extension',
'object.value',
]);
expect(formatted.map(([fieldName]) => fieldName)).toEqual([
'message',
'extension',
'object.value',
'_index',
'_score',
]);
});
it('only limits count of pairs based on advanced setting', () => {
(discoverServiceMock.uiSettings.get as jest.Mock).mockImplementation(
(key) => key === MAX_DOC_FIELDS_DISPLAYED && 2
);
const formatted = formatHit(hit, dataViewMock, ['message', 'extension', 'object.value']);
expect(formatted).toEqual([
['extension', 'formatted:png'],
['message', 'formatted:foobar'],
]);
});
it('should not include fields not mentioned in fieldsToShow', () => {
const formatted = formatHit(hit, dataViewMock, ['message', 'object.value']);
expect(formatted).toEqual([
['message', 'formatted:foobar'],
['object.value', 'formatted:42,13'],
['_index', 'formatted:logs'],
['_score', undefined],
]);
});
it('should filter fields based on their real name not displayName', () => {
const formatted = formatHit(hit, dataViewMock, ['bytes']);
expect(formatted).toEqual([
['bytesDisplayName', 'formatted:123'],
['_index', 'formatted:logs'],
['_score', undefined],
]);
});
});

View file

@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { estypes } from '@elastic/elasticsearch';
import { DataView, flattenHit } from '../../../../data/common';
import { MAX_DOC_FIELDS_DISPLAYED } from '../../../common';
import { getServices } from '../../kibana_services';
import { formatFieldValue } from './format_value';
const formattedHitCache = new WeakMap<estypes.SearchHit, FormattedHit>();
type FormattedHit = Array<[fieldName: string, formattedValue: string]>;
/**
* Returns a formatted document in form of key/value pairs of the fields name and a formatted value.
* The value returned in each pair is an HTML string which is safe to be applied to the DOM, since
* it's formatted using field formatters.
* @param hit The hit to format
* @param dataView The corresponding data view
* @param fieldsToShow A list of fields that should be included in the document summary.
*/
export function formatHit(
hit: estypes.SearchHit,
dataView: DataView,
fieldsToShow: string[]
): FormattedHit {
const cached = formattedHitCache.get(hit);
if (cached) {
return cached;
}
const highlights = hit.highlight ?? {};
// Flatten the object using the flattenHit implementation we use across Discover for flattening documents.
const flattened = flattenHit(hit, dataView, { includeIgnoredValues: true, source: true });
const highlightPairs: Array<[fieldName: string, formattedValue: string]> = [];
const sourcePairs: Array<[fieldName: string, formattedValue: string]> = [];
// Add each flattened field into the corresponding array for highlighted or other fields,
// depending on whether the original hit had a highlight for it. That way we can later
// put highlighted fields first in the document summary.
Object.entries(flattened).forEach(([key, val]) => {
// Retrieve the (display) name of the fields, if it's a mapped field on the data view
const displayKey = dataView.fields.getByName(key)?.displayName;
const pairs = highlights[key] ? highlightPairs : sourcePairs;
// Format the raw value using the regular field formatters for that field
const formattedValue = formatFieldValue(val, hit, dataView, dataView.fields.getByName(key));
// If the field was a mapped field, we validate it against the fieldsToShow list, if not
// we always include it into the result.
if (displayKey) {
if (fieldsToShow.includes(key)) {
pairs.push([displayKey, formattedValue]);
}
} else {
pairs.push([key, formattedValue]);
}
});
const maxEntries = getServices().uiSettings.get<number>(MAX_DOC_FIELDS_DISPLAYED);
const formatted = [...highlightPairs, ...sourcePairs].slice(0, maxEntries);
formattedHitCache.set(hit, formatted);
return formatted;
}

View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { FieldFormat } from '../../../../field_formats/common';
import { indexPatternMock } from '../../__mocks__/index_pattern';
import { formatFieldValue } from './format_value';
import { getServices } from '../../kibana_services';
jest.mock('../../kibana_services', () => {
const services = {
fieldFormats: {
getDefaultInstance: jest.fn<FieldFormat, [string]>(
() => ({ convert: (value: unknown) => value } as FieldFormat)
),
},
};
return { getServices: () => services };
});
const hit = {
_id: '1',
_index: 'index',
fields: {
message: 'foo',
},
};
describe('formatFieldValue', () => {
afterEach(() => {
(indexPatternMock.getFormatterForField as jest.Mock).mockReset();
(getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReset();
});
it('should call correct fieldFormatter for field', () => {
const formatterForFieldMock = indexPatternMock.getFormatterForField as jest.Mock;
const convertMock = jest.fn((value: unknown) => `formatted:${value}`);
formatterForFieldMock.mockReturnValue({ convert: convertMock });
const field = indexPatternMock.fields.getByName('message');
expect(formatFieldValue('foo', hit, indexPatternMock, field)).toBe('formatted:foo');
expect(indexPatternMock.getFormatterForField).toHaveBeenCalledWith(field);
expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field, hit });
});
it('should call default string formatter if no field specified', () => {
const convertMock = jest.fn((value: unknown) => `formatted:${value}`);
(getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReturnValue({
convert: convertMock,
});
expect(formatFieldValue('foo', hit, indexPatternMock)).toBe('formatted:foo');
expect(getServices().fieldFormats.getDefaultInstance).toHaveBeenCalledWith('string');
expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field: undefined, hit });
});
it('should call default string formatter if no indexPattern is specified', () => {
const convertMock = jest.fn((value: unknown) => `formatted:${value}`);
(getServices().fieldFormats.getDefaultInstance as jest.Mock).mockReturnValue({
convert: convertMock,
});
expect(formatFieldValue('foo', hit)).toBe('formatted:foo');
expect(getServices().fieldFormats.getDefaultInstance).toHaveBeenCalledWith('string');
expect(convertMock).toHaveBeenCalledWith('foo', 'html', { field: undefined, hit });
});
});

View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { estypes } from '@elastic/elasticsearch';
import { DataView, DataViewField, KBN_FIELD_TYPES } from '../../../../data/common';
import { getServices } from '../../kibana_services';
/**
* Formats the value of a specific field using the appropriate field formatter if available
* or the default string field formatter otherwise.
*
* @param value The value to format
* @param hit The actual search hit (required to get highlight information from)
* @param dataView The data view if available
* @param field The field that value was from if available
* @returns An sanitized HTML string, that is safe to be applied via dangerouslySetInnerHTML
*/
export function formatFieldValue(
value: unknown,
hit: estypes.SearchHit,
dataView?: DataView,
field?: DataViewField
): string {
if (!dataView || !field) {
// If either no field is available or no data view, we'll use the default
// string formatter to format that field.
return getServices()
.fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.STRING)
.convert(value, 'html', { hit, field });
}
// If we have a data view and field we use that fields field formatter
return dataView.getFormatterForField(field).convert(value, 'html', { hit, field });
}

View file

@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { getIgnoredReason, IgnoredReason } from './get_ignored_reason';
import { DataViewField, KBN_FIELD_TYPES } from '../../../../data/common';
function field(params: Partial<DataViewField>): DataViewField {
return {
name: 'text',
type: 'keyword',
...params,
} as unknown as DataViewField;
}
describe('getIgnoredReason', () => {
it('will correctly return undefined when no value was ignored', () => {
expect(getIgnoredReason(field({ name: 'foo' }), undefined)).toBeUndefined();
expect(getIgnoredReason(field({ name: 'foo' }), ['bar', 'baz'])).toBeUndefined();
});
it('will return UNKNOWN if the field passed in was only a name, and thus no type information is present', () => {
expect(getIgnoredReason('foo', ['foo'])).toBe(IgnoredReason.UNKNOWN);
});
it('will return IGNORE_ABOVE for string types', () => {
expect(getIgnoredReason(field({ name: 'foo', type: KBN_FIELD_TYPES.STRING }), ['foo'])).toBe(
IgnoredReason.IGNORE_ABOVE
);
});
// Each type that can have malformed values
[
KBN_FIELD_TYPES.DATE,
KBN_FIELD_TYPES.IP,
KBN_FIELD_TYPES.GEO_POINT,
KBN_FIELD_TYPES.GEO_SHAPE,
KBN_FIELD_TYPES.NUMBER,
].forEach((type) => {
it(`will return MALFORMED for ${type} fields`, () => {
expect(getIgnoredReason(field({ name: 'foo', type }), ['foo'])).toBe(IgnoredReason.MALFORMED);
});
});
it('will return unknown reasons if it does not know what the reason was', () => {
expect(getIgnoredReason(field({ name: 'foo', type: 'range' }), ['foo'])).toBe(
IgnoredReason.UNKNOWN
);
});
});

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { estypes } from '@elastic/elasticsearch';
import { DataViewField, KBN_FIELD_TYPES } from '../../../../data/common';
export enum IgnoredReason {
IGNORE_ABOVE = 'ignore_above',
MALFORMED = 'malformed',
UNKNOWN = 'unknown',
}
/**
* Returns the reason why a specific field was ignored in the response.
* Will return undefined if the field had no ignored values in it.
* This implementation will make some assumptions based on specific types
* of ignored values can only happen with specific field types in Elasticsearch.
*
* @param field Either the data view field or the string name of it.
* @param ignoredFields The hit._ignored value of the hit to validate.
*/
export function getIgnoredReason(
field: DataViewField | string,
ignoredFields: estypes.SearchHit['_ignored']
): IgnoredReason | undefined {
const fieldName = typeof field === 'string' ? field : field.name;
if (!ignoredFields?.includes(fieldName)) {
return undefined;
}
if (typeof field === 'string') {
return IgnoredReason.UNKNOWN;
}
switch (field.type) {
case KBN_FIELD_TYPES.STRING:
return IgnoredReason.IGNORE_ABOVE;
case KBN_FIELD_TYPES.NUMBER:
case KBN_FIELD_TYPES.DATE:
case KBN_FIELD_TYPES.GEO_POINT:
case KBN_FIELD_TYPES.GEO_SHAPE:
case KBN_FIELD_TYPES.IP:
return IgnoredReason.MALFORMED;
default:
return IgnoredReason.UNKNOWN;
}
}

View file

@ -36,6 +36,7 @@ import { KibanaLegacyStart } from '../../kibana_legacy/public';
import { UrlForwardingStart } from '../../url_forwarding/public';
import { NavigationPublicPluginStart } from '../../navigation/public';
import { IndexPatternFieldEditorStart } from '../../index_pattern_field_editor/public';
import { FieldFormatsStart } from '../../field_formats/public';
import type { SpacesApi } from '../../../../x-pack/plugins/spaces/public';
@ -49,6 +50,7 @@ export interface DiscoverServices {
history: () => History;
theme: ChartsPluginStart['theme'];
filterManager: FilterManager;
fieldFormats: FieldFormatsStart;
indexPatterns: IndexPatternsContract;
inspector: InspectorPublicPluginStart;
metadata: { branch: string };
@ -82,6 +84,7 @@ export function buildServices(
data: plugins.data,
docLinks: core.docLinks,
theme: plugins.charts.theme,
fieldFormats: plugins.fieldFormats,
filterManager: plugins.data.query.filterManager,
history: getHistory,
indexPatterns: plugins.data.indexPatterns,

View file

@ -61,6 +61,7 @@ import { IndexPatternFieldEditorStart } from '../../../plugins/index_pattern_fie
import { DeferredSpinner } from './shared';
import { ViewSavedSearchAction } from './application/embeddable/view_saved_search_action';
import type { SpacesPluginStart } from '../../../../x-pack/plugins/spaces/public';
import { FieldFormatsStart } from '../../field_formats/public';
declare module '../../share/public' {
export interface UrlGeneratorStateMapping {
@ -180,6 +181,7 @@ export interface DiscoverStartPlugins {
navigation: NavigationStart;
charts: ChartsPluginStart;
data: DataPublicPluginStart;
fieldFormats: FieldFormatsStart;
share?: SharePluginStart;
kibanaLegacy: KibanaLegacyStart;
urlForwarding: UrlForwardingStart;

View file

@ -19,7 +19,7 @@ describe('Source Format', () => {
convertHtml = source.getConverterFor(HTML_CONTEXT_TYPE) as HtmlContextTypeConvert;
});
test('should use the text content type if a field is not passed', () => {
test('should render stringified object', () => {
const hit = {
foo: 'bar',
number: 42,
@ -27,23 +27,8 @@ describe('Source Format', () => {
also: 'with "quotes" or \'single quotes\'',
};
expect(convertHtml(hit)).toBe(
'{&quot;foo&quot;:&quot;bar&quot;,&quot;number&quot;:42,&quot;hello&quot;:&quot;&lt;h1&gt;World&lt;/h1&gt;&quot;,&quot;also&quot;:&quot;with \\&quot;quotes\\&quot; or &#39;single quotes&#39;&quot;}'
);
});
test('should render a description list if a field is passed', () => {
const hit = {
foo: 'bar',
number: 42,
hello: '<h1>World</h1>',
also: 'with "quotes" or \'single quotes\'',
};
expect(
convertHtml(hit, { field: 'field', indexPattern: { formatHit: (h: string) => h }, hit })
).toMatchInlineSnapshot(
`"<dl class=\\"source truncate-by-height\\"><dt>foo:</dt><dd>bar</dd> <dt>number:</dt><dd>42</dd> <dt>hello:</dt><dd><h1>World</h1></dd> <dt>also:</dt><dd>with \\"quotes\\" or 'single quotes'</dd> </dl>"`
expect(convertHtml(hit, { field: 'field', hit })).toMatchInlineSnapshot(
`"{&quot;foo&quot;:&quot;bar&quot;,&quot;number&quot;:42,&quot;hello&quot;:&quot;&lt;h1&gt;World&lt;/h1&gt;&quot;,&quot;also&quot;:&quot;with \\\\&quot;quotes\\\\&quot; or &#39;single quotes&#39;&quot;}"`
);
});
});

View file

@ -7,33 +7,8 @@
*/
import { KBN_FIELD_TYPES } from '@kbn/field-types';
import React, { Fragment } from 'react';
import ReactDOM from 'react-dom/server';
import { escape, keys } from 'lodash';
import { shortenDottedString } from '../utils';
import { FieldFormat } from '../field_format';
import { TextContextTypeConvert, HtmlContextTypeConvert, FIELD_FORMAT_IDS } from '../types';
import { FORMATS_UI_SETTINGS } from '../constants/ui_settings';
interface Props {
defPairs: Array<[string, string]>;
}
const TemplateComponent = ({ defPairs }: Props) => {
return (
<dl className={'source truncate-by-height'}>
{defPairs.map((pair, idx) => (
<Fragment key={idx}>
<dt
dangerouslySetInnerHTML={{ __html: `${escape(pair[0])}:` }} // eslint-disable-line react/no-danger
/>
<dd
dangerouslySetInnerHTML={{ __html: `${pair[1]}` }} // eslint-disable-line react/no-danger
/>{' '}
</Fragment>
))}
</dl>
);
};
import { TextContextTypeConvert, FIELD_FORMAT_IDS } from '../types';
/** @public */
export class SourceFormat extends FieldFormat {
@ -42,32 +17,4 @@ export class SourceFormat extends FieldFormat {
static fieldType = KBN_FIELD_TYPES._SOURCE;
textConvert: TextContextTypeConvert = (value: string) => JSON.stringify(value);
htmlConvert: HtmlContextTypeConvert = (value: string, options = {}) => {
const { field, hit, indexPattern } = options;
if (!field) {
const converter = this.getConverterFor('text') as Function;
return escape(converter(value));
}
const highlights: Record<string, string[]> = (hit && hit.highlight) || {};
// TODO: remove index pattern dependency
const formatted = hit ? indexPattern!.formatHit(hit) : {};
const highlightPairs: Array<[string, string]> = [];
const sourcePairs: Array<[string, string]> = [];
const isShortDots = this.getConfig!(FORMATS_UI_SETTINGS.SHORT_DOTS_ENABLE);
keys(formatted).forEach((key) => {
const pairs = highlights[key] ? highlightPairs : sourcePairs;
const newField = isShortDots ? shortenDottedString(key) : key;
const val = formatted![key];
pairs.push([newField as string, val]);
}, []);
return ReactDOM.renderToStaticMarkup(
<TemplateComponent defPairs={highlightPairs.concat(sourcePairs)} />
);
};
}

View file

@ -17,10 +17,6 @@ export type FieldFormatsContentType = 'html' | 'text';
*/
export interface HtmlContextTypeOptions {
field?: { name: string };
// TODO: get rid of indexPattern dep completely
indexPattern?: {
formatHit: (hit: { highlight: Record<string, string[]> }) => Record<string, string>;
};
hit?: { highlight: Record<string, string[]> };
}

View file

@ -356,7 +356,7 @@ export class CsvGenerator {
let table: Datatable | undefined;
try {
table = tabifyDocs(results, index, { shallow: true });
table = tabifyDocs(results, index, { shallow: true, includeIgnoredValues: true });
} catch (err) {
this.logger.error(err);
}

View file

@ -2424,7 +2424,6 @@
"discover.docViews.table.toggleFieldDetails": "フィールド詳細を切り替える",
"discover.docViews.table.unableToFilterForPresenceOfMetaFieldsTooltip": "メタフィールドの有無でフィルタリングできません",
"discover.docViews.table.unableToFilterForPresenceOfScriptedFieldsTooltip": "スクリプトフィールドの有無でフィルタリングできません",
"discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip": "インデックスされていないフィールドは検索できません",
"discover.embeddable.inspectorRequestDataTitle": "データ",
"discover.embeddable.inspectorRequestDescription": "このリクエストはElasticsearchにクエリをかけ、検索データを取得します。",
"discover.embeddable.search.displayName": "検索",

View file

@ -2449,7 +2449,6 @@
"discover.docViews.table.toggleFieldDetails": "切换字段详细信息",
"discover.docViews.table.unableToFilterForPresenceOfMetaFieldsTooltip": "无法筛选元数据字段是否存在",
"discover.docViews.table.unableToFilterForPresenceOfScriptedFieldsTooltip": "无法筛选脚本字段是否存在",
"discover.docViews.table.unindexedFieldsCanNotBeSearchedTooltip": "无法搜索未索引字段",
"discover.embeddable.inspectorRequestDataTitle": "数据",
"discover.embeddable.inspectorRequestDescription": "此请求将查询 Elasticsearch 以获取搜索的数据。",
"discover.embeddable.search.displayName": "搜索",