[7.x] [Index patterns] Remove field cache (#82223) (#84109)

* [Index patterns] Remove field cache (#82223)

# Conflicts:
#	src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

* fix test
This commit is contained in:
Matthew Kime 2020-11-23 15:57:11 -06:00 committed by GitHub
parent 885af43ba1
commit aa37078288
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 5406 additions and 141 deletions

View file

@ -8,8 +8,6 @@
```typescript
getFieldAttrs: () => {
[x: string]: {
customLabel: string;
};
[x: string]: FieldAttrSet;
};
```

View file

@ -27,7 +27,7 @@ export declare class IndexPattern implements IIndexPattern
| [flattenHit](./kibana-plugin-plugins-data-public.indexpattern.flattenhit.md) | | <code>(hit: Record&lt;string, any&gt;, deep?: boolean) =&gt; Record&lt;string, any&gt;</code> | |
| [formatField](./kibana-plugin-plugins-data-public.indexpattern.formatfield.md) | | <code>FormatFieldFn</code> | |
| [formatHit](./kibana-plugin-plugins-data-public.indexpattern.formathit.md) | | <code>{</code><br/><code> (hit: Record&lt;string, any&gt;, type?: string): any;</code><br/><code> formatField: FormatFieldFn;</code><br/><code> }</code> | |
| [getFieldAttrs](./kibana-plugin-plugins-data-public.indexpattern.getfieldattrs.md) | | <code>() =&gt; {</code><br/><code> [x: string]: {</code><br/><code> customLabel: string;</code><br/><code> };</code><br/><code> }</code> | |
| [getFieldAttrs](./kibana-plugin-plugins-data-public.indexpattern.getfieldattrs.md) | | <code>() =&gt; {</code><br/><code> [x: string]: FieldAttrSet;</code><br/><code> }</code> | |
| [getOriginalSavedObjectBody](./kibana-plugin-plugins-data-public.indexpattern.getoriginalsavedobjectbody.md) | | <code>() =&gt; {</code><br/><code> fieldAttrs?: string &#124; undefined;</code><br/><code> title?: string &#124; undefined;</code><br/><code> timeFieldName?: string &#124; undefined;</code><br/><code> intervalName?: string &#124; undefined;</code><br/><code> fields?: string &#124; undefined;</code><br/><code> sourceFilters?: string &#124; undefined;</code><br/><code> fieldFormatMap?: string &#124; undefined;</code><br/><code> typeMeta?: string &#124; undefined;</code><br/><code> type?: string &#124; undefined;</code><br/><code> }</code> | Get last saved saved object fields |
| [id](./kibana-plugin-plugins-data-public.indexpattern.id.md) | | <code>string</code> | |
| [intervalName](./kibana-plugin-plugins-data-public.indexpattern.intervalname.md) | | <code>string &#124; undefined</code> | |

View file

@ -8,8 +8,6 @@
```typescript
getFieldAttrs: () => {
[x: string]: {
customLabel: string;
};
[x: string]: FieldAttrSet;
};
```

View file

@ -27,7 +27,7 @@ export declare class IndexPattern implements IIndexPattern
| [flattenHit](./kibana-plugin-plugins-data-server.indexpattern.flattenhit.md) | | <code>(hit: Record&lt;string, any&gt;, deep?: boolean) =&gt; Record&lt;string, any&gt;</code> | |
| [formatField](./kibana-plugin-plugins-data-server.indexpattern.formatfield.md) | | <code>FormatFieldFn</code> | |
| [formatHit](./kibana-plugin-plugins-data-server.indexpattern.formathit.md) | | <code>{</code><br/><code> (hit: Record&lt;string, any&gt;, type?: string): any;</code><br/><code> formatField: FormatFieldFn;</code><br/><code> }</code> | |
| [getFieldAttrs](./kibana-plugin-plugins-data-server.indexpattern.getfieldattrs.md) | | <code>() =&gt; {</code><br/><code> [x: string]: {</code><br/><code> customLabel: string;</code><br/><code> };</code><br/><code> }</code> | |
| [getFieldAttrs](./kibana-plugin-plugins-data-server.indexpattern.getfieldattrs.md) | | <code>() =&gt; {</code><br/><code> [x: string]: FieldAttrSet;</code><br/><code> }</code> | |
| [getOriginalSavedObjectBody](./kibana-plugin-plugins-data-server.indexpattern.getoriginalsavedobjectbody.md) | | <code>() =&gt; {</code><br/><code> fieldAttrs?: string &#124; undefined;</code><br/><code> title?: string &#124; undefined;</code><br/><code> timeFieldName?: string &#124; undefined;</code><br/><code> intervalName?: string &#124; undefined;</code><br/><code> fields?: string &#124; undefined;</code><br/><code> sourceFilters?: string &#124; undefined;</code><br/><code> fieldFormatMap?: string &#124; undefined;</code><br/><code> typeMeta?: string &#124; undefined;</code><br/><code> type?: string &#124; undefined;</code><br/><code> }</code> | Get last saved saved object fields |
| [id](./kibana-plugin-plugins-data-server.indexpattern.id.md) | | <code>string</code> | |
| [intervalName](./kibana-plugin-plugins-data-server.indexpattern.intervalname.md) | | <code>string &#124; undefined</code> | |

View file

@ -18,7 +18,7 @@
*/
import _, { each, reject } from 'lodash';
import { FieldAttrs } from '../..';
import { FieldAttrs, FieldAttrSet } from '../..';
import { DuplicateField } from '../../../../kibana_utils/common';
import { ES_FIELD_TYPES, KBN_FIELD_TYPES, IIndexPattern, IFieldType } from '../../../common';
@ -135,8 +135,19 @@ export class IndexPattern implements IIndexPattern {
const newFieldAttrs = { ...this.fieldAttrs };
this.fields.forEach((field) => {
const attrs: FieldAttrSet = {};
let hasAttr = false;
if (field.customLabel) {
newFieldAttrs[field.name] = { customLabel: field.customLabel };
attrs.customLabel = field.customLabel;
hasAttr = true;
}
if (field.count) {
attrs.count = field.count;
hasAttr = true;
}
if (hasAttr) {
newFieldAttrs[field.name] = attrs;
} else {
delete newFieldAttrs[field.name];
}
@ -321,7 +332,9 @@ export class IndexPattern implements IIndexPattern {
timeFieldName: this.timeFieldName,
intervalName: this.intervalName,
sourceFilters: this.sourceFilters ? JSON.stringify(this.sourceFilters) : undefined,
fields: this.fields ? JSON.stringify(this.fields) : undefined,
fields: this.fields
? JSON.stringify(this.fields.filter((field) => field.scripted))
: undefined,
fieldFormatMap,
type: this.type,
typeMeta: this.typeMeta ? JSON.stringify(this.typeMeta) : undefined,

View file

@ -156,15 +156,11 @@ describe('IndexPatterns', () => {
// Create a normal index patterns
const pattern = await indexPatterns.get('foo');
expect(pattern.version).toBe('fooa');
indexPatterns.clearCache();
// Create the same one - we're going to handle concurrency
const samePattern = await indexPatterns.get('foo');
expect(samePattern.version).toBe('fooaa');
// This will conflict because samePattern did a save (from refreshFields)
// but the resave should work fine
pattern.title = 'foo2';

View file

@ -202,22 +202,6 @@ export class IndexPatternsService {
}
};
private isFieldRefreshRequired(specs?: IndexPatternFieldMap): boolean {
if (!specs) {
return true;
}
return Object.values(specs).every((spec) => {
// See https://github.com/elastic/kibana/pull/8421
const hasFieldCaps = 'aggregatable' in spec && 'searchable' in spec;
// See https://github.com/elastic/kibana/pull/11969
const hasDocValuesFlag = 'readFromDocValues' in spec;
return !hasFieldCaps || !hasDocValuesFlag;
});
}
/**
* Get field list by providing { pattern }
* @param options
@ -304,8 +288,8 @@ export class IndexPatternsService {
values: { id, title },
}),
});
throw err;
}
return fields;
};
/**
@ -314,7 +298,11 @@ export class IndexPatternsService {
*/
fieldArrayToMap = (fields: FieldSpec[], fieldAttrs?: FieldAttrs) =>
fields.reduce<IndexPatternFieldMap>((collector, field) => {
collector[field.name] = { ...field, customLabel: fieldAttrs?.[field.name]?.customLabel };
collector[field.name] = {
...field,
customLabel: fieldAttrs?.[field.name]?.customLabel,
count: fieldAttrs?.[field.name]?.count,
};
return collector;
}, {});
@ -377,25 +365,20 @@ export class IndexPatternsService {
? JSON.parse(savedObject.attributes.fieldAttrs)
: {};
const isFieldRefreshRequired = this.isFieldRefreshRequired(spec.fields);
let isSaveRequired = isFieldRefreshRequired;
try {
spec.fields = isFieldRefreshRequired
? await this.refreshFieldSpecMap(
spec.fields || {},
id,
spec.title as string,
{
pattern: title as string,
metaFields: await this.config.get(UI_SETTINGS.META_FIELDS),
type,
rollupIndex: typeMeta?.params?.rollupIndex,
},
spec.fieldAttrs
)
: spec.fields;
spec.fields = await this.refreshFieldSpecMap(
spec.fields || {},
id,
spec.title as string,
{
pattern: title as string,
metaFields: await this.config.get(UI_SETTINGS.META_FIELDS),
type,
rollupIndex: typeMeta?.params?.rollup_index,
},
spec.fieldAttrs
);
} catch (err) {
isSaveRequired = false;
if (err instanceof IndexPatternMissingIndices) {
this.onNotification({
title: (err as any).message,
@ -417,22 +400,6 @@ export class IndexPatternsService {
: {};
const indexPattern = await this.create(spec, true);
if (isSaveRequired) {
try {
this.updateSavedObject(indexPattern);
} catch (err) {
this.onError(err, {
title: i18n.translate('data.indexPatterns.fetchFieldSaveErrorTitle', {
defaultMessage:
'Error saving after fetching fields for index pattern {title} (ID: {id})',
values: {
id: indexPattern.id,
title: indexPattern.title,
},
}),
});
}
}
if (indexPattern.isUnsupportedTimePattern()) {
this.onUnsupportedTimePattern({
@ -441,7 +408,6 @@ export class IndexPatternsService {
index: indexPattern.getIndex(),
});
}
indexPattern.resetOriginalSavedObjectBody();
return indexPattern;
};

View file

@ -53,7 +53,12 @@ export interface IndexPatternAttributes {
}
export interface FieldAttrs {
[key: string]: { customLabel: string };
[key: string]: FieldAttrSet;
}
export interface FieldAttrSet {
customLabel?: string;
count?: number;
}
export type OnNotification = (toastInputFields: ToastInputFields) => void;

View file

@ -1153,9 +1153,7 @@ export class IndexPattern implements IIndexPattern {
};
// (undocumented)
getFieldAttrs: () => {
[x: string]: {
customLabel: string;
};
[x: string]: FieldAttrSet;
};
// (undocumented)
getFieldByName(name: string): IndexPatternField | undefined;
@ -2374,6 +2372,7 @@ export const UI_SETTINGS: {
// src/plugins/data/common/es_query/filters/phrase_filter.ts:33:3 - (ae-forgotten-export) The symbol "PhraseFilterMeta" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/es_query/filters/phrases_filter.ts:31:3 - (ae-forgotten-export) The symbol "PhrasesFilterMeta" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:135:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/search/aggs/types.ts:113:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts
// src/plugins/data/public/field_formats/field_formats_service.ts:67:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts
// src/plugins/data/public/index.ts:66:23 - (ae-forgotten-export) The symbol "FILTERS" needs to be exported by the entry point index.d.ts

View file

@ -611,9 +611,7 @@ export class IndexPattern implements IIndexPattern {
};
// (undocumented)
getFieldAttrs: () => {
[x: string]: {
customLabel: string;
};
[x: string]: FieldAttrSet;
};
// (undocumented)
getFieldByName(name: string): IndexPatternField | undefined;
@ -1219,6 +1217,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage;
// src/plugins/data/common/es_query/filters/meta_filter.ts:54:3 - (ae-forgotten-export) The symbol "FilterMeta" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:58:45 - (ae-forgotten-export) The symbol "IndexPatternFieldMap" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:64:5 - (ae-forgotten-export) The symbol "FormatFieldFn" needs to be exported by the entry point index.d.ts
// src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:135:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts
// src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildCustomFilter" needs to be exported by the entry point index.d.ts
// src/plugins/data/server/index.ts:40:23 - (ae-forgotten-export) The symbol "buildFilter" needs to be exported by the entry point index.d.ts
// src/plugins/data/server/index.ts:71:21 - (ae-forgotten-export) The symbol "getEsQueryConfig" needs to be exported by the entry point index.d.ts

View file

@ -58,19 +58,6 @@ const mappingConflictHeader = i18n.translate(
}
);
const confirmMessage = i18n.translate('indexPatternManagement.editIndexPattern.refreshLabel', {
defaultMessage: 'This action resets the popularity counter of each field.',
});
const confirmModalOptionsRefresh = {
confirmButtonText: i18n.translate('indexPatternManagement.editIndexPattern.refreshButton', {
defaultMessage: 'Refresh',
}),
title: i18n.translate('indexPatternManagement.editIndexPattern.refreshHeader', {
defaultMessage: 'Refresh field list?',
}),
};
const confirmModalOptionsDelete = {
confirmButtonText: i18n.translate('indexPatternManagement.editIndexPattern.deleteButton', {
defaultMessage: 'Delete',
@ -118,16 +105,6 @@ export const EditIndexPattern = withRouter(
setDefaultIndex(indexPattern.id || '');
}, [uiSettings, indexPattern.id]);
const refreshFields = () => {
overlays.openConfirm(confirmMessage, confirmModalOptionsRefresh).then(async (isConfirmed) => {
if (isConfirmed) {
await data.indexPatterns.refreshFields(indexPattern);
await data.indexPatterns.updateSavedObject(indexPattern);
setFields(indexPattern.getNonScriptedFields());
}
});
};
const removePattern = () => {
async function doRemove() {
if (indexPattern.id === defaultIndex) {
@ -190,7 +167,6 @@ export const EditIndexPattern = withRouter(
<IndexHeader
indexPattern={indexPattern}
setDefault={setDefaultPattern}
refreshFields={refreshFields}
deleteIndexPatternClick={removePattern}
defaultIndex={defaultIndex}
/>

View file

@ -26,7 +26,6 @@ interface IndexHeaderProps {
indexPattern: IIndexPattern;
defaultIndex?: string;
setDefault?: () => void;
refreshFields?: () => void;
deleteIndexPatternClick?: () => void;
}
@ -44,14 +43,6 @@ const setDefaultTooltip = i18n.translate(
}
);
const refreshAriaLabel = i18n.translate('indexPatternManagement.editIndexPattern.refreshAria', {
defaultMessage: 'Reload field list.',
});
const refreshTooltip = i18n.translate('indexPatternManagement.editIndexPattern.refreshTooltip', {
defaultMessage: 'Refresh field list.',
});
const removeAriaLabel = i18n.translate('indexPatternManagement.editIndexPattern.removeAria', {
defaultMessage: 'Remove index pattern.',
});
@ -64,7 +55,6 @@ export function IndexHeader({
defaultIndex,
indexPattern,
setDefault,
refreshFields,
deleteIndexPatternClick,
}: IndexHeaderProps) {
return (
@ -90,20 +80,6 @@ export function IndexHeader({
</EuiFlexItem>
)}
{refreshFields && (
<EuiFlexItem>
<EuiToolTip content={refreshTooltip}>
<EuiButtonIcon
color="text"
onClick={refreshFields}
iconType="refresh"
aria-label={refreshAriaLabel}
data-test-subj="refreshFieldsIndexPatternButton"
/>
</EuiToolTip>
</EuiFlexItem>
)}
{deleteIndexPatternClick && (
<EuiFlexItem>
<EuiToolTip content={removeTooltip}>

View file

@ -60,7 +60,7 @@ export default function ({ getService, getPageObjects }) {
// check that it is 0 (previous increase was cancelled
const popularity = await PageObjects.settings.getPopularity();
log.debug('popularity = ' + popularity);
expect(popularity).to.be('0');
expect(popularity).to.be('');
});
it('can be saved', async function () {

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load diff

View file

@ -19,9 +19,14 @@
import { FtrProviderContext } from '../functional/ftr_provider_context';
// eslint-disable-next-line import/no-default-export
export default function ({ loadTestFile }: FtrProviderContext) {
export default function ({ loadTestFile, getService }: FtrProviderContext) {
describe('New Visualize Flow', function () {
this.tags('ciGroup2');
const esArchiver = getService('esArchiver');
before(async () => {
await esArchiver.loadIfNeeded('logstash_functional');
});
loadTestFile(require.resolve('./dashboard_embedding'));
});
}

View file

@ -867,7 +867,6 @@
"data.indexPatterns.editIndexPattern": "インデックスパターンを編集",
"data.indexPatterns.ensureDefaultIndexPattern.bannerLabel": "Kibanaでデータの可視化と閲覧を行うには、Elasticsearchからデータを取得するためのインデックスパターンの作成が必要です。",
"data.indexPatterns.fetchFieldErrorTitle": "インデックスパターンのフィールド取得中にエラーが発生 {title} (ID: {id})",
"data.indexPatterns.fetchFieldSaveErrorTitle": "インデックスパターンのフィールド取得後の保存中にエラーが発生 {title}ID: {id}",
"data.indexPatterns.unableWriteLabel": "インデックスパターンを書き込めません!このインデックスパターンへの最新の変更を取得するには、ページを更新してください。",
"data.indexPatterns.warningText": "現在{index}に一致するすべてのインデックスにクエリを実行しています。{title}はワイルドカードベースのインデックスパターンに移行されるはずです。",
"data.indexPatterns.warningTitle": "時間間隔インデックスパターンのサポートは廃止されました",
@ -2518,11 +2517,6 @@
"indexPatternManagement.editIndexPattern.list.defaultIndexPatternListName": "デフォルト",
"indexPatternManagement.editIndexPattern.mappingConflictHeader": "マッピングの矛盾",
"indexPatternManagement.editIndexPattern.mappingConflictLabel": "{conflictFieldsLength, plural, one {フィールド} other {フィールド}}が、このパターンと一致するインデックスの間で異なるタイプ文字列、整数などに定義されています。これらの矛盾したフィールドはKibanaの一部で使用できますが、Kibanaがタイプを把握しなければならない機能には使用できません。この問題を修正するにはデータのレンダリングが必要です。",
"indexPatternManagement.editIndexPattern.refreshAria": "フィールドリストを再度読み込みます。",
"indexPatternManagement.editIndexPattern.refreshButton": "更新",
"indexPatternManagement.editIndexPattern.refreshHeader": "フィールドリストを更新しますか?",
"indexPatternManagement.editIndexPattern.refreshLabel": "この操作は各フィールドの使用頻度をリセットします。",
"indexPatternManagement.editIndexPattern.refreshTooltip": "フィールドリストを更新します。",
"indexPatternManagement.editIndexPattern.removeAria": "インデックスパターンを削除します。",
"indexPatternManagement.editIndexPattern.removeTooltip": "インデックスパターンを削除します。",
"indexPatternManagement.editIndexPattern.scripted.addFieldButton": "スクリプトフィールドを追加",

View file

@ -867,7 +867,6 @@
"data.indexPatterns.editIndexPattern": "编辑索引模式",
"data.indexPatterns.ensureDefaultIndexPattern.bannerLabel": "要在 Kibana 中可视化和浏览数据,必须创建索引模式,以从 Elasticsearch 中检索数据。",
"data.indexPatterns.fetchFieldErrorTitle": "提取索引模式 {title} (ID: {id}) 的字段时出错",
"data.indexPatterns.fetchFieldSaveErrorTitle": "在提取索引模式 {title}ID{id})的字段后保存出错",
"data.indexPatterns.unableWriteLabel": "无法写入索引模式!请刷新页面以获取此索引模式的最新更改。",
"data.indexPatterns.warningText": "当前正在查询所有匹配 {index} 的索引。{title} 应迁移到基于通配符的索引模式。",
"data.indexPatterns.warningTitle": "已移除对时间间隔索引模式的支持",
@ -2519,11 +2518,6 @@
"indexPatternManagement.editIndexPattern.list.defaultIndexPatternListName": "默认值",
"indexPatternManagement.editIndexPattern.mappingConflictHeader": "映射冲突",
"indexPatternManagement.editIndexPattern.mappingConflictLabel": "{conflictFieldsLength, plural, one {一个字段} other {# 个字段}}已在匹配此模式的各个索引中定义为多个类型(字符串、整数等)。您也许仍能够在 Kibana 的各个部分中使用这些冲突类型,但它们将无法用于需要 Kibana 知道其类型的函数。要解决此问题,需要重新索引您的数据。",
"indexPatternManagement.editIndexPattern.refreshAria": "重新加载字段列表。",
"indexPatternManagement.editIndexPattern.refreshButton": "刷新",
"indexPatternManagement.editIndexPattern.refreshHeader": "刷新字段列表?",
"indexPatternManagement.editIndexPattern.refreshLabel": "此操作重置每个字段的常见度计数器。",
"indexPatternManagement.editIndexPattern.refreshTooltip": "刷新字段列表。",
"indexPatternManagement.editIndexPattern.removeAria": "移除索引模式。",
"indexPatternManagement.editIndexPattern.removeTooltip": "移除索引模式。",
"indexPatternManagement.editIndexPattern.scripted.addFieldButton": "添加脚本字段",

View file

@ -68,10 +68,10 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
it('keeps selected index pattern after refresh', async () => {
await PageObjects.lens.switchDataPanelIndexPattern('otherpattern');
await PageObjects.lens.switchDataPanelIndexPattern('log*');
await browser.refresh();
await PageObjects.header.waitUntilLoadingHasFinished();
expect(await PageObjects.lens.getDataPanelIndexPattern()).to.equal('otherpattern');
expect(await PageObjects.lens.getDataPanelIndexPattern()).to.equal('log*');
});
it('keeps time range and pinned filters after refreshing directly after saving', async () => {

View file

@ -327,9 +327,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
});
it('should allow to change index pattern', async () => {
await PageObjects.lens.switchFirstLayerIndexPattern('otherpattern');
expect(await PageObjects.lens.getFirstLayerIndexPattern()).to.equal('otherpattern');
expect(await PageObjects.lens.isShowingNoResults()).to.equal(true);
await PageObjects.lens.switchFirstLayerIndexPattern('log*');
expect(await PageObjects.lens.getFirstLayerIndexPattern()).to.equal('log*');
});
});
}