Add button for adding index.query.default_field setting to metricbeat indices (#32829)

* Add button for adding `index.query.default_field` setting to metricbeat indices

* Add button to 'group by index' view

* Refactor to more generic API

* Remove comment

* Update functional tests
This commit is contained in:
Josh Dover 2019-03-12 15:14:51 -05:00 committed by GitHub
parent baf77580b3
commit c97eff3bae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 11240 additions and 17 deletions

View file

@ -16,13 +16,16 @@ import {
EuiTitle,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { FixDefaultFieldsButton } from './default_fields/button';
import { DeleteTasksButton } from './delete_tasks_button';
import { ReindexButton } from './reindex';
interface DeprecationCellProps {
items?: Array<{ title?: string; body: string }>;
reindexIndexName?: string;
deleteIndexName?: string;
indexName?: string;
reindex?: boolean;
deleteIndex?: boolean;
needsDefaultFields?: boolean;
docUrl?: string;
headline?: string;
healthColor?: string;
@ -35,8 +38,10 @@ interface DeprecationCellProps {
export const DeprecationCell: StatelessComponent<DeprecationCellProps> = ({
headline,
healthColor,
reindexIndexName,
deleteIndexName,
indexName,
reindex,
deleteIndex,
needsDefaultFields,
docUrl,
items = [],
children,
@ -78,17 +83,23 @@ export const DeprecationCell: StatelessComponent<DeprecationCellProps> = ({
))}
</EuiFlexItem>
{reindexIndexName && (
{reindex && (
<EuiFlexItem grow={false}>
<ReindexButton indexName={reindexIndexName} />
<ReindexButton indexName={indexName!} />
</EuiFlexItem>
)}
{deleteIndexName && (
{deleteIndex && (
<EuiFlexItem grow={false}>
<DeleteTasksButton />
</EuiFlexItem>
)}
{needsDefaultFields && (
<EuiFlexItem grow={false}>
<FixDefaultFieldsButton indexName={indexName!} />
</EuiFlexItem>
)}
</EuiFlexGroup>
<EuiSpacer size="s" />

View file

@ -0,0 +1,129 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { ReactNode } from 'react';
import { EuiButton } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { kfetch } from 'ui/kfetch';
import { LoadingState } from '../../../../types';
/**
* Field types used by Metricbeat to generate the default_field setting.
* Matches Beats code here:
* https://github.com/elastic/beats/blob/eee127cb59b56f2ed7c7e317398c3f79c4158216/libbeat/template/processor.go#L104
*/
const METRICBEAT_DEFAULT_FIELD_TYPES: ReadonlySet<string> = new Set(['keyword', 'text', 'ip']);
const METRICBEAT_OTHER_DEFAULT_FIELDS: ReadonlySet<string> = new Set(['fields.*']);
interface FixDefaultFieldsButtonProps {
indexName: string;
}
interface FixDefaultFieldsButtonState {
fixLoadingState?: LoadingState;
}
/**
* Renders a button if given index is a valid Metricbeat index to add a default_field setting.
*/
export class FixDefaultFieldsButton extends React.Component<
FixDefaultFieldsButtonProps,
FixDefaultFieldsButtonState
> {
constructor(props: FixDefaultFieldsButtonProps) {
super(props);
this.state = {};
}
public render() {
const { fixLoadingState } = this.state;
if (!this.isMetricbeatIndex()) {
return null;
}
const buttonProps: any = { size: 's', onClick: this.fixMetricbeatIndex };
let buttonContent: ReactNode;
switch (fixLoadingState) {
case LoadingState.Loading:
buttonProps.disabled = true;
buttonProps.isLoading = true;
buttonContent = (
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.fixMetricbeatIndexButton.fixingLabel"
defaultMessage="Fixing…"
/>
);
break;
case LoadingState.Success:
buttonProps.iconSide = 'left';
buttonProps.iconType = 'check';
buttonProps.disabled = true;
buttonContent = (
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.fixMetricbeatIndexButton.fixedLabel"
defaultMessage="Fixed"
/>
);
break;
case LoadingState.Error:
buttonProps.color = 'danger';
buttonProps.iconSide = 'left';
buttonProps.iconType = 'cross';
buttonContent = (
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.fixMetricbeatIndexButton.failedLabel"
defaultMessage="Failed"
/>
);
break;
default:
buttonContent = (
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.fixMetricbeatIndexButton.reindexLabel"
defaultMessage="Fix"
/>
);
}
return <EuiButton {...buttonProps}>{buttonContent}</EuiButton>;
}
private isMetricbeatIndex = () => {
return this.props.indexName.startsWith('metricbeat-');
};
private fixMetricbeatIndex = async () => {
if (!this.isMetricbeatIndex()) {
return;
}
this.setState({
fixLoadingState: LoadingState.Loading,
});
try {
await kfetch({
pathname: `/api/upgrade_assistant/add_query_default_field/${this.props.indexName}`,
method: 'POST',
body: JSON.stringify({
fieldTypes: [...METRICBEAT_DEFAULT_FIELD_TYPES],
otherFields: [...METRICBEAT_OTHER_DEFAULT_FIELDS],
}),
});
this.setState({
fixLoadingState: LoadingState.Success,
});
} catch (e) {
this.setState({
fixLoadingState: LoadingState.Error,
});
}
};
}

View file

@ -9,6 +9,7 @@ import React from 'react';
import { EuiBasicTable } from '@elastic/eui';
import { injectI18n } from '@kbn/i18n/react';
import { FixDefaultFieldsButton } from './default_fields/button';
import { DeleteTasksButton } from './delete_tasks_button';
import { ReindexButton } from './reindex';
@ -18,6 +19,7 @@ export interface IndexDeprecationDetails {
index: string;
reindex: boolean;
delete: boolean;
needsDefaultFields: boolean;
details?: string;
}
@ -137,7 +139,10 @@ export class IndexDeprecationTableUI extends React.Component<
// NOTE: this naive implementation assumes all indices in the table are
// should show the reindex button. This should work for known usecases.
const { indices } = this.props;
if (!indices.find(i => i.reindex || i.delete)) {
const showDeleteButton = indices.find(i => i.delete === true);
const showReindexButton = indices.find(i => i.reindex === true);
const showNeedsDefaultFieldsButton = indices.find(i => i.needsDefaultFields === true);
if (!showDeleteButton && !showReindexButton && !showNeedsDefaultFieldsButton) {
return null;
}
@ -145,11 +150,13 @@ export class IndexDeprecationTableUI extends React.Component<
actions: [
{
render(indexDep: IndexDeprecationDetails) {
return indexDep.delete ? (
<DeleteTasksButton />
) : (
<ReindexButton indexName={indexDep.index} />
);
if (showDeleteButton) {
return <DeleteTasksButton />;
} else if (showReindexButton) {
return <ReindexButton indexName={indexDep.index!} />;
} else {
return <FixDefaultFieldsButton indexName={indexDep.index!} />;
}
},
},
],

View file

@ -74,12 +74,14 @@ describe('DeprecationList', () => {
"delete": false,
"details": undefined,
"index": "0",
"needsDefaultFields": false,
"reindex": false,
},
Object {
"delete": false,
"details": undefined,
"index": "1",
"needsDefaultFields": false,
"reindex": false,
},
]

View file

@ -17,6 +17,7 @@ import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table';
const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`;
const DELETE_INDEX_MESSAGE = `.tasks index must be re-created`;
const NEEDS_DEFAULT_FIELD_MESSAGE = 'Number of fields exceeds automatic field expansion limit';
const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => {
return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]);
@ -38,10 +39,10 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI
<DeprecationCell
headline={deprecation.message}
healthColor={COLOR_MAP[deprecation.level]}
reindexIndexName={deprecation.message === OLD_INDEX_MESSAGE ? deprecation.index! : undefined}
deleteIndexName={
deprecation.message === DELETE_INDEX_MESSAGE ? deprecation.index! : undefined
}
indexName={deprecation.index}
reindex={deprecation.message === OLD_INDEX_MESSAGE}
deleteIndex={deprecation.message === DELETE_INDEX_MESSAGE}
needsDefaultFields={deprecation.message === NEEDS_DEFAULT_FIELD_MESSAGE}
docUrl={deprecation.url}
items={items}
/>
@ -97,6 +98,7 @@ export const DeprecationList: StatelessComponent<{
details: dep.details,
reindex: dep.message === OLD_INDEX_MESSAGE,
delete: dep.message === DELETE_INDEX_MESSAGE,
needsDefaultFields: dep.message === NEEDS_DEFAULT_FIELD_MESSAGE,
}));
return <IndexDeprecation indices={indices} deprecation={deprecations[0]} />;

View file

@ -11,6 +11,7 @@ import { makeUpgradeAssistantUsageCollector } from './lib/telemetry';
import { registerClusterCheckupRoutes } from './routes/cluster_checkup';
import { registerDeleteTasksRoutes } from './routes/delete_tasks';
import { registerDeprecationLoggingRoutes } from './routes/deprecation_logging';
import { registerQueryDefaultFieldRoutes } from './routes/query_default_field';
import { registerReindexIndicesRoutes, registerReindexWorker } from './routes/reindex_indices';
import { registerTelemetryRoutes } from './routes/telemetry';
@ -18,6 +19,7 @@ export function initServer(server: Legacy.Server) {
registerClusterCheckupRoutes(server);
registerDeleteTasksRoutes(server);
registerDeprecationLoggingRoutes(server);
registerQueryDefaultFieldRoutes(server);
// The ReindexWorker uses a map of request headers that contain the authentication credentials
// for a given reindex. We cannot currently store these in an the .kibana index b/c we do not

View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { MappingProperties } from './reindexing/types';
import { addDefaultField, generateDefaultFields } from './query_default_field';
const defaultFieldTypes = new Set(['keyword', 'text', 'ip']);
describe('getDefaultFieldList', () => {
it('returns dot-delimited flat list', () => {
const mapping: MappingProperties = {
nested1: {
properties: {
included2: { type: 'ip' },
ignored2: { type: 'geopoint' },
nested2: {
properties: {
included3: { type: 'keyword' },
'included4.keyword': { type: 'keyword' },
},
},
},
},
ignored1: { type: 'object' },
included1: { type: 'text' },
};
expect(generateDefaultFields(mapping, defaultFieldTypes)).toMatchInlineSnapshot(`
Array [
"nested1.included2",
"nested1.nested2.included3",
"nested1.nested2.included4.keyword",
"included1",
]
`);
});
});
describe('fixMetricbeatIndex', () => {
const mockMappings = {
'metricbeat-1': {
mappings: { _doc: { properties: { field1: { type: 'text' }, field2: { type: 'float' } } } },
},
};
const mockSettings = {
'metricbeat-1': {
settings: {},
},
};
it('fails if index already has index.query.default_field setting', async () => {
const callWithRequest = jest.fn().mockResolvedValueOnce({
'metricbeat-1': {
settings: { index: { query: { default_field: [] } } },
},
});
await expect(
addDefaultField(callWithRequest, {} as any, 'metricbeat-1', defaultFieldTypes)
).rejects.toThrowErrorMatchingInlineSnapshot(
`"Index metricbeat-1 already has index.query.default_field set"`
);
});
it('updates index settings with default_field generated from mappings and otherFields', async () => {
const callWithRequest = jest
.fn()
.mockResolvedValueOnce(mockSettings)
.mockResolvedValueOnce(mockMappings)
.mockResolvedValueOnce({ acknowledged: true });
await expect(
addDefaultField(
callWithRequest,
{} as any,
'metricbeat-1',
defaultFieldTypes,
new Set(['fields.*', 'myCustomField'])
)
).resolves.toEqual({
acknowledged: true,
});
expect(callWithRequest.mock.calls[2]).toMatchInlineSnapshot(`
Array [
Object {},
"indices.putSettings",
Object {
"body": Object {
"index": Object {
"query": Object {
"default_field": Array [
"field1",
"fields.*",
"myCustomField",
],
},
},
},
"index": "metricbeat-1",
},
]
`);
});
});

View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import { Request } from 'hapi';
import { get } from 'lodash';
import { CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch';
import { MappingProperties } from './reindexing/types';
/**
* Adds the index.query.default_field setting, generated from the index's mapping.
*
* @param callWithRequest
* @param request
* @param indexName
* @param fieldTypes - Elasticsearch field types that should be used to generate the default_field from the index mapping
* @param otherFields - Other fields that should be included in the generated default_field that do not match `fieldTypes`
*/
export const addDefaultField = async (
callWithRequest: CallClusterWithRequest,
request: Request,
indexName: string,
fieldTypes: ReadonlySet<string>,
otherFields: ReadonlySet<string> = new Set()
) => {
// Verify index.query.default_field is not already set.
const settings = await callWithRequest(request, 'indices.getSettings', {
index: indexName,
});
if (get(settings, `${indexName}.settings.index.query.default_field`)) {
throw Boom.badRequest(`Index ${indexName} already has index.query.default_field set`);
}
// Get the mapping and generate the default_field based on `fieldTypes`
const mappingResp = await callWithRequest(request, 'indices.getMapping', {
index: indexName,
include_type_name: true,
});
const typeName = Object.getOwnPropertyNames(mappingResp[indexName].mappings)[0];
const mapping = mappingResp[indexName].mappings[typeName].properties as MappingProperties;
const generatedDefaultFields = new Set(generateDefaultFields(mapping, fieldTypes));
// Update the setting with the generated default_field
return await callWithRequest(request, 'indices.putSettings', {
index: indexName,
body: {
index: { query: { default_field: [...generatedDefaultFields, ...otherFields] } },
},
});
};
/**
* Recursively walks an index mapping and returns a flat array of dot-delimited
* strings represent all fields that are of a type included in `DEFAULT_FIELD_TYPES`
* @param mapping
*/
export const generateDefaultFields = (
mapping: MappingProperties,
fieldTypes: ReadonlySet<string>
): string[] =>
Object.getOwnPropertyNames(mapping).reduce(
(defaultFields, fieldName) => {
const { type, properties } = mapping[fieldName];
if (type && fieldTypes.has(type)) {
defaultFields.push(fieldName);
} else if (properties) {
generateDefaultFields(properties, fieldTypes).forEach(subField =>
defaultFields.push(`${fieldName}.${subField}`)
);
}
return defaultFields;
},
[] as string[]
);

View file

@ -0,0 +1,87 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Server } from 'hapi';
jest.mock('../lib/es_version_precheck');
const mockAddDefaultField = jest.fn();
jest.mock('../lib/query_default_field', () => ({
addDefaultField: mockAddDefaultField,
}));
import { registerQueryDefaultFieldRoutes } from './query_default_field';
const callWithRequest = jest.fn();
const server = new Server();
server.plugins = {
elasticsearch: {
getCluster: () => ({ callWithRequest } as any),
} as any,
} as any;
server.config = () => ({ get: () => '' } as any);
registerQueryDefaultFieldRoutes(server);
describe('add query default field API', () => {
beforeEach(() => {
mockAddDefaultField.mockClear();
});
it('calls addDefaultField with index, field types, and other fields', async () => {
mockAddDefaultField.mockResolvedValueOnce({ acknowledged: true });
const resp = await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/add_query_default_field/myIndex',
payload: {
fieldTypes: ['text', 'boolean'],
otherFields: ['myCustomField'],
},
});
expect(mockAddDefaultField).toHaveBeenCalledWith(
callWithRequest,
expect.anything(),
'myIndex',
new Set(['text', 'boolean']),
new Set(['myCustomField'])
);
expect(resp.statusCode).toEqual(200);
expect(resp.payload).toMatchInlineSnapshot(`"{\\"acknowledged\\":true}"`);
});
it('calls addDefaultField with index, field types if other fields is not specified', async () => {
mockAddDefaultField.mockResolvedValueOnce({ acknowledged: true });
const resp = await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/add_query_default_field/myIndex',
payload: {
fieldTypes: ['text', 'boolean'],
},
});
expect(mockAddDefaultField).toHaveBeenCalledWith(
callWithRequest,
expect.anything(),
'myIndex',
new Set(['text', 'boolean']),
undefined
);
expect(resp.statusCode).toEqual(200);
expect(resp.payload).toMatchInlineSnapshot(`"{\\"acknowledged\\":true}"`);
});
it('fails if fieldTypes is not specified', async () => {
const resp = await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/add_query_default_field/myIndex',
});
expect(mockAddDefaultField).not.toHaveBeenCalled();
expect(resp.statusCode).toEqual(400);
});
});

View file

@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import Joi from 'joi';
import { Legacy } from 'kibana';
import _ from 'lodash';
import { EsVersionPrecheck } from '../lib/es_version_precheck';
import { addDefaultField } from '../lib/query_default_field';
/**
* Adds routes for detecting and fixing 6.x Metricbeat indices that need the
* `index.query.default_field` index setting added.
*
* @param server
*/
export function registerQueryDefaultFieldRoutes(server: Legacy.Server) {
const { callWithRequest } = server.plugins.elasticsearch.getCluster('admin');
server.route({
path: '/api/upgrade_assistant/add_query_default_field/{indexName}',
method: 'POST',
options: {
pre: [EsVersionPrecheck],
validate: {
params: Joi.object({
indexName: Joi.string().required(),
}),
payload: Joi.object({
fieldTypes: Joi.array()
.items(Joi.string())
.required(),
otherFields: Joi.array().items(Joi.string()),
}),
},
},
async handler(request) {
try {
const { indexName } = request.params;
const { fieldTypes, otherFields } = request.payload as {
fieldTypes: string[];
otherFields?: string[];
};
return await addDefaultField(
callWithRequest,
request,
indexName,
new Set(fieldTypes),
otherFields ? new Set(otherFields) : undefined
);
} catch (e) {
if (e.status === 403) {
return Boom.forbidden(e.message);
}
return Boom.boomify(e, {
statusCode: 500,
});
}
},
});
}

File diff suppressed because it is too large Load diff

View file

@ -8,6 +8,7 @@ export default function ({ loadTestFile }) {
describe('upgrade assistant', function () {
this.tags('ciGroup5');
loadTestFile(require.resolve('./query_default_field'));
loadTestFile(require.resolve('./reindexing'));
});
}

View file

@ -0,0 +1,46 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from 'expect.js';
export default function ({ getService }) {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
const indexName = `metricbeat-6.7.0-2019.03.11`;
describe('add default_field setting', () => {
beforeEach(async () => {
await esArchiver.load('upgrade_assistant/metricbeat');
});
afterEach(async () => {
await esArchiver.unload('upgrade_assistant/metricbeat');
});
it('adds index.query.default_field to metricbeat index', async () => {
const { body } = await supertest
.post(`/api/upgrade_assistant/add_query_default_field/${indexName}`)
.set('kbn-xsrf', 'xxx')
.send({ fieldTypes: ['text', 'keyword', 'ip'], otherFields: ['fields.*'] })
.expect(200);
expect(body.acknowledged).to.be(true);
// The index.query.default_field setting should now be set
const settingsResp = await es.indices.getSettings({ index: indexName });
expect(settingsResp[indexName].settings.index.query.default_field).to.not.be(undefined);
// Deprecation message should be gone
const { body: uaBody } = await supertest.get('/api/upgrade_assistant/status').expect(200);
const depMessage = uaBody.indices.find(
dep => dep.index === indexName && dep.message === 'Number of fields exceeds automatic field expansion limit'
);
expect(depMessage).to.be(undefined);
});
});
}