[Security Solution][Lists] - Update lists index template logic (#133067)

## Summary

The lists plugin stores large value lists in two data indices - `.lists-*` and `.items-*`. These were still using the legacy ES template. This PR updates relevant routes to now use the new index templates.

- `createListsIndexRoute` now uses the new template routes and checks for legacy templates to delete them
- `deleteListsIndex` now uses up to date ES API 
- Updates the templates to follow new structure
This commit is contained in:
Yara Tercero 2022-07-26 13:43:03 -07:00 committed by GitHub
parent 30121495f4
commit 1327c88117
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 465 additions and 54 deletions

View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
export const deleteIndexTemplate = async (
esClient: ElasticsearchClient,
name: string
): Promise<unknown> => {
return (
await esClient.indices.deleteIndexTemplate(
{
name,
},
{ meta: true }
)
).body;
};

View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
export const getIndexTemplateExists = async (
esClient: ElasticsearchClient,
template: string
): Promise<boolean> => {
return (
await esClient.indices.existsIndexTemplate(
{
name: template,
},
{ meta: true }
)
).body;
};

View file

@ -10,6 +10,7 @@ export * from './bad_request_error';
export * from './create_boostrap_index';
export * from './decode_version';
export * from './delete_all_index';
export * from './delete_index_template';
export * from './delete_policy';
export * from './delete_template';
export * from './encode_hit_version';
@ -17,10 +18,12 @@ export * from './get_bootstrap_index_exists';
export * from './get_index_aliases';
export * from './get_index_count';
export * from './get_index_exists';
export * from './get_index_template_exists';
export * from './get_policy_exists';
export * from './get_template_exists';
export * from './read_index';
export * from './read_privileges';
export * from './set_index_template';
export * from './set_policy';
export * from './set_template';
export * from './transform_error';

View file

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import type { ElasticsearchClient } from '../elasticsearch_client';
export const setIndexTemplate = async (
esClient: ElasticsearchClient,
name: string,
body: Record<string, unknown>
): Promise<unknown> => {
return (
await esClient.indices.putIndexTemplate(
{
name,
body,
},
{ meta: true }
)
).body;
};

View file

@ -33,33 +33,49 @@ export const createListIndexRoute = (router: ListsPluginRouter): void => {
const listIndexExists = await lists.getListIndexExists();
const listItemIndexExists = await lists.getListItemIndexExists();
const policyExists = await lists.getListPolicyExists();
const policyListItemExists = await lists.getListItemPolicyExists();
if (!policyExists) {
await lists.setListPolicy();
}
if (!policyListItemExists) {
await lists.setListItemPolicy();
}
const templateExists = await lists.getListTemplateExists();
const templateListItemsExists = await lists.getListItemTemplateExists();
const legacyTemplateExists = await lists.getLegacyListTemplateExists();
const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists();
if (!templateExists) {
await lists.setListTemplate();
}
if (!templateListItemsExists) {
await lists.setListItemTemplate();
}
try {
// Check if the old legacy lists and items template exists and remove it
if (legacyTemplateExists) {
await lists.deleteLegacyListTemplate();
}
if (legacyTemplateListItemsExists) {
await lists.deleteLegacyListItemTemplate();
}
} catch (err) {
if (err.statusCode !== 404) {
throw err;
}
}
if (listIndexExists && listItemIndexExists) {
return siemResponse.error({
body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`,
statusCode: 409,
});
} else {
const policyExists = await lists.getListPolicyExists();
const policyListItemExists = await lists.getListItemPolicyExists();
if (!policyExists) {
await lists.setListPolicy();
}
if (!policyListItemExists) {
await lists.setListItemPolicy();
}
const templateExists = await lists.getListTemplateExists();
const templateListItemsExists = await lists.getListItemTemplateExists();
if (!templateExists) {
await lists.setListTemplate();
}
if (!templateListItemsExists) {
await lists.setListItemTemplate();
}
if (!listIndexExists) {
await lists.createListBootStrapIndex();
}

View file

@ -82,6 +82,17 @@ export const deleteListIndexRoute = (router: ListsPluginRouter): void => {
await lists.deleteListItemTemplate();
}
// check if legacy template exists
const legacyTemplateExists = await lists.getLegacyListTemplateExists();
const legacyItemTemplateExists = await lists.getLegacyListItemTemplateExists();
if (legacyTemplateExists) {
await lists.deleteLegacyListTemplate();
}
if (legacyItemTemplateExists) {
await lists.deleteLegacyListItemTemplate();
}
const [validated, errors] = validate({ acknowledged: true }, acknowledgeSchema);
if (errors != null) {
return siemResponse.error({ body: errors, statusCode: 500 });

View file

@ -24,8 +24,17 @@ describe('get_list_item_template', () => {
const template = getListItemTemplate('some_index');
expect(template).toEqual({
index_patterns: ['some_index-*'],
mappings: { listMappings: {} },
settings: { index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } } },
template: {
mappings: { listMappings: {} },
settings: {
index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } },
mapping: {
total_fields: {
limit: 10000,
},
},
},
},
});
});
});

View file

@ -10,12 +10,19 @@ import listsItemsMappings from './list_item_mappings.json';
export const getListItemTemplate = (index: string): Record<string, unknown> => {
const template = {
index_patterns: [`${index}-*`],
mappings: listsItemsMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
template: {
mappings: listsItemsMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,
},
},
},
},

View file

@ -8,7 +8,8 @@
import { getListTemplate } from './get_list_template';
jest.mock('./list_mappings.json', () => ({
listMappings: {},
dynamic: 'strict',
properties: {},
}));
describe('get_list_template', () => {
@ -24,8 +25,13 @@ describe('get_list_template', () => {
const template = getListTemplate('some_index');
expect(template).toEqual({
index_patterns: ['some_index-*'],
mappings: { listMappings: {} },
settings: { index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } } },
template: {
mappings: { dynamic: 'strict', properties: {} },
settings: {
index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } },
mapping: { total_fields: { limit: 10000 } },
},
},
});
});
});

View file

@ -9,12 +9,19 @@ import listMappings from './list_mappings.json';
export const getListTemplate = (index: string): Record<string, unknown> => ({
index_patterns: [`${index}-*`],
mappings: listMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
template: {
mappings: listMappings,
settings: {
index: {
lifecycle: {
name: index,
rollover_alias: index,
},
},
mapping: {
total_fields: {
limit: 10000,
},
},
},
},

View file

@ -9,13 +9,15 @@ import type { ElasticsearchClient } from '@kbn/core/server';
import {
createBootstrapIndex,
deleteAllIndex,
deleteIndexTemplate,
deletePolicy,
deleteTemplate,
getIndexExists,
getBootstrapIndexExists,
getIndexTemplateExists,
getPolicyExists,
getTemplateExists,
setIndexTemplate,
setPolicy,
setTemplate,
} from '@kbn/securitysolution-es-utils';
import type {
FoundListItemSchema,
@ -244,7 +246,7 @@ export class ListClient {
public getListIndexExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getIndexExists(esClient, listIndex);
return getBootstrapIndexExists(esClient, listIndex);
};
/**
@ -254,7 +256,7 @@ export class ListClient {
public getListItemIndexExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getIndexExists(esClient, listItemIndex);
return getBootstrapIndexExists(esClient, listItemIndex);
};
/**
@ -304,7 +306,7 @@ export class ListClient {
public getListTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getTemplateExists(esClient, listIndex);
return getIndexTemplateExists(esClient, listIndex);
};
/**
@ -312,6 +314,26 @@ export class ListClient {
* @returns True if the list item template for ILM exists, otherwise false.
*/
public getListItemTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getIndexTemplateExists(esClient, listItemIndex);
};
/**
* Returns true if the list template for ILM exists, otherwise false
* @returns True if the list template for ILM exists, otherwise false.
*/
public getLegacyListTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return getTemplateExists(esClient, listIndex);
};
/**
* Returns true if the list item template for ILM exists, otherwise false
* @returns True if the list item template for ILM exists, otherwise false.
*/
public getLegacyListItemTemplateExists = async (): Promise<boolean> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return getTemplateExists(esClient, listItemIndex);
@ -343,7 +365,7 @@ export class ListClient {
const { esClient } = this;
const template = this.getListTemplate();
const listIndex = this.getListIndex();
return setTemplate(esClient, listIndex, template);
return setIndexTemplate(esClient, listIndex, template);
};
/**
@ -354,7 +376,7 @@ export class ListClient {
const { esClient } = this;
const template = this.getListItemTemplate();
const listItemIndex = this.getListItemIndex();
return setTemplate(esClient, listItemIndex, template);
return setIndexTemplate(esClient, listItemIndex, template);
};
/**
@ -424,7 +446,7 @@ export class ListClient {
public deleteListTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deleteTemplate(esClient, listIndex);
return deleteIndexTemplate(esClient, listIndex);
};
/**
@ -432,6 +454,26 @@ export class ListClient {
* @returns The contents of the list item template
*/
public deleteListItemTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteIndexTemplate(esClient, listItemIndex);
};
/**
* Deletes the list boot strap index for ILM policies.
* @returns The contents of the bootstrap response from Elasticsearch
*/
public deleteLegacyListTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listIndex = this.getListIndex();
return deleteTemplate(esClient, listIndex);
};
/**
* Delete the list item boot strap index for ILM policies.
* @returns The contents of the bootstrap response from Elasticsearch
*/
public deleteLegacyListItemTemplate = async (): Promise<unknown> => {
const { esClient } = this;
const listItemIndex = this.getListItemIndex();
return deleteTemplate(esClient, listItemIndex);

View file

@ -42,9 +42,13 @@ describe('Detections > Need Admin Callouts indicating an admin is needed to migr
beforeEach(() => {
// Index mapping outdated is forced to return true as being outdated so that we get the
// need admin callouts being shown.
cy.intercept('GET', '/api/detection_engine/index', {
index_mapping_outdated: true,
name: '.alerts-security.alerts-default',
cy.intercept('GET', '/api/detection_engine/index', (req) => {
req.reply((res) => {
res.send(200, {
index_mapping_outdated: true,
name: '.alerts-security.alerts-default',
});
});
});
});
context('On Detections home page', () => {

View file

@ -26,7 +26,7 @@ export const useListsIndex = (): UseListsIndexReturn => {
const { lists } = useKibana().services;
const http = useHttp();
const { addError } = useAppToasts();
const { canReadIndex } = useListsPrivileges();
const { canReadIndex, canManageIndex, canWriteIndex } = useListsPrivileges();
const { loading: readLoading, start: readListIndex, ...readListIndexState } = useReadListIndex();
const {
loading: createLoading,
@ -35,17 +35,19 @@ export const useListsIndex = (): UseListsIndexReturn => {
} = useCreateListIndex();
const loading = readLoading || createLoading;
// read route utilizes `esClient.indices.getAlias` which requires
// management privileges
const readIndex = useCallback(() => {
if (lists && canReadIndex) {
if (lists && canReadIndex && canManageIndex) {
readListIndex({ http });
}
}, [http, lists, readListIndex, canReadIndex]);
}, [http, lists, readListIndex, canReadIndex, canManageIndex]);
const createIndex = useCallback(() => {
if (lists) {
if (lists && canManageIndex && canWriteIndex) {
createListIndex({ http });
}
}, [createListIndex, http, lists]);
}, [createListIndex, http, lists, canManageIndex, canWriteIndex]);
// initial read list
useEffect(() => {

View file

@ -0,0 +1,88 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { LIST_INDEX } from '@kbn/securitysolution-list-constants';
import { getTemplateExists, getIndexTemplateExists } from '@kbn/securitysolution-es-utils';
import { FtrProviderContext } from '../../common/ftr_provider_context';
import { createLegacyListsIndices, deleteListsIndex } from '../../utils';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertest = getService('supertest');
const log = getService('log');
const es = getService('es');
describe('create_list_index_route', () => {
beforeEach(async () => {
await deleteListsIndex(supertest, log);
});
afterEach(async () => {
await deleteListsIndex(supertest, log);
});
it('should create lists indices', async () => {
const { body: fetchedIndices } = await supertest
.get(LIST_INDEX)
.set('kbn-xsrf', 'true')
.expect(404);
expect(fetchedIndices).to.eql({
message: 'index .lists-default and index .items-default does not exist',
status_code: 404,
});
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200);
const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200);
expect(body).to.eql({ list_index: true, list_item_index: true });
});
it('should update lists indices if old legacy templates exists', async () => {
// create legacy indices
await createLegacyListsIndices(es);
const { body: listsIndex } = await supertest
.get(LIST_INDEX)
.set('kbn-xsrf', 'true')
.expect(200);
// confirm that legacy templates are in use
const legacyListsTemplateExists = await getTemplateExists(es, '.lists-default');
const legacyItemsTemplateExists = await getTemplateExists(es, '.items-default');
const nonLegacyListsTemplateExists = await getIndexTemplateExists(es, '.lists-default');
const nonLegacyItemsTemplateExists = await getIndexTemplateExists(es, '.items-default');
expect(legacyListsTemplateExists).to.equal(true);
expect(legacyItemsTemplateExists).to.equal(true);
expect(nonLegacyListsTemplateExists).to.equal(false);
expect(nonLegacyItemsTemplateExists).to.equal(false);
expect(listsIndex).to.eql({ list_index: true, list_item_index: true });
// Expected 409 as index exists already, but now the templates should have been updated
await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(409);
const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200);
const legacyListsTemplateExistsPostMigration = await getTemplateExists(es, '.lists-default');
const legacyItemsTemplateExistsPostMigration = await getTemplateExists(es, '.items-default');
const newListsTemplateExists = await getIndexTemplateExists(es, '.lists-default');
const newItemsTemplateExists = await getIndexTemplateExists(es, '.items-default');
expect(legacyListsTemplateExistsPostMigration).to.equal(false);
expect(legacyItemsTemplateExistsPostMigration).to.equal(false);
expect(newListsTemplateExists).to.equal(true);
expect(newItemsTemplateExists).to.equal(true);
expect(body).to.eql({ list_index: true, list_item_index: true });
});
});
};

View file

@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../common/ftr_provider_context';
export default ({ loadTestFile }: FtrProviderContext): void => {
describe('lists api security and spaces enabled', function () {
loadTestFile(require.resolve('./create_lists'));
loadTestFile(require.resolve('./create_lists_index'));
loadTestFile(require.resolve('./create_list_items'));
loadTestFile(require.resolve('./read_lists'));
loadTestFile(require.resolve('./read_list_items'));

View file

@ -21,8 +21,11 @@ import {
LIST_INDEX,
LIST_ITEM_URL,
} from '@kbn/securitysolution-list-constants';
import { setPolicy, setTemplate, createBootstrapIndex } from '@kbn/securitysolution-es-utils';
import { Client } from '@elastic/elasticsearch';
import { ToolingLog } from '@kbn/tooling-log';
import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock';
import { countDownTest } from '../detection_engine_api_integration/utils';
/**
@ -414,3 +417,144 @@ export const waitForTextListItems = async (
): Promise<void> => {
await Promise.all(itemValues.map((item) => waitForTextListItem(supertest, log, item, fileName)));
};
/**
* Convenience function for creating legacy index templates to
* test out logic updating to new index templates
* @param es es client
*/
export const createLegacyListsIndices = async (es: Client) => {
await setPolicy(es, '.lists-default', {
policy: {
phases: {
hot: {
min_age: '0ms',
actions: {
rollover: {
max_size: '50gb',
},
},
},
},
},
});
await setPolicy(es, '.items-default', {
policy: {
phases: {
hot: {
min_age: '0ms',
actions: {
rollover: {
max_size: '50gb',
},
},
},
},
},
});
await setTemplate(es, '.lists-default', {
index_patterns: [`.lists-default-*`],
mappings: {
dynamic: 'strict',
properties: {
name: {
type: 'keyword',
},
deserializer: {
type: 'keyword',
},
serializer: {
type: 'keyword',
},
description: {
type: 'keyword',
},
type: {
type: 'keyword',
},
tie_breaker_id: {
type: 'keyword',
},
meta: {
enabled: 'false',
type: 'object',
},
created_at: {
type: 'date',
},
updated_at: {
type: 'date',
},
created_by: {
type: 'keyword',
},
updated_by: {
type: 'keyword',
},
version: {
type: 'keyword',
},
immutable: {
type: 'boolean',
},
},
},
settings: {
index: {
lifecycle: {
name: '.lists-default',
rollover_alias: '.lists-default',
},
},
},
});
await setTemplate(es, '.items-default', {
index_patterns: [`.items-default-*`],
mappings: {
dynamic: 'strict',
properties: {
tie_breaker_id: {
type: 'keyword',
},
list_id: {
type: 'keyword',
},
deserializer: {
type: 'keyword',
},
serializer: {
type: 'keyword',
},
meta: {
enabled: 'false',
type: 'object',
},
created_at: {
type: 'date',
},
updated_at: {
type: 'date',
},
created_by: {
type: 'keyword',
},
updated_by: {
type: 'keyword',
},
ip: {
type: 'ip',
},
},
},
settings: {
index: {
lifecycle: {
name: '.items-default',
rollover_alias: '.items-default',
},
},
},
});
await createBootstrapIndex(es, '.lists-default');
await createBootstrapIndex(es, '.items-default');
};