[Enterprise Search] Add API index creation (#136970)

This commit is contained in:
Sander Philipse 2022-07-22 21:06:50 +02:00 committed by GitHub
parent 757001f3c6
commit 007addfa3b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 269 additions and 46 deletions

View file

@ -34,6 +34,7 @@ export interface Connector {
configuration: ConnectorConfiguration;
id: string;
index_name: string;
language: string | null;
last_seen: string | null;
last_sync_error: string | null;
last_sync_status: string | null;

View file

@ -25,6 +25,7 @@ export const indices: ElasticsearchIndexWithIngestion[] = [
configuration: {},
id: '2',
index_name: 'connector',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: SyncStatus.COMPLETED,

View file

@ -34,6 +34,7 @@ export const connectorIndex: ConnectorViewIndex = {
configuration: {},
id: '2',
index_name: 'connector',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: SyncStatus.COMPLETED,

View file

@ -20,20 +20,28 @@ describe('addConnectorPackageApiLogic', () => {
it('calls correct api', async () => {
const promise = Promise.resolve({ id: 'unique id', index_name: 'indexName' });
http.post.mockReturnValue(promise);
const result = addConnectorPackage({ indexName: 'indexName' });
const result = addConnectorPackage({ indexName: 'indexName', language: 'en' });
await nextTick();
expect(http.post).toHaveBeenCalledWith('/internal/enterprise_search/connectors', {
body: JSON.stringify({ index_name: 'indexName' }),
body: JSON.stringify({ index_name: 'indexName', language: 'en' }),
});
await expect(result).resolves.toEqual({ id: 'unique id', indexName: 'indexName' });
});
it('adds delete param if specific', async () => {
const promise = Promise.resolve({ id: 'unique id', index_name: 'indexName' });
http.post.mockReturnValue(promise);
const result = addConnectorPackage({ deleteExistingConnector: true, indexName: 'indexName' });
const result = addConnectorPackage({
deleteExistingConnector: true,
indexName: 'indexName',
language: null,
});
await nextTick();
expect(http.post).toHaveBeenCalledWith('/internal/enterprise_search/connectors', {
body: JSON.stringify({ index_name: 'indexName', delete_existing_connector: true }),
body: JSON.stringify({
delete_existing_connector: true,
index_name: 'indexName',
language: null,
}),
});
await expect(result).resolves.toEqual({ id: 'unique id', indexName: 'indexName' });
});

View file

@ -16,6 +16,7 @@ interface AddConnectorValue {
export interface AddConnectorPackageApiLogicArgs {
deleteExistingConnector?: boolean;
indexName: string;
language: string | null;
}
export interface AddConnectorPackageApiLogicResponse {
@ -24,8 +25,9 @@ export interface AddConnectorPackageApiLogicResponse {
}
export const addConnectorPackage = async ({
indexName,
deleteExistingConnector,
indexName,
language,
}: AddConnectorPackageApiLogicArgs): Promise<AddConnectorPackageApiLogicResponse> => {
const route = '/internal/enterprise_search/connectors';
@ -33,8 +35,9 @@ export const addConnectorPackage = async ({
? { delete_existing_connector: deleteExistingConnector }
: {};
const params = {
index_name: indexName,
...deleteParam,
index_name: indexName,
language,
};
const result = await HttpLogic.values.http.post<AddConnectorValue>(route, {
body: JSON.stringify(params),

View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { mockHttpValues } from '../../../__mocks__/kea_logic';
import { nextTick } from '@kbn/test-jest-helpers';
import { createApiIndex } from './create_api_index_api_logic';
describe('createApiIndexApiLogic', () => {
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
});
describe('createApiIndex', () => {
it('calls correct api', async () => {
const promise = Promise.resolve({ index: 'indexName' });
http.post.mockReturnValue(promise);
const result = createApiIndex({ indexName: 'indexName', language: 'en' });
await nextTick();
expect(http.post).toHaveBeenCalledWith('/internal/enterprise_search/indices', {
body: JSON.stringify({ index_name: 'indexName', language: 'en' }),
});
await expect(result).resolves.toEqual({ indexName: 'indexName' });
});
});
});

View file

@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../shared/http';
interface CreateApiIndexValue {
index: string;
}
export interface CreateApiIndexApiLogicArgs {
deleteExistingConnector?: boolean;
indexName: string;
language: string | null;
}
export interface CreateApiIndexApiLogicResponse {
indexName: string;
}
export const createApiIndex = async ({
indexName,
language,
}: CreateApiIndexApiLogicArgs): Promise<CreateApiIndexApiLogicResponse> => {
const route = '/internal/enterprise_search/indices';
const params = {
index_name: indexName,
language,
};
const result = await HttpLogic.values.http.post<CreateApiIndexValue>(route, {
body: JSON.stringify(params),
});
return {
indexName: result.index,
};
};
export const CreateApiIndexApiLogic = createApiLogic(
['create_api_index_api_logic'],
createApiIndex
);

View file

@ -11,8 +11,9 @@ import { shallow } from 'enzyme';
import { EuiSteps } from '@elastic/eui';
import { NewSearchIndexTemplate } from '../new_search_index_template';
import { MethodApi } from './method_api';
import { NewSearchIndexTemplate } from './new_search_index_template';
describe('MethodApi', () => {
beforeEach(() => {

View file

@ -5,20 +5,20 @@
* 2.0.
*/
/**
* TODO:
* - Need to add documentation URLs (search for `#`s)
*/
import React from 'react';
import { useActions } from 'kea';
import { EuiSteps, EuiText } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { NewSearchIndexTemplate } from './new_search_index_template';
import { NewSearchIndexTemplate } from '../new_search_index_template';
import { MethodApiLogic } from './method_api_logic';
export const MethodApi: React.FC = () => {
const { makeRequest } = useActions(MethodApiLogic);
return (
<NewSearchIndexTemplate
title={
@ -28,7 +28,7 @@ export const MethodApi: React.FC = () => {
/>
}
type="api"
onSubmit={() => null}
onSubmit={(indexName, language) => makeRequest({ indexName, language })}
>
<EuiSteps
steps={[

View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
LogicMounter,
mockFlashMessageHelpers,
mockKibanaValues,
} from '../../../../__mocks__/kea_logic';
import { HttpError } from '../../../../../../common/types/api';
import { MethodApiLogic } from './method_api_logic';
describe('MethodApiLogic', () => {
const { mount } = new LogicMounter(MethodApiLogic);
const { clearFlashMessages, flashAPIErrors } = mockFlashMessageHelpers;
const { navigateToUrl } = mockKibanaValues;
beforeEach(() => {
jest.clearAllMocks();
mount();
});
describe('listeners', () => {
describe('apiSuccess', () => {
it('navigates user to index detail view', () => {
MethodApiLogic.actions.apiSuccess({ indexName: 'my-index' });
expect(navigateToUrl).toHaveBeenCalledWith('/search_indices/my-index/overview');
});
});
describe('makeRequest', () => {
it('clears any displayed errors', () => {
MethodApiLogic.actions.makeRequest({ indexName: 'my-index', language: 'Universal' });
expect(clearFlashMessages).toHaveBeenCalled();
});
});
describe('apiError', () => {
it('displays the error to the user', () => {
const error = {} as HttpError;
MethodApiLogic.actions.apiError(error);
expect(flashAPIErrors).toHaveBeenCalledWith(error);
});
});
});
});

View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kea, MakeLogicType } from 'kea';
import { Actions } from '../../../../shared/api_logic/create_api_logic';
import { generateEncodedPath } from '../../../../shared/encode_path_params';
import { clearFlashMessages, flashAPIErrors } from '../../../../shared/flash_messages';
import { KibanaLogic } from '../../../../shared/kibana';
import {
CreateApiIndexApiLogic,
CreateApiIndexApiLogicArgs,
CreateApiIndexApiLogicResponse,
} from '../../../api/index/create_api_index_api_logic';
import { SEARCH_INDEX_TAB_PATH } from '../../../routes';
import { SearchIndexTabId } from '../../search_index/search_index';
type MethodApiActions = Pick<
Actions<CreateApiIndexApiLogicArgs, CreateApiIndexApiLogicResponse>,
'apiError' | 'apiSuccess' | 'makeRequest'
>;
export const MethodApiLogic = kea<MakeLogicType<{}, MethodApiActions>>({
connect: {
actions: [CreateApiIndexApiLogic, ['apiError', 'apiSuccess', 'makeRequest']],
},
listeners: {
apiError: (error) => {
flashAPIErrors(error);
},
apiSuccess: ({ indexName }) => {
KibanaLogic.values.navigateToUrl(
generateEncodedPath(SEARCH_INDEX_TAB_PATH, {
indexName,
tabId: SearchIndexTabId.OVERVIEW,
})
);
},
makeRequest: () => clearFlashMessages(),
},
path: ['enterprise_search', 'method_api'],
});

View file

@ -68,7 +68,7 @@ export const MethodConnector: React.FC = () => {
const { error, status } = useValues(AddConnectorPackageApiLogic);
const { isModalVisible } = useValues(AddConnectorPackageLogic);
const { setIsModalVisible } = useActions(AddConnectorPackageLogic);
const { fullIndexName } = useValues(NewSearchIndexLogic);
const { fullIndexName, language } = useValues(NewSearchIndexLogic);
const confirmModal = isModalVisible && (
<EuiConfirmModal
@ -84,7 +84,7 @@ export const MethodConnector: React.FC = () => {
}}
onConfirm={(event) => {
event.preventDefault();
makeRequest({ deleteExistingConnector: true, indexName: fullIndexName });
makeRequest({ deleteExistingConnector: true, indexName: fullIndexName, language });
}}
cancelButtonText={i18n.translate(
'xpack.enterpriseSearch.content.newIndex.steps.buildConnector.confirmModal.cancelButton.label',
@ -123,7 +123,7 @@ export const MethodConnector: React.FC = () => {
onNameChange={() => {
apiReset();
}}
onSubmit={(name) => makeRequest({ indexName: name })}
onSubmit={(name, lang) => makeRequest({ indexName: name, language: lang })}
buttonLoading={status === Status.LOADING}
>
<EuiSteps

View file

@ -15,8 +15,7 @@ import { HttpError } from '../../../../../../common/types/api';
import { MethodCrawlerLogic } from './method_crawler_logic';
// Failing https://github.com/elastic/kibana/issues/135440
describe.skip('MethodCrawlerLogic', () => {
describe('MethodCrawlerLogic', () => {
const { mount } = new LogicMounter(MethodCrawlerLogic);
const { clearFlashMessages, flashAPIErrors } = mockFlashMessageHelpers;
const { navigateToUrl } = mockKibanaValues;
@ -31,7 +30,7 @@ describe.skip('MethodCrawlerLogic', () => {
it('navigates user to index detail view', () => {
MethodCrawlerLogic.actions.apiSuccess({ created: 'my-index' });
expect(navigateToUrl).toHaveBeenCalledWith('/search_indices/my-index');
expect(navigateToUrl).toHaveBeenCalledWith('/search_indices/my-index/domain_management');
});
});

View file

@ -30,7 +30,7 @@ import { baseBreadcrumbs } from '../search_indices';
import { ButtonGroup, ButtonGroupOption } from './button_group';
import { SearchIndexEmptyState } from './empty_state';
import { MethodApi } from './method_api';
import { MethodApi } from './method_api/method_api';
import { MethodConnector } from './method_connector/method_connector';
import { MethodCrawler } from './method_crawler/method_crawler';

View file

@ -46,13 +46,17 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).resolves.toEqual({ id: 'fakeId', index_name: 'index_name' });
expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
document: {
api_key_id: null,
configuration: {},
index_name: 'index_name',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: null,
@ -73,7 +77,10 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => undefined);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).rejects.toEqual(new Error(ErrorCode.INDEX_ALREADY_EXISTS));
expect(mockClient.asCurrentUser.indices.create).not.toHaveBeenCalled();
});
@ -84,7 +91,10 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => true);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).rejects.toEqual(new Error(ErrorCode.CONNECTOR_DOCUMENT_ALREADY_EXISTS));
expect(mockClient.asCurrentUser.indices.create).not.toHaveBeenCalled();
});
@ -95,7 +105,10 @@ describe('addConnector lib function', () => {
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => true);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).rejects.toEqual(new Error(ErrorCode.INDEX_ALREADY_EXISTS));
expect(mockClient.asCurrentUser.indices.create).not.toHaveBeenCalled();
});
@ -109,6 +122,7 @@ describe('addConnector lib function', () => {
addConnector(mockClient as unknown as IScopedClusterClient, {
delete_existing_connector: true,
index_name: 'index_name',
language: null,
})
).resolves.toEqual({ id: 'fakeId', index_name: 'index_name' });
expect(mockClient.asCurrentUser.delete).toHaveBeenCalledWith({
@ -120,6 +134,7 @@ describe('addConnector lib function', () => {
api_key_id: null,
configuration: {},
index_name: 'index_name',
language: null,
last_seen: null,
last_sync_error: null,
last_sync_status: null,
@ -144,7 +159,10 @@ describe('addConnector lib function', () => {
mockClient.asCurrentUser.indices.exists.mockImplementation(() => false);
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => false);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).resolves.toEqual({ id: 'fakeId', index_name: 'index_name' });
expect(setupConnectorsIndices as jest.Mock).toHaveBeenCalledWith(mockClient.asCurrentUser);
expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
@ -152,6 +170,7 @@ describe('addConnector lib function', () => {
api_key_id: null,
configuration: {},
index_name: 'index_name',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: null,
@ -163,7 +182,7 @@ describe('addConnector lib function', () => {
},
index: CONNECTORS_INDEX,
});
expect(mockClient.asCurrentUser.index).toHaveBeenCalledTimes(2);
expect(mockClient.asCurrentUser.indices.create).toHaveBeenCalledWith({ index: 'index_name' });
});
it('should not create index if status code is not 404', async () => {
mockClient.asCurrentUser.index.mockImplementationOnce(() => {
@ -172,7 +191,10 @@ describe('addConnector lib function', () => {
mockClient.asCurrentUser.indices.exists.mockImplementation(() => false);
(fetchConnectorByIndexName as jest.Mock).mockImplementation(() => false);
await expect(
addConnector(mockClient as unknown as IScopedClusterClient, { index_name: 'index_name' })
addConnector(mockClient as unknown as IScopedClusterClient, {
index_name: 'index_name',
language: 'en',
})
).rejects.toEqual({ statusCode: 500 });
expect(setupConnectorsIndices).not.toHaveBeenCalled();
expect(mockClient.asCurrentUser.index).toHaveBeenCalledTimes(1);

View file

@ -20,6 +20,7 @@ import { fetchConnectorByIndexName } from './fetch_connectors';
const createConnector = async (
document: ConnectorDocument,
client: IScopedClusterClient,
language: string | null,
deleteExisting: boolean
): Promise<{ id: string; index_name: string }> => {
const index = document.index_name;
@ -42,7 +43,7 @@ const createConnector = async (
document,
index: CONNECTORS_INDEX,
});
await client.asCurrentUser.indices.create({ index: document.index_name });
await client.asCurrentUser.indices.create({ index });
await client.asCurrentUser.indices.refresh({ index: CONNECTORS_INDEX });
return { id: result._id, index_name: document.index_name };
@ -50,12 +51,13 @@ const createConnector = async (
export const addConnector = async (
client: IScopedClusterClient,
input: { delete_existing_connector?: boolean; index_name: string }
input: { delete_existing_connector?: boolean; index_name: string; language: string | null }
): Promise<{ id: string; index_name: string }> => {
const document: ConnectorDocument = {
api_key_id: null,
configuration: {},
index_name: input.index_name,
language: input.language,
last_seen: null,
last_sync_error: null,
last_sync_status: null,
@ -66,13 +68,18 @@ export const addConnector = async (
sync_now: false,
};
try {
return await createConnector(document, client, !!input.delete_existing_connector);
return await createConnector(
document,
client,
input.language,
!!input.delete_existing_connector
);
} catch (error) {
if (isIndexNotFoundException(error)) {
// This means .ent-search-connectors index doesn't exist yet
// So we first have to create it, and then try inserting the document again
await setupConnectorsIndices(client.asCurrentUser);
return await createConnector(document, client, false);
return await createConnector(document, client, input.language, false);
} else {
throw error;
}

View file

@ -11,23 +11,23 @@ import { IScopedClusterClient } from '@kbn/core/server';
import { textAnalysisSettings } from './text_analysis';
const prefixMapping: MappingTextProperty = {
search_analyzer: 'q_prefix',
analyzer: 'i_prefix',
type: 'text',
index_options: 'docs',
search_analyzer: 'q_prefix',
type: 'text',
};
const delimiterMapping: MappingTextProperty = {
analyzer: 'iq_text_delimiter',
type: 'text',
index_options: 'freqs',
type: 'text',
};
const joinedMapping: MappingTextProperty = {
search_analyzer: 'q_text_bigram',
analyzer: 'i_text_bigram',
type: 'text',
index_options: 'freqs',
search_analyzer: 'q_text_bigram',
type: 'text',
};
const enumMapping: MappingKeywordProperty = {
@ -45,17 +45,17 @@ const defaultMappings = {
dynamic_templates: [
{
all_text_fields: {
match_mapping_type: 'string',
mapping: {
analyzer: 'iq_text_base',
fields: {
prefix: prefixMapping,
delimiter: delimiterMapping,
joined: joinedMapping,
enum: enumMapping,
joined: joinedMapping,
prefix: prefixMapping,
stem: stemMapping,
},
},
match_mapping_type: 'string',
},
},
],
@ -64,13 +64,13 @@ const defaultMappings = {
export const createApiIndex = async (
client: IScopedClusterClient,
indexName: string,
language: string | undefined
language: string | undefined | null
) => {
return await client.asCurrentUser.indices.create({
index: indexName,
body: {
mappings: defaultMappings,
settings: textAnalysisSettings(language),
settings: textAnalysisSettings(language ?? undefined),
},
index: indexName,
});
};

View file

@ -25,6 +25,7 @@ export function registerConnectorRoutes({ router }: RouteDependencies) {
body: schema.object({
delete_existing_connector: schema.maybe(schema.boolean()),
index_name: schema.string(),
language: schema.nullable(schema.string()),
}),
},
},

View file

@ -187,13 +187,13 @@ export function registerIndexRoutes({ router }: RouteDependencies) {
path: '/internal/enterprise_search/indices',
validate: {
body: schema.object({
indexName: schema.string(),
language: schema.maybe(schema.string()),
index_name: schema.string(),
language: schema.maybe(schema.nullable(schema.string())),
}),
},
},
async (context, request, response) => {
const { indexName, language } = request.body;
const { ['index_name']: indexName, language } = request.body;
const { client } = (await context.core).elasticsearch;
try {
const createIndexResponse = await createApiIndex(client, indexName, language);