From 5a03678b9841d0f0869e69bb7df8de099f1aed0e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 28 Feb 2023 18:28:27 -0500 Subject: [PATCH] [8.7] [Enterprise Search] Show error for crawlers without a connector document (#150928) (#152400) # Backport This will backport the following commits from `main` to `8.7`: - [[Enterprise Search] Show error for crawlers without a connector document (#150928)](https://github.com/elastic/kibana/pull/150928) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Sander Philipse <94373878+sphilipse@users.noreply.github.com> --- ...create_crawler_connector_api_logic.test.ts | 33 +++ .../recreate_crawler_connector_api_logic.ts | 33 +++ .../api/index/delete_index_api_logic.ts | 4 +- .../header_actions/header_actions.tsx | 2 +- .../crawler/no_connector_record.tsx | 94 +++++++++ .../crawler/no_connector_record_logic.test.ts | 37 ++++ .../crawler/no_connector_record_logic.ts | 48 +++++ .../components/search_index/search_index.tsx | 17 +- .../search_indices/indices_logic.test.ts | 4 +- .../search_indices/indices_logic.ts | 10 +- .../search_indices/indices_table.tsx | 4 +- .../utils/indices.ts | 2 +- .../server/lib/connectors/add_connector.ts | 98 ++------- .../server/lib/crawler/post_connector.test.ts | 108 ++++++++++ .../server/lib/crawler/post_connector.ts | 35 ++++ .../enterprise_search/crawler/crawler.ts | 32 +++ .../utils/create_connector_document.test.ts | 195 ++++++++++++++++++ .../server/utils/create_connector_document.ts | 103 +++++++++ 18 files changed, 757 insertions(+), 102 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.test.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.ts create mode 100644 x-pack/plugins/enterprise_search/server/utils/create_connector_document.test.ts create mode 100644 x-pack/plugins/enterprise_search/server/utils/create_connector_document.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.test.ts new file mode 100644 index 000000000000..340a089bf3c1 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockHttpValues } from '../../../__mocks__/kea_logic'; + +import { nextTick } from '@kbn/test-jest-helpers'; + +import { recreateCrawlerConnector } from './recreate_crawler_connector_api_logic'; + +describe('CreateCrawlerIndexApiLogic', () => { + const { http } = mockHttpValues; + beforeEach(() => { + jest.clearAllMocks(); + }); + describe('createCrawlerIndex', () => { + it('calls correct api', async () => { + const indexName = 'elastic-co-crawler'; + http.post.mockReturnValue(Promise.resolve({ connector_id: 'connectorId' })); + + const result = recreateCrawlerConnector({ indexName }); + await nextTick(); + + expect(http.post).toHaveBeenCalledWith( + '/internal/enterprise_search/indices/elastic-co-crawler/crawler/connector' + ); + await expect(result).resolves.toEqual({ connector_id: 'connectorId' }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.ts new file mode 100644 index 000000000000..6982850b0066 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { HttpLogic } from '../../../shared/http'; + +export interface RecreateCrawlerConnectorArgs { + indexName: string; +} + +export interface RecreateCrawlerConnectorResponse { + created: string; // the name of the newly created index +} + +export const recreateCrawlerConnector = async ({ indexName }: RecreateCrawlerConnectorArgs) => { + const route = `/internal/enterprise_search/indices/${indexName}/crawler/connector`; + + return await HttpLogic.values.http.post(route); +}; + +export const RecreateCrawlerConnectorApiLogic = createApiLogic( + ['recreate_crawler_connector_api_logic'], + recreateCrawlerConnector +); + +export type RecreateCrawlerConnectorActions = Actions< + RecreateCrawlerConnectorArgs, + RecreateCrawlerConnectorResponse +>; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/delete_index_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/delete_index_api_logic.ts index 391e3e102383..7c3f80b05e7a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/delete_index_api_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/delete_index_api_logic.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; -import { createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic'; import { HttpLogic } from '../../../shared/http'; export interface DeleteIndexApiLogicArgs { @@ -36,3 +36,5 @@ export const DeleteIndexApiLogic = createApiLogic(['delete_index_api_logic'], de }, }), }); + +export type DeleteIndexApiActions = Actions; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/components/header_actions/header_actions.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/components/header_actions/header_actions.tsx index 0bf12fa6e200..f6cb956478fd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/components/header_actions/header_actions.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/components/header_actions/header_actions.tsx @@ -18,7 +18,7 @@ import { SyncsContextMenu } from './syncs_context_menu'; export const getHeaderActions = (indexData?: ElasticsearchIndexWithIngestion) => { const ingestionMethod = getIngestionMethod(indexData); return [ - ...(isCrawlerIndex(indexData) ? [] : []), + ...(isCrawlerIndex(indexData) && indexData.connector ? [] : []), ...(isConnectorIndex(indexData) ? [] : []), { + const { indexName } = useValues(IndexViewLogic); + const { isDeleteLoading } = useValues(IndicesLogic); + const { openDeleteModal } = useActions(IndicesLogic); + const { makeRequest } = useActions(RecreateCrawlerConnectorApiLogic); + const { status } = useValues(RecreateCrawlerConnectorApiLogic); + NoConnectorRecordLogic.mount(); + const buttonsDisabled = status === Status.LOADING || isDeleteLoading; + + return ( + <> + + + + {i18n.translate( + 'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.title', + { + defaultMessage: "This index's connector configuration has been removed", + } + )} + + } + body={ +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.description', + { + defaultMessage: + 'We could not find a connector configuration for this crawler index. The record should be recreated, or the index should be deleted.', + } + )} +

+ } + actions={[ + makeRequest({ indexName })} + > + {i18n.translate( + 'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.recreateConnectorRecord', + { + defaultMessage: 'Recreate connector record', + } + )} + , + openDeleteModal(indexName)} + > + {i18n.translate( + 'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.deleteIndex', + { + defaultMessage: 'Delete index', + } + )} + , + ]} + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.test.ts new file mode 100644 index 000000000000..3d9ce1c235fb --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter } from '../../../../__mocks__/kea_logic'; + +import { KibanaLogic } from '../../../../shared/kibana'; + +import { RecreateCrawlerConnectorApiLogic } from '../../../api/crawler/recreate_crawler_connector_api_logic'; +import { DeleteIndexApiLogic } from '../../../api/index/delete_index_api_logic'; +import { SEARCH_INDICES_PATH } from '../../../routes'; + +import { NoConnectorRecordLogic } from './no_connector_record_logic'; + +describe('NoConnectorRecordLogic', () => { + const { mount: deleteMount } = new LogicMounter(DeleteIndexApiLogic); + const { mount: recreateMount } = new LogicMounter(RecreateCrawlerConnectorApiLogic); + const { mount } = new LogicMounter(NoConnectorRecordLogic); + beforeEach(() => { + deleteMount(); + recreateMount(); + mount(); + }); + it('should redirect to search indices on delete', () => { + KibanaLogic.values.navigateToUrl = jest.fn(); + DeleteIndexApiLogic.actions.apiSuccess({} as any); + expect(KibanaLogic.values.navigateToUrl).toHaveBeenCalledWith(SEARCH_INDICES_PATH); + }); + it('should fetch index on recreate', () => { + NoConnectorRecordLogic.actions.fetchIndex = jest.fn(); + RecreateCrawlerConnectorApiLogic.actions.apiSuccess({} as any); + expect(NoConnectorRecordLogic.actions.fetchIndex).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.ts new file mode 100644 index 000000000000..a3d1d1a653ec --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { KibanaLogic } from '../../../../shared/kibana'; + +import { + RecreateCrawlerConnectorActions, + RecreateCrawlerConnectorApiLogic, +} from '../../../api/crawler/recreate_crawler_connector_api_logic'; +import { + DeleteIndexApiActions, + DeleteIndexApiLogic, +} from '../../../api/index/delete_index_api_logic'; +import { SEARCH_INDICES_PATH } from '../../../routes'; +import { IndexViewActions, IndexViewLogic } from '../index_view_logic'; + +type NoConnectorRecordActions = RecreateCrawlerConnectorActions['apiSuccess'] & { + deleteSuccess: DeleteIndexApiActions['apiSuccess']; + fetchIndex: IndexViewActions['fetchIndex']; +}; + +export const NoConnectorRecordLogic = kea>({ + connect: { + actions: [ + RecreateCrawlerConnectorApiLogic, + ['apiSuccess'], + IndexViewLogic, + ['fetchIndex'], + DeleteIndexApiLogic, + ['apiSuccess as deleteSuccess'], + ], + }, + listeners: ({ actions }) => ({ + apiSuccess: () => { + actions.fetchIndex(); + }, + deleteSuccess: () => { + KibanaLogic.values.navigateToUrl(SEARCH_INDICES_PATH); + }, + }), + path: ['enterprise_search', 'content', 'no_connector_record'], +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx index 0643cce0f817..52ae6628f808 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx @@ -38,6 +38,7 @@ import { AutomaticCrawlScheduler } from './crawler/automatic_crawl_scheduler/aut import { CrawlCustomSettingsFlyout } from './crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout'; import { CrawlerConfiguration } from './crawler/crawler_configuration/crawler_configuration'; import { SearchIndexDomainManagement } from './crawler/domain_management/domain_management'; +import { NoConnectorRecord } from './crawler/no_connector_record'; import { SearchIndexDocuments } from './documents'; import { SearchIndexIndexMappings } from './index_mappings'; import { IndexNameLogic } from './index_name_logic'; @@ -224,12 +225,16 @@ export const SearchIndex: React.FC = () => { rightSideItems: getHeaderActions(index), }} > - <> - {indexName === index?.name && ( - - )} - {isCrawlerIndex(index) && } - + {isCrawlerIndex(index) && !index.connector ? ( + + ) : ( + <> + {indexName === index?.name && ( + + )} + {isCrawlerIndex(index) && } + + )} ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.test.ts index 16ee5faff914..89043f1d7c0c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.test.ts @@ -85,7 +85,7 @@ describe('IndicesLogic', () => { describe('openDeleteModal', () => { it('should set deleteIndexName and set isDeleteModalVisible to true', () => { IndicesLogic.actions.fetchIndexDetails = jest.fn(); - IndicesLogic.actions.openDeleteModal(connectorIndex); + IndicesLogic.actions.openDeleteModal(connectorIndex.name); expect(IndicesLogic.values).toEqual({ ...DEFAULT_VALUES, deleteModalIndexName: 'connector', @@ -98,7 +98,7 @@ describe('IndicesLogic', () => { }); describe('closeDeleteModal', () => { it('should set deleteIndexName to empty and set isDeleteModalVisible to false', () => { - IndicesLogic.actions.openDeleteModal(connectorIndex); + IndicesLogic.actions.openDeleteModal(connectorIndex.name); IndicesLogic.actions.fetchIndexDetails = jest.fn(); IndicesLogic.actions.closeDeleteModal(); expect(IndicesLogic.values).toEqual({ diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.ts index 953fce853904..9489a005d0b1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.ts @@ -69,7 +69,7 @@ export interface IndicesActions { }): { meta: Meta; returnHiddenIndices: boolean; searchQuery?: string }; makeRequest: typeof FetchIndicesAPILogic.actions.makeRequest; onPaginate(newPageIndex: number): { newPageIndex: number }; - openDeleteModal(index: ElasticsearchViewIndex): { index: ElasticsearchViewIndex }; + openDeleteModal(indexName: string): { indexName: string }; setIsFirstRequest(): void; } export interface IndicesValues { @@ -102,7 +102,7 @@ export const IndicesLogic = kea>({ searchQuery, }), onPaginate: (newPageIndex) => ({ newPageIndex }), - openDeleteModal: (index) => ({ index }), + openDeleteModal: (indexName) => ({ indexName }), setIsFirstRequest: true, }, connect: { @@ -137,8 +137,8 @@ export const IndicesLogic = kea>({ await breakpoint(150); actions.makeRequest(input); }, - openDeleteModal: ({ index }) => { - actions.fetchIndexDetails({ indexName: index.name }); + openDeleteModal: ({ indexName }) => { + actions.fetchIndexDetails({ indexName }); }, }), path: ['enterprise_search', 'content', 'indices_logic'], @@ -147,7 +147,7 @@ export const IndicesLogic = kea>({ '', { closeDeleteModal: () => '', - openDeleteModal: (_, { index: { name } }) => name, + openDeleteModal: (_, { indexName }) => indexName, }, ], isDeleteModalVisible: [ diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx index fd0bc55fdc7e..c73f567f31ed 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx @@ -38,7 +38,7 @@ interface IndicesTableProps { isLoading?: boolean; meta: Meta; onChange: (criteria: CriteriaWithPagination) => void; - onDelete: (index: ElasticsearchViewIndex) => void; + onDelete: (indexName: string) => void; } export const IndicesTable: React.FC = ({ @@ -175,7 +175,7 @@ export const IndicesTable: React.FC = ({ }, } ), - onClick: (index) => onDelete(index), + onClick: (index) => onDelete(index.name), type: 'icon', }, ], diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts index fd87b60bd99a..f88620faf5b1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts @@ -81,7 +81,7 @@ export function getIngestionStatus(index?: ElasticsearchIndexWithIngestion): Ing if (!index || isApiIndex(index)) { return IngestionStatus.CONNECTED; } - if (isConnectorIndex(index) || isCrawlerIndex(index)) { + if (isConnectorIndex(index) || (isCrawlerIndex(index) && index.connector)) { if ( index.connector.last_seen && moment(index.connector.last_seen).isBefore(moment().subtract(30, 'minutes')) diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts index 507ec3fd0bb4..2019cd06e008 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts @@ -8,18 +8,13 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { CONNECTORS_INDEX, CONNECTORS_VERSION } from '../..'; -import { - ConnectorDocument, - ConnectorStatus, - FilteringPolicy, - FilteringRuleRule, - FilteringValidationState, -} from '../../../common/types/connectors'; +import { ConnectorDocument } from '../../../common/types/connectors'; import { ErrorCode } from '../../../common/types/error_codes'; import { DefaultConnectorsPipelineMeta, setupConnectorsIndices, } from '../../index_management/setup_indices'; +import { createConnectorDocument } from '../../utils/create_connector_document'; import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers'; import { createIndex } from '../indices/create_index'; @@ -85,89 +80,24 @@ export const addConnector = async ( const connectorsIndicesMapping = await client.asCurrentUser.indices.getMapping({ index: CONNECTORS_INDEX, }); - const connectorsPipelineMeta: DefaultConnectorsPipelineMeta = + const pipeline: DefaultConnectorsPipelineMeta = connectorsIndicesMapping[`${CONNECTORS_INDEX}-v${CONNECTORS_VERSION}`]?.mappings?._meta ?.pipeline; - const currentTimestamp = new Date().toISOString(); - const document: ConnectorDocument = { - api_key_id: null, - configuration: {}, - custom_scheduling: {}, - description: null, - error: null, - features: null, - filtering: [ - { - active: { - advanced_snippet: { - created_at: currentTimestamp, - updated_at: currentTimestamp, - value: {}, - }, - rules: [ - { - created_at: currentTimestamp, - field: '_', - id: 'DEFAULT', - order: 0, - policy: FilteringPolicy.INCLUDE, - rule: FilteringRuleRule.REGEX, - updated_at: currentTimestamp, - value: '.*', - }, - ], - validation: { - errors: [], - state: FilteringValidationState.VALID, - }, - }, - domain: 'DEFAULT', - draft: { - advanced_snippet: { - created_at: currentTimestamp, - updated_at: currentTimestamp, - value: {}, - }, - rules: [ - { - created_at: currentTimestamp, - field: '_', - id: 'DEFAULT', - order: 0, - policy: FilteringPolicy.INCLUDE, - rule: FilteringRuleRule.REGEX, - updated_at: currentTimestamp, - value: '.*', - }, - ], - validation: { - errors: [], - state: FilteringValidationState.VALID, - }, - }, - }, - ], - index_name: input.index_name, - is_native: input.is_native, + const document = createConnectorDocument({ + indexName: input.index_name, + isNative: input.is_native, language: input.language, - last_seen: null, - last_sync_error: null, - last_sync_status: null, - last_synced: null, - name: input.index_name.startsWith('search-') ? input.index_name.substring(7) : input.index_name, - pipeline: connectorsPipelineMeta + pipeline: pipeline ? { - extract_binary_content: connectorsPipelineMeta.default_extract_binary_content, - name: connectorsPipelineMeta.default_name, - reduce_whitespace: connectorsPipelineMeta.default_reduce_whitespace, - run_ml_inference: connectorsPipelineMeta.default_run_ml_inference, + extract_binary_content: pipeline.default_extract_binary_content, + name: pipeline.default_name, + reduce_whitespace: pipeline.default_reduce_whitespace, + run_ml_inference: pipeline.default_run_ml_inference, } : null, - scheduling: { enabled: false, interval: '0 0 0 * * ?' }, - service_type: input.service_type || null, - status: ConnectorStatus.CREATED, - sync_now: false, - }; + serviceType: input.service_type, + }); + return await createConnector(document, client, input.language, !!input.delete_existing_connector); }; diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.test.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.test.ts new file mode 100644 index 000000000000..729b72e8f643 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.test.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core/server'; + +import { CONNECTORS_INDEX } from '../..'; +import { ConnectorStatus } from '../../../common/types/connectors'; + +import { recreateConnectorDocument } from './post_connector'; + +describe('recreateConnectorDocument lib function', () => { + const mockClient = { + asCurrentUser: { + index: jest.fn(), + }, + asInternalUser: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should recreate connector document', async () => { + mockClient.asCurrentUser.index.mockResolvedValue({ _id: 'connectorId' }); + + await recreateConnectorDocument(mockClient as unknown as IScopedClusterClient, 'indexName'); + expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({ + document: { + api_key_id: null, + configuration: {}, + custom_scheduling: {}, + description: null, + error: null, + features: null, + filtering: [ + { + active: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + domain: 'DEFAULT', + draft: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + }, + ], + index_name: 'indexName', + is_native: false, + language: '', + last_seen: null, + last_sync_error: null, + last_sync_status: null, + last_synced: null, + name: 'indexName', + pipeline: null, + scheduling: { enabled: false, interval: '0 0 0 * * ?' }, + service_type: 'elastic-crawler', + status: ConnectorStatus.CONFIGURED, + sync_now: false, + }, + index: CONNECTORS_INDEX, + refresh: 'wait_for', + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.ts new file mode 100644 index 000000000000..17bf6945d0d8 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +import { CONNECTORS_INDEX } from '../..'; + +import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../common/constants'; +import { ConnectorStatus } from '../../../common/types/connectors'; + +import { createConnectorDocument } from '../../utils/create_connector_document'; + +export const recreateConnectorDocument = async ( + client: IScopedClusterClient, + indexName: string +) => { + const document = createConnectorDocument({ + indexName, + isNative: false, + // The search index has already been created so we don't need the language, which we can't retrieve anymore anyway + language: '', + pipeline: null, + serviceType: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE, + }); + const result = await client.asCurrentUser.index({ + document: { ...document, status: ConnectorStatus.CONFIGURED }, + index: CONNECTORS_INDEX, + refresh: 'wait_for', + }); + return result._id; +}; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts index 08cea961709f..8d0c1e73df84 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts @@ -16,6 +16,7 @@ import { addConnector } from '../../../lib/connectors/add_connector'; import { deleteConnectorById } from '../../../lib/connectors/delete_connector'; import { fetchConnectorByIndexName } from '../../../lib/connectors/fetch_connectors'; import { fetchCrawlerByIndexName } from '../../../lib/crawler/fetch_crawlers'; +import { recreateConnectorDocument } from '../../../lib/crawler/post_connector'; import { updateHtmlExtraction } from '../../../lib/crawler/put_html_extraction'; import { deleteIndex } from '../../../lib/indices/delete_index'; import { RouteDependencies } from '../../../plugin'; @@ -429,6 +430,37 @@ export function registerCrawlerRoutes(routeDependencies: RouteDependencies) { }) ); + router.post( + { + path: '/internal/enterprise_search/indices/{indexName}/crawler/connector', + validate: { + params: schema.object({ + indexName: schema.string(), + }), + }, + }, + elasticsearchErrorHandler(log, async (context, request, response) => { + const { client } = (await context.core).elasticsearch; + const connector = await fetchConnectorByIndexName(client, request.params.indexName); + if (connector) { + return createError({ + errorCode: ErrorCode.CONNECTOR_DOCUMENT_ALREADY_EXISTS, + message: i18n.translate( + 'xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError', + { + defaultMessage: 'A connector for this index already exists', + } + ), + response, + statusCode: 409, + }); + } + + const connectorId = await recreateConnectorDocument(client, request.params.indexName); + return response.ok({ body: { connector_id: connectorId } }); + }) + ); + registerCrawlerCrawlRulesRoutes(routeDependencies); registerCrawlerEntryPointRoutes(routeDependencies); registerCrawlerSitemapRoutes(routeDependencies); diff --git a/x-pack/plugins/enterprise_search/server/utils/create_connector_document.test.ts b/x-pack/plugins/enterprise_search/server/utils/create_connector_document.test.ts new file mode 100644 index 000000000000..62851572cf79 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/utils/create_connector_document.test.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConnectorStatus } from '../../common/types/connectors'; + +import { createConnectorDocument } from './create_connector_document'; + +describe('createConnectorDocument', () => { + it('should create a connector document', () => { + expect( + createConnectorDocument({ + indexName: 'indexName', + isNative: false, + language: 'fr', + pipeline: { + extract_binary_content: true, + name: 'ent-search-generic-ingestion', + reduce_whitespace: true, + run_ml_inference: false, + }, + }) + ).toEqual({ + api_key_id: null, + configuration: {}, + custom_scheduling: {}, + description: null, + error: null, + features: null, + filtering: [ + { + active: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + domain: 'DEFAULT', + draft: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + }, + ], + index_name: 'indexName', + is_native: false, + language: 'fr', + last_seen: null, + last_sync_error: null, + last_sync_status: null, + last_synced: null, + name: 'indexName', + pipeline: { + extract_binary_content: true, + name: 'ent-search-generic-ingestion', + reduce_whitespace: true, + run_ml_inference: false, + }, + scheduling: { enabled: false, interval: '0 0 0 * * ?' }, + service_type: null, + status: ConnectorStatus.CREATED, + sync_now: false, + }); + }); + it('should remove search- from name', () => { + expect( + createConnectorDocument({ + indexName: 'search-indexName', + isNative: false, + language: 'fr', + pipeline: { + extract_binary_content: true, + name: 'ent-search-generic-ingestion', + reduce_whitespace: true, + run_ml_inference: false, + }, + }) + ).toEqual({ + api_key_id: null, + configuration: {}, + custom_scheduling: {}, + description: null, + error: null, + features: null, + filtering: [ + { + active: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + domain: 'DEFAULT', + draft: { + advanced_snippet: { + created_at: expect.any(String), + updated_at: expect.any(String), + value: {}, + }, + rules: [ + { + created_at: expect.any(String), + field: '_', + id: 'DEFAULT', + order: 0, + policy: 'include', + rule: 'regex', + updated_at: expect.any(String), + value: '.*', + }, + ], + validation: { + errors: [], + state: 'valid', + }, + }, + }, + ], + index_name: 'search-indexName', + is_native: false, + language: 'fr', + last_seen: null, + last_sync_error: null, + last_sync_status: null, + last_synced: null, + name: 'indexName', + pipeline: { + extract_binary_content: true, + name: 'ent-search-generic-ingestion', + reduce_whitespace: true, + run_ml_inference: false, + }, + scheduling: { enabled: false, interval: '0 0 0 * * ?' }, + service_type: null, + status: ConnectorStatus.CREATED, + sync_now: false, + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/utils/create_connector_document.ts b/x-pack/plugins/enterprise_search/server/utils/create_connector_document.ts new file mode 100644 index 000000000000..d4776e3941ce --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/utils/create_connector_document.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ConnectorDocument, + ConnectorStatus, + FilteringPolicy, + FilteringRuleRule, + FilteringValidationState, + IngestPipelineParams, +} from '../../common/types/connectors'; + +export function createConnectorDocument({ + indexName, + isNative, + pipeline, + serviceType, + language, +}: { + indexName: string; + isNative: boolean; + language: string | null; + pipeline?: IngestPipelineParams | null; + serviceType?: string | null; +}): ConnectorDocument { + const currentTimestamp = new Date().toISOString(); + return { + api_key_id: null, + configuration: {}, + custom_scheduling: {}, + description: null, + error: null, + features: null, + filtering: [ + { + active: { + advanced_snippet: { + created_at: currentTimestamp, + updated_at: currentTimestamp, + value: {}, + }, + rules: [ + { + created_at: currentTimestamp, + field: '_', + id: 'DEFAULT', + order: 0, + policy: FilteringPolicy.INCLUDE, + rule: FilteringRuleRule.REGEX, + updated_at: currentTimestamp, + value: '.*', + }, + ], + validation: { + errors: [], + state: FilteringValidationState.VALID, + }, + }, + domain: 'DEFAULT', + draft: { + advanced_snippet: { + created_at: currentTimestamp, + updated_at: currentTimestamp, + value: {}, + }, + rules: [ + { + created_at: currentTimestamp, + field: '_', + id: 'DEFAULT', + order: 0, + policy: FilteringPolicy.INCLUDE, + rule: FilteringRuleRule.REGEX, + updated_at: currentTimestamp, + value: '.*', + }, + ], + validation: { + errors: [], + state: FilteringValidationState.VALID, + }, + }, + }, + ], + index_name: indexName, + is_native: isNative, + language, + last_seen: null, + last_sync_error: null, + last_sync_status: null, + last_synced: null, + name: indexName.startsWith('search-') ? indexName.substring(7) : indexName, + pipeline, + scheduling: { enabled: false, interval: '0 0 0 * * ?' }, + service_type: serviceType || null, + status: ConnectorStatus.CREATED, + sync_now: false, + }; +}