[8.7] [Enterprise Search] Show error for crawlers without a connector document (#150928) (#152400)

# Backport

This will backport the following commits from `main` to `8.7`:
- [[Enterprise Search] Show error for crawlers without a connector
document (#150928)](https://github.com/elastic/kibana/pull/150928)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Sander
Philipse","email":"94373878+sphilipse@users.noreply.github.com"},"sourceCommit":{"committedDate":"2023-02-28T22:23:04Z","message":"[Enterprise
Search] Show error for crawlers without a connector document
(#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers,
where a connector document has\r\nbeen deleted. It offers users the
choice to recreate the connector or\r\ndelete their
index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341","branchLabelMapping":{"^v8.8.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:EnterpriseSearch","v8.7.0","v8.8.0"],"number":150928,"url":"https://github.com/elastic/kibana/pull/150928","mergeCommit":{"message":"[Enterprise
Search] Show error for crawlers without a connector document
(#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers,
where a connector document has\r\nbeen deleted. It offers users the
choice to recreate the connector or\r\ndelete their
index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341"}},"sourceBranch":"main","suggestedTargetBranches":["8.7"],"targetPullRequestStates":[{"branch":"8.7","label":"v8.7.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.8.0","labelRegex":"^v8.8.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/150928","number":150928,"mergeCommit":{"message":"[Enterprise
Search] Show error for crawlers without a connector document
(#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers,
where a connector document has\r\nbeen deleted. It offers users the
choice to recreate the connector or\r\ndelete their
index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341"}}]}]
BACKPORT-->

Co-authored-by: Sander Philipse <94373878+sphilipse@users.noreply.github.com>

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.test.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mockHttpValues } from '../../../__mocks__/kea_logic';
+
+import { nextTick } from '@kbn/test-jest-helpers';
+
+import { recreateCrawlerConnector } from './recreate_crawler_connector_api_logic';
+
+describe('CreateCrawlerIndexApiLogic', () => {
+  const { http } = mockHttpValues;
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  describe('createCrawlerIndex', () => {
+    it('calls correct api', async () => {
+      const indexName = 'elastic-co-crawler';
+      http.post.mockReturnValue(Promise.resolve({ connector_id: 'connectorId' }));
+
+      const result = recreateCrawlerConnector({ indexName });
+      await nextTick();
+
+      expect(http.post).toHaveBeenCalledWith(
+        '/internal/enterprise_search/indices/elastic-co-crawler/crawler/connector'
+      );
+      await expect(result).resolves.toEqual({ connector_id: 'connectorId' });
+    });
+  });
+});

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/recreate_crawler_connector_api_logic.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { HttpLogic } from '../../../shared/http';
+
+export interface RecreateCrawlerConnectorArgs {
+  indexName: string;
+}
+
+export interface RecreateCrawlerConnectorResponse {
+  created: string; // the name of the newly created index
+}
+
+export const recreateCrawlerConnector = async ({ indexName }: RecreateCrawlerConnectorArgs) => {
+  const route = `/internal/enterprise_search/indices/${indexName}/crawler/connector`;
+
+  return await HttpLogic.values.http.post<RecreateCrawlerConnectorResponse>(route);
+};
+
+export const RecreateCrawlerConnectorApiLogic = createApiLogic(
+  ['recreate_crawler_connector_api_logic'],
+  recreateCrawlerConnector
+);
+
+export type RecreateCrawlerConnectorActions = Actions<
+  RecreateCrawlerConnectorArgs,
+  RecreateCrawlerConnectorResponse
+>;

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/delete_index_api_logic.ts

@@ -7,7 +7,7 @@
 
 import { i18n } from '@kbn/i18n';
 
-import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic';
 import { HttpLogic } from '../../../shared/http';
 
 export interface DeleteIndexApiLogicArgs {
@@ -36,3 +36,5 @@ export const DeleteIndexApiLogic = createApiLogic(['delete_index_api_logic'], de
       },
     }),
 });
+
+export type DeleteIndexApiActions = Actions<DeleteIndexApiLogicArgs, DeleteIndexApiLogicValues>;

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/components/header_actions/header_actions.tsx

@@ -18,7 +18,7 @@ import { SyncsContextMenu } from './syncs_context_menu';
 export const getHeaderActions = (indexData?: ElasticsearchIndexWithIngestion) => {
   const ingestionMethod = getIngestionMethod(indexData);
   return [
-    ...(isCrawlerIndex(indexData) ? [<CrawlerStatusIndicator />] : []),
+    ...(isCrawlerIndex(indexData) && indexData.connector ? [<CrawlerStatusIndicator />] : []),
     ...(isConnectorIndex(indexData) ? [<SyncsContextMenu />] : []),
     <SearchEnginesPopover
       indexName={indexData?.name}

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record.tsx

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import { useActions, useValues } from 'kea';
+
+import { EuiButton, EuiPageTemplate } from '@elastic/eui';
+
+import { i18n } from '@kbn/i18n';
+
+import { Status } from '../../../../../../common/types/api';
+
+import { RecreateCrawlerConnectorApiLogic } from '../../../api/crawler/recreate_crawler_connector_api_logic';
+import { DeleteIndexModal } from '../../search_indices/delete_index_modal';
+import { IndicesLogic } from '../../search_indices/indices_logic';
+import { IndexViewLogic } from '../index_view_logic';
+
+import { NoConnectorRecordLogic } from './no_connector_record_logic';
+
+export const NoConnectorRecord: React.FC = () => {
+  const { indexName } = useValues(IndexViewLogic);
+  const { isDeleteLoading } = useValues(IndicesLogic);
+  const { openDeleteModal } = useActions(IndicesLogic);
+  const { makeRequest } = useActions(RecreateCrawlerConnectorApiLogic);
+  const { status } = useValues(RecreateCrawlerConnectorApiLogic);
+  NoConnectorRecordLogic.mount();
+  const buttonsDisabled = status === Status.LOADING || isDeleteLoading;
+
+  return (
+    <>
+      <DeleteIndexModal />
+
+      <EuiPageTemplate.EmptyPrompt
+        iconType="alert"
+        color="danger"
+        title={
+          <h2>
+            {i18n.translate(
+              'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.title',
+              {
+                defaultMessage: "This index's connector configuration has been removed",
+              }
+            )}
+          </h2>
+        }
+        body={
+          <p>
+            {i18n.translate(
+              'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.description',
+              {
+                defaultMessage:
+                  'We could not find a connector configuration for this crawler index. The record should be recreated, or the index should be deleted.',
+              }
+            )}
+          </p>
+        }
+        actions={[
+          <EuiButton
+            color="danger"
+            disabled={buttonsDisabled}
+            isLoading={status === Status.LOADING}
+            onClick={() => makeRequest({ indexName })}
+          >
+            {i18n.translate(
+              'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.recreateConnectorRecord',
+              {
+                defaultMessage: 'Recreate connector record',
+              }
+            )}
+          </EuiButton>,
+          <EuiButton
+            color="danger"
+            disabled={buttonsDisabled}
+            isLoading={isDeleteLoading}
+            fill
+            onClick={() => openDeleteModal(indexName)}
+          >
+            {i18n.translate(
+              'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.deleteIndex',
+              {
+                defaultMessage: 'Delete index',
+              }
+            )}
+          </EuiButton>,
+        ]}
+      />
+    </>
+  );
+};

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.test.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { LogicMounter } from '../../../../__mocks__/kea_logic';
+
+import { KibanaLogic } from '../../../../shared/kibana';
+
+import { RecreateCrawlerConnectorApiLogic } from '../../../api/crawler/recreate_crawler_connector_api_logic';
+import { DeleteIndexApiLogic } from '../../../api/index/delete_index_api_logic';
+import { SEARCH_INDICES_PATH } from '../../../routes';
+
+import { NoConnectorRecordLogic } from './no_connector_record_logic';
+
+describe('NoConnectorRecordLogic', () => {
+  const { mount: deleteMount } = new LogicMounter(DeleteIndexApiLogic);
+  const { mount: recreateMount } = new LogicMounter(RecreateCrawlerConnectorApiLogic);
+  const { mount } = new LogicMounter(NoConnectorRecordLogic);
+  beforeEach(() => {
+    deleteMount();
+    recreateMount();
+    mount();
+  });
+  it('should redirect to search indices on delete', () => {
+    KibanaLogic.values.navigateToUrl = jest.fn();
+    DeleteIndexApiLogic.actions.apiSuccess({} as any);
+    expect(KibanaLogic.values.navigateToUrl).toHaveBeenCalledWith(SEARCH_INDICES_PATH);
+  });
+  it('should fetch index on recreate', () => {
+    NoConnectorRecordLogic.actions.fetchIndex = jest.fn();
+    RecreateCrawlerConnectorApiLogic.actions.apiSuccess({} as any);
+    expect(NoConnectorRecordLogic.actions.fetchIndex).toHaveBeenCalled();
+  });
+});

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/no_connector_record_logic.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { kea, MakeLogicType } from 'kea';
+
+import { KibanaLogic } from '../../../../shared/kibana';
+
+import {
+  RecreateCrawlerConnectorActions,
+  RecreateCrawlerConnectorApiLogic,
+} from '../../../api/crawler/recreate_crawler_connector_api_logic';
+import {
+  DeleteIndexApiActions,
+  DeleteIndexApiLogic,
+} from '../../../api/index/delete_index_api_logic';
+import { SEARCH_INDICES_PATH } from '../../../routes';
+import { IndexViewActions, IndexViewLogic } from '../index_view_logic';
+
+type NoConnectorRecordActions = RecreateCrawlerConnectorActions['apiSuccess'] & {
+  deleteSuccess: DeleteIndexApiActions['apiSuccess'];
+  fetchIndex: IndexViewActions['fetchIndex'];
+};
+
+export const NoConnectorRecordLogic = kea<MakeLogicType<{}, NoConnectorRecordActions>>({
+  connect: {
+    actions: [
+      RecreateCrawlerConnectorApiLogic,
+      ['apiSuccess'],
+      IndexViewLogic,
+      ['fetchIndex'],
+      DeleteIndexApiLogic,
+      ['apiSuccess as deleteSuccess'],
+    ],
+  },
+  listeners: ({ actions }) => ({
+    apiSuccess: () => {
+      actions.fetchIndex();
+    },
+    deleteSuccess: () => {
+      KibanaLogic.values.navigateToUrl(SEARCH_INDICES_PATH);
+    },
+  }),
+  path: ['enterprise_search', 'content', 'no_connector_record'],
+});

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx

@@ -38,6 +38,7 @@ import { AutomaticCrawlScheduler } from './crawler/automatic_crawl_scheduler/aut
 import { CrawlCustomSettingsFlyout } from './crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout';
 import { CrawlerConfiguration } from './crawler/crawler_configuration/crawler_configuration';
 import { SearchIndexDomainManagement } from './crawler/domain_management/domain_management';
+import { NoConnectorRecord } from './crawler/no_connector_record';
 import { SearchIndexDocuments } from './documents';
 import { SearchIndexIndexMappings } from './index_mappings';
 import { IndexNameLogic } from './index_name_logic';
@@ -224,12 +225,16 @@ export const SearchIndex: React.FC = () => {
         rightSideItems: getHeaderActions(index),
       }}
     >
-      <>
+      {isCrawlerIndex(index) && !index.connector ? (
-        {indexName === index?.name && (
+        <NoConnectorRecord />
-          <EuiTabbedContent tabs={tabs} selectedTab={selectedTab} onTabClick={onTabClick} />
+      ) : (
-        )}
+        <>
-        {isCrawlerIndex(index) && <CrawlCustomSettingsFlyout />}
+          {indexName === index?.name && (
-      </>
+            <EuiTabbedContent tabs={tabs} selectedTab={selectedTab} onTabClick={onTabClick} />
+          )}
+          {isCrawlerIndex(index) && <CrawlCustomSettingsFlyout />}
+        </>
+      )}
     </EnterpriseSearchContentPageTemplate>
   );
 };

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.test.ts

@@ -85,7 +85,7 @@ describe('IndicesLogic', () => {
     describe('openDeleteModal', () => {
       it('should set deleteIndexName and set isDeleteModalVisible to true', () => {
         IndicesLogic.actions.fetchIndexDetails = jest.fn();
-        IndicesLogic.actions.openDeleteModal(connectorIndex);
+        IndicesLogic.actions.openDeleteModal(connectorIndex.name);
         expect(IndicesLogic.values).toEqual({
           ...DEFAULT_VALUES,
           deleteModalIndexName: 'connector',
@@ -98,7 +98,7 @@ describe('IndicesLogic', () => {
     });
     describe('closeDeleteModal', () => {
       it('should set deleteIndexName to empty and set isDeleteModalVisible to false', () => {
-        IndicesLogic.actions.openDeleteModal(connectorIndex);
+        IndicesLogic.actions.openDeleteModal(connectorIndex.name);
         IndicesLogic.actions.fetchIndexDetails = jest.fn();
         IndicesLogic.actions.closeDeleteModal();
         expect(IndicesLogic.values).toEqual({

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_logic.ts

@@ -69,7 +69,7 @@ export interface IndicesActions {
   }): { meta: Meta; returnHiddenIndices: boolean; searchQuery?: string };
   makeRequest: typeof FetchIndicesAPILogic.actions.makeRequest;
   onPaginate(newPageIndex: number): { newPageIndex: number };
-  openDeleteModal(index: ElasticsearchViewIndex): { index: ElasticsearchViewIndex };
+  openDeleteModal(indexName: string): { indexName: string };
   setIsFirstRequest(): void;
 }
 export interface IndicesValues {
@@ -102,7 +102,7 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
       searchQuery,
     }),
     onPaginate: (newPageIndex) => ({ newPageIndex }),
-    openDeleteModal: (index) => ({ index }),
+    openDeleteModal: (indexName) => ({ indexName }),
     setIsFirstRequest: true,
   },
   connect: {
@@ -137,8 +137,8 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
       await breakpoint(150);
       actions.makeRequest(input);
     },
-    openDeleteModal: ({ index }) => {
+    openDeleteModal: ({ indexName }) => {
-      actions.fetchIndexDetails({ indexName: index.name });
+      actions.fetchIndexDetails({ indexName });
     },
   }),
   path: ['enterprise_search', 'content', 'indices_logic'],
@@ -147,7 +147,7 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
       '',
       {
         closeDeleteModal: () => '',
-        openDeleteModal: (_, { index: { name } }) => name,
+        openDeleteModal: (_, { indexName }) => indexName,
       },
     ],
     isDeleteModalVisible: [

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx

@@ -38,7 +38,7 @@ interface IndicesTableProps {
   isLoading?: boolean;
   meta: Meta;
   onChange: (criteria: CriteriaWithPagination<ElasticsearchViewIndex>) => void;
-  onDelete: (index: ElasticsearchViewIndex) => void;
+  onDelete: (indexName: string) => void;
 }
 
 export const IndicesTable: React.FC<IndicesTableProps> = ({
@@ -175,7 +175,7 @@ export const IndicesTable: React.FC<IndicesTableProps> = ({
                 },
               }
             ),
-          onClick: (index) => onDelete(index),
+          onClick: (index) => onDelete(index.name),
           type: 'icon',
         },
       ],

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts

@@ -81,7 +81,7 @@ export function getIngestionStatus(index?: ElasticsearchIndexWithIngestion): Ing
   if (!index || isApiIndex(index)) {
     return IngestionStatus.CONNECTED;
   }
-  if (isConnectorIndex(index) || isCrawlerIndex(index)) {
+  if (isConnectorIndex(index) || (isCrawlerIndex(index) && index.connector)) {
     if (
       index.connector.last_seen &&
       moment(index.connector.last_seen).isBefore(moment().subtract(30, 'minutes'))

x-pack/plugins/enterprise_search/server/lib/connectors/add_connector.ts

@@ -8,18 +8,13 @@
 import { IScopedClusterClient } from '@kbn/core/server';
 
 import { CONNECTORS_INDEX, CONNECTORS_VERSION } from '../..';
-import {
+import { ConnectorDocument } from '../../../common/types/connectors';
-  ConnectorDocument,
-  ConnectorStatus,
-  FilteringPolicy,
-  FilteringRuleRule,
-  FilteringValidationState,
-} from '../../../common/types/connectors';
 import { ErrorCode } from '../../../common/types/error_codes';
 import {
   DefaultConnectorsPipelineMeta,
   setupConnectorsIndices,
 } from '../../index_management/setup_indices';
+import { createConnectorDocument } from '../../utils/create_connector_document';
 
 import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers';
 import { createIndex } from '../indices/create_index';
@@ -85,89 +80,24 @@ export const addConnector = async (
   const connectorsIndicesMapping = await client.asCurrentUser.indices.getMapping({
     index: CONNECTORS_INDEX,
   });
-  const connectorsPipelineMeta: DefaultConnectorsPipelineMeta =
+  const pipeline: DefaultConnectorsPipelineMeta =
     connectorsIndicesMapping[`${CONNECTORS_INDEX}-v${CONNECTORS_VERSION}`]?.mappings?._meta
       ?.pipeline;
 
-  const currentTimestamp = new Date().toISOString();
+  const document = createConnectorDocument({
-  const document: ConnectorDocument = {
+    indexName: input.index_name,
-    api_key_id: null,
+    isNative: input.is_native,
-    configuration: {},
-    custom_scheduling: {},
-    description: null,
-    error: null,
-    features: null,
-    filtering: [
-      {
-        active: {
-          advanced_snippet: {
-            created_at: currentTimestamp,
-            updated_at: currentTimestamp,
-            value: {},
-          },
-          rules: [
-            {
-              created_at: currentTimestamp,
-              field: '_',
-              id: 'DEFAULT',
-              order: 0,
-              policy: FilteringPolicy.INCLUDE,
-              rule: FilteringRuleRule.REGEX,
-              updated_at: currentTimestamp,
-              value: '.*',
-            },
-          ],
-          validation: {
-            errors: [],
-            state: FilteringValidationState.VALID,
-          },
-        },
-        domain: 'DEFAULT',
-        draft: {
-          advanced_snippet: {
-            created_at: currentTimestamp,
-            updated_at: currentTimestamp,
-            value: {},
-          },
-          rules: [
-            {
-              created_at: currentTimestamp,
-              field: '_',
-              id: 'DEFAULT',
-              order: 0,
-              policy: FilteringPolicy.INCLUDE,
-              rule: FilteringRuleRule.REGEX,
-              updated_at: currentTimestamp,
-              value: '.*',
-            },
-          ],
-          validation: {
-            errors: [],
-            state: FilteringValidationState.VALID,
-          },
-        },
-      },
-    ],
-    index_name: input.index_name,
-    is_native: input.is_native,
     language: input.language,
-    last_seen: null,
+    pipeline: pipeline
-    last_sync_error: null,
-    last_sync_status: null,
-    last_synced: null,
-    name: input.index_name.startsWith('search-') ? input.index_name.substring(7) : input.index_name,
-    pipeline: connectorsPipelineMeta
       ? {
-          extract_binary_content: connectorsPipelineMeta.default_extract_binary_content,
+          extract_binary_content: pipeline.default_extract_binary_content,
-          name: connectorsPipelineMeta.default_name,
+          name: pipeline.default_name,
-          reduce_whitespace: connectorsPipelineMeta.default_reduce_whitespace,
+          reduce_whitespace: pipeline.default_reduce_whitespace,
-          run_ml_inference: connectorsPipelineMeta.default_run_ml_inference,
+          run_ml_inference: pipeline.default_run_ml_inference,
         }
       : null,
-    scheduling: { enabled: false, interval: '0 0 0 * * ?' },
+    serviceType: input.service_type,
-    service_type: input.service_type || null,
+  });
-    status: ConnectorStatus.CREATED,
+
-    sync_now: false,
-  };
   return await createConnector(document, client, input.language, !!input.delete_existing_connector);
 };

x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.test.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IScopedClusterClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX } from '../..';
+import { ConnectorStatus } from '../../../common/types/connectors';
+
+import { recreateConnectorDocument } from './post_connector';
+
+describe('recreateConnectorDocument lib function', () => {
+  const mockClient = {
+    asCurrentUser: {
+      index: jest.fn(),
+    },
+    asInternalUser: {},
+  };
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should recreate connector document', async () => {
+    mockClient.asCurrentUser.index.mockResolvedValue({ _id: 'connectorId' });
+
+    await recreateConnectorDocument(mockClient as unknown as IScopedClusterClient, 'indexName');
+    expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
+      document: {
+        api_key_id: null,
+        configuration: {},
+        custom_scheduling: {},
+        description: null,
+        error: null,
+        features: null,
+        filtering: [
+          {
+            active: {
+              advanced_snippet: {
+                created_at: expect.any(String),
+                updated_at: expect.any(String),
+                value: {},
+              },
+              rules: [
+                {
+                  created_at: expect.any(String),
+                  field: '_',
+                  id: 'DEFAULT',
+                  order: 0,
+                  policy: 'include',
+                  rule: 'regex',
+                  updated_at: expect.any(String),
+                  value: '.*',
+                },
+              ],
+              validation: {
+                errors: [],
+                state: 'valid',
+              },
+            },
+            domain: 'DEFAULT',
+            draft: {
+              advanced_snippet: {
+                created_at: expect.any(String),
+                updated_at: expect.any(String),
+                value: {},
+              },
+              rules: [
+                {
+                  created_at: expect.any(String),
+                  field: '_',
+                  id: 'DEFAULT',
+                  order: 0,
+                  policy: 'include',
+                  rule: 'regex',
+                  updated_at: expect.any(String),
+                  value: '.*',
+                },
+              ],
+              validation: {
+                errors: [],
+                state: 'valid',
+              },
+            },
+          },
+        ],
+        index_name: 'indexName',
+        is_native: false,
+        language: '',
+        last_seen: null,
+        last_sync_error: null,
+        last_sync_status: null,
+        last_synced: null,
+        name: 'indexName',
+        pipeline: null,
+        scheduling: { enabled: false, interval: '0 0 0 * * ?' },
+        service_type: 'elastic-crawler',
+        status: ConnectorStatus.CONFIGURED,
+        sync_now: false,
+      },
+      index: CONNECTORS_INDEX,
+      refresh: 'wait_for',
+    });
+  });
+});

x-pack/plugins/enterprise_search/server/lib/crawler/post_connector.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
+
+import { CONNECTORS_INDEX } from '../..';
+
+import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../common/constants';
+import { ConnectorStatus } from '../../../common/types/connectors';
+
+import { createConnectorDocument } from '../../utils/create_connector_document';
+
+export const recreateConnectorDocument = async (
+  client: IScopedClusterClient,
+  indexName: string
+) => {
+  const document = createConnectorDocument({
+    indexName,
+    isNative: false,
+    // The search index has already been created so we don't need the language, which we can't retrieve anymore anyway
+    language: '',
+    pipeline: null,
+    serviceType: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE,
+  });
+  const result = await client.asCurrentUser.index({
+    document: { ...document, status: ConnectorStatus.CONFIGURED },
+    index: CONNECTORS_INDEX,
+    refresh: 'wait_for',
+  });
+  return result._id;
+};

x-pack/plugins/enterprise_search/server/routes/enterprise_search/crawler/crawler.ts

@@ -16,6 +16,7 @@ import { addConnector } from '../../../lib/connectors/add_connector';
 import { deleteConnectorById } from '../../../lib/connectors/delete_connector';
 import { fetchConnectorByIndexName } from '../../../lib/connectors/fetch_connectors';
 import { fetchCrawlerByIndexName } from '../../../lib/crawler/fetch_crawlers';
+import { recreateConnectorDocument } from '../../../lib/crawler/post_connector';
 import { updateHtmlExtraction } from '../../../lib/crawler/put_html_extraction';
 import { deleteIndex } from '../../../lib/indices/delete_index';
 import { RouteDependencies } from '../../../plugin';
@@ -429,6 +430,37 @@ export function registerCrawlerRoutes(routeDependencies: RouteDependencies) {
     })
   );
 
+  router.post(
+    {
+      path: '/internal/enterprise_search/indices/{indexName}/crawler/connector',
+      validate: {
+        params: schema.object({
+          indexName: schema.string(),
+        }),
+      },
+    },
+    elasticsearchErrorHandler(log, async (context, request, response) => {
+      const { client } = (await context.core).elasticsearch;
+      const connector = await fetchConnectorByIndexName(client, request.params.indexName);
+      if (connector) {
+        return createError({
+          errorCode: ErrorCode.CONNECTOR_DOCUMENT_ALREADY_EXISTS,
+          message: i18n.translate(
+            'xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError',
+            {
+              defaultMessage: 'A connector for this index already exists',
+            }
+          ),
+          response,
+          statusCode: 409,
+        });
+      }
+
+      const connectorId = await recreateConnectorDocument(client, request.params.indexName);
+      return response.ok({ body: { connector_id: connectorId } });
+    })
+  );
+
   registerCrawlerCrawlRulesRoutes(routeDependencies);
   registerCrawlerEntryPointRoutes(routeDependencies);
   registerCrawlerSitemapRoutes(routeDependencies);

x-pack/plugins/enterprise_search/server/utils/create_connector_document.test.ts

@@ -0,0 +1,195 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ConnectorStatus } from '../../common/types/connectors';
+
+import { createConnectorDocument } from './create_connector_document';
+
+describe('createConnectorDocument', () => {
+  it('should create a connector document', () => {
+    expect(
+      createConnectorDocument({
+        indexName: 'indexName',
+        isNative: false,
+        language: 'fr',
+        pipeline: {
+          extract_binary_content: true,
+          name: 'ent-search-generic-ingestion',
+          reduce_whitespace: true,
+          run_ml_inference: false,
+        },
+      })
+    ).toEqual({
+      api_key_id: null,
+      configuration: {},
+      custom_scheduling: {},
+      description: null,
+      error: null,
+      features: null,
+      filtering: [
+        {
+          active: {
+            advanced_snippet: {
+              created_at: expect.any(String),
+              updated_at: expect.any(String),
+              value: {},
+            },
+            rules: [
+              {
+                created_at: expect.any(String),
+                field: '_',
+                id: 'DEFAULT',
+                order: 0,
+                policy: 'include',
+                rule: 'regex',
+                updated_at: expect.any(String),
+                value: '.*',
+              },
+            ],
+            validation: {
+              errors: [],
+              state: 'valid',
+            },
+          },
+          domain: 'DEFAULT',
+          draft: {
+            advanced_snippet: {
+              created_at: expect.any(String),
+              updated_at: expect.any(String),
+              value: {},
+            },
+            rules: [
+              {
+                created_at: expect.any(String),
+                field: '_',
+                id: 'DEFAULT',
+                order: 0,
+                policy: 'include',
+                rule: 'regex',
+                updated_at: expect.any(String),
+                value: '.*',
+              },
+            ],
+            validation: {
+              errors: [],
+              state: 'valid',
+            },
+          },
+        },
+      ],
+      index_name: 'indexName',
+      is_native: false,
+      language: 'fr',
+      last_seen: null,
+      last_sync_error: null,
+      last_sync_status: null,
+      last_synced: null,
+      name: 'indexName',
+      pipeline: {
+        extract_binary_content: true,
+        name: 'ent-search-generic-ingestion',
+        reduce_whitespace: true,
+        run_ml_inference: false,
+      },
+      scheduling: { enabled: false, interval: '0 0 0 * * ?' },
+      service_type: null,
+      status: ConnectorStatus.CREATED,
+      sync_now: false,
+    });
+  });
+  it('should remove search- from name', () => {
+    expect(
+      createConnectorDocument({
+        indexName: 'search-indexName',
+        isNative: false,
+        language: 'fr',
+        pipeline: {
+          extract_binary_content: true,
+          name: 'ent-search-generic-ingestion',
+          reduce_whitespace: true,
+          run_ml_inference: false,
+        },
+      })
+    ).toEqual({
+      api_key_id: null,
+      configuration: {},
+      custom_scheduling: {},
+      description: null,
+      error: null,
+      features: null,
+      filtering: [
+        {
+          active: {
+            advanced_snippet: {
+              created_at: expect.any(String),
+              updated_at: expect.any(String),
+              value: {},
+            },
+            rules: [
+              {
+                created_at: expect.any(String),
+                field: '_',
+                id: 'DEFAULT',
+                order: 0,
+                policy: 'include',
+                rule: 'regex',
+                updated_at: expect.any(String),
+                value: '.*',
+              },
+            ],
+            validation: {
+              errors: [],
+              state: 'valid',
+            },
+          },
+          domain: 'DEFAULT',
+          draft: {
+            advanced_snippet: {
+              created_at: expect.any(String),
+              updated_at: expect.any(String),
+              value: {},
+            },
+            rules: [
+              {
+                created_at: expect.any(String),
+                field: '_',
+                id: 'DEFAULT',
+                order: 0,
+                policy: 'include',
+                rule: 'regex',
+                updated_at: expect.any(String),
+                value: '.*',
+              },
+            ],
+            validation: {
+              errors: [],
+              state: 'valid',
+            },
+          },
+        },
+      ],
+      index_name: 'search-indexName',
+      is_native: false,
+      language: 'fr',
+      last_seen: null,
+      last_sync_error: null,
+      last_sync_status: null,
+      last_synced: null,
+      name: 'indexName',
+      pipeline: {
+        extract_binary_content: true,
+        name: 'ent-search-generic-ingestion',
+        reduce_whitespace: true,
+        run_ml_inference: false,
+      },
+      scheduling: { enabled: false, interval: '0 0 0 * * ?' },
+      service_type: null,
+      status: ConnectorStatus.CREATED,
+      sync_now: false,
+    });
+  });
+});

x-pack/plugins/enterprise_search/server/utils/create_connector_document.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ConnectorDocument,
+  ConnectorStatus,
+  FilteringPolicy,
+  FilteringRuleRule,
+  FilteringValidationState,
+  IngestPipelineParams,
+} from '../../common/types/connectors';
+
+export function createConnectorDocument({
+  indexName,
+  isNative,
+  pipeline,
+  serviceType,
+  language,
+}: {
+  indexName: string;
+  isNative: boolean;
+  language: string | null;
+  pipeline?: IngestPipelineParams | null;
+  serviceType?: string | null;
+}): ConnectorDocument {
+  const currentTimestamp = new Date().toISOString();
+  return {
+    api_key_id: null,
+    configuration: {},
+    custom_scheduling: {},
+    description: null,
+    error: null,
+    features: null,
+    filtering: [
+      {
+        active: {
+          advanced_snippet: {
+            created_at: currentTimestamp,
+            updated_at: currentTimestamp,
+            value: {},
+          },
+          rules: [
+            {
+              created_at: currentTimestamp,
+              field: '_',
+              id: 'DEFAULT',
+              order: 0,
+              policy: FilteringPolicy.INCLUDE,
+              rule: FilteringRuleRule.REGEX,
+              updated_at: currentTimestamp,
+              value: '.*',
+            },
+          ],
+          validation: {
+            errors: [],
+            state: FilteringValidationState.VALID,
+          },
+        },
+        domain: 'DEFAULT',
+        draft: {
+          advanced_snippet: {
+            created_at: currentTimestamp,
+            updated_at: currentTimestamp,
+            value: {},
+          },
+          rules: [
+            {
+              created_at: currentTimestamp,
+              field: '_',
+              id: 'DEFAULT',
+              order: 0,
+              policy: FilteringPolicy.INCLUDE,
+              rule: FilteringRuleRule.REGEX,
+              updated_at: currentTimestamp,
+              value: '.*',
+            },
+          ],
+          validation: {
+            errors: [],
+            state: FilteringValidationState.VALID,
+          },
+        },
+      },
+    ],
+    index_name: indexName,
+    is_native: isNative,
+    language,
+    last_seen: null,
+    last_sync_error: null,
+    last_sync_status: null,
+    last_synced: null,
+    name: indexName.startsWith('search-') ? indexName.substring(7) : indexName,
+    pipeline,
+    scheduling: { enabled: false, interval: '0 0 0 * * ?' },
+    service_type: serviceType || null,
+    status: ConnectorStatus.CREATED,
+    sync_now: false,
+  };
+}