mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
# Backport This will backport the following commits from `main` to `8.7`: - [[Enterprise Search] Show error for crawlers without a connector document (#150928)](https://github.com/elastic/kibana/pull/150928) <!--- Backport version: 8.9.7 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Sander Philipse","email":"94373878+sphilipse@users.noreply.github.com"},"sourceCommit":{"committedDate":"2023-02-28T22:23:04Z","message":"[Enterprise Search] Show error for crawlers without a connector document (#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers, where a connector document has\r\nbeen deleted. It offers users the choice to recreate the connector or\r\ndelete their index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341","branchLabelMapping":{"^v8.8.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:EnterpriseSearch","v8.7.0","v8.8.0"],"number":150928,"url":"https://github.com/elastic/kibana/pull/150928","mergeCommit":{"message":"[Enterprise Search] Show error for crawlers without a connector document (#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers, where a connector document has\r\nbeen deleted. It offers users the choice to recreate the connector or\r\ndelete their index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341"}},"sourceBranch":"main","suggestedTargetBranches":["8.7"],"targetPullRequestStates":[{"branch":"8.7","label":"v8.7.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.8.0","labelRegex":"^v8.8.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/150928","number":150928,"mergeCommit":{"message":"[Enterprise Search] Show error for crawlers without a connector document (#150928)\n\n## Summary\r\n\r\nThis shows an error for broken crawlers, where a connector document has\r\nbeen deleted. It offers users the choice to recreate the connector or\r\ndelete their index.\r\n\r\n\r\n221916935-b416a782-92b4-4795-a1af-73e97fa523c8.mov","sha":"6be7e3df0b8f55e45a114e8da66c2dd1927ca341"}}]}] BACKPORT--> Co-authored-by: Sander Philipse <94373878+sphilipse@users.noreply.github.com>
This commit is contained in:
Kibana Machine
GitHub
parent
d48e527c4d
commit
5a03678b98
18 changed files with 757 additions and 102 deletions
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { mockHttpValues } from '../../../__mocks__/kea_logic';
|
||||
|
||||
import { nextTick } from '@kbn/test-jest-helpers';
|
||||
|
||||
import { recreateCrawlerConnector } from './recreate_crawler_connector_api_logic';
|
||||
|
||||
describe('CreateCrawlerIndexApiLogic', () => {
|
||||
const { http } = mockHttpValues;
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
describe('createCrawlerIndex', () => {
|
||||
it('calls correct api', async () => {
|
||||
const indexName = 'elastic-co-crawler';
|
||||
http.post.mockReturnValue(Promise.resolve({ connector_id: 'connectorId' }));
|
||||
|
||||
const result = recreateCrawlerConnector({ indexName });
|
||||
await nextTick();
|
||||
|
||||
expect(http.post).toHaveBeenCalledWith(
|
||||
'/internal/enterprise_search/indices/elastic-co-crawler/crawler/connector'
|
||||
);
|
||||
await expect(result).resolves.toEqual({ connector_id: 'connectorId' });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic';
|
||||
import { HttpLogic } from '../../../shared/http';
|
||||
|
||||
export interface RecreateCrawlerConnectorArgs {
|
||||
indexName: string;
|
||||
}
|
||||
|
||||
export interface RecreateCrawlerConnectorResponse {
|
||||
created: string; // the name of the newly created index
|
||||
}
|
||||
|
||||
export const recreateCrawlerConnector = async ({ indexName }: RecreateCrawlerConnectorArgs) => {
|
||||
const route = `/internal/enterprise_search/indices/${indexName}/crawler/connector`;
|
||||
|
||||
return await HttpLogic.values.http.post<RecreateCrawlerConnectorResponse>(route);
|
||||
};
|
||||
|
||||
export const RecreateCrawlerConnectorApiLogic = createApiLogic(
|
||||
['recreate_crawler_connector_api_logic'],
|
||||
recreateCrawlerConnector
|
||||
);
|
||||
|
||||
export type RecreateCrawlerConnectorActions = Actions<
|
||||
RecreateCrawlerConnectorArgs,
|
||||
RecreateCrawlerConnectorResponse
|
||||
>;
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
|
||||
import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic';
|
||||
import { HttpLogic } from '../../../shared/http';
|
||||
|
||||
export interface DeleteIndexApiLogicArgs {
|
||||
|
|
@ -36,3 +36,5 @@ export const DeleteIndexApiLogic = createApiLogic(['delete_index_api_logic'], de
|
|||
},
|
||||
}),
|
||||
});
|
||||
|
||||
export type DeleteIndexApiActions = Actions<DeleteIndexApiLogicArgs, DeleteIndexApiLogicValues>;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import { SyncsContextMenu } from './syncs_context_menu';
|
|||
export const getHeaderActions = (indexData?: ElasticsearchIndexWithIngestion) => {
|
||||
const ingestionMethod = getIngestionMethod(indexData);
|
||||
return [
|
||||
...(isCrawlerIndex(indexData) ? [<CrawlerStatusIndicator />] : []),
|
||||
...(isCrawlerIndex(indexData) && indexData.connector ? [<CrawlerStatusIndicator />] : []),
|
||||
...(isConnectorIndex(indexData) ? [<SyncsContextMenu />] : []),
|
||||
<SearchEnginesPopover
|
||||
indexName={indexData?.name}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
|
||||
import { useActions, useValues } from 'kea';
|
||||
|
||||
import { EuiButton, EuiPageTemplate } from '@elastic/eui';
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
||||
import { Status } from '../../../../../../common/types/api';
|
||||
|
||||
import { RecreateCrawlerConnectorApiLogic } from '../../../api/crawler/recreate_crawler_connector_api_logic';
|
||||
import { DeleteIndexModal } from '../../search_indices/delete_index_modal';
|
||||
import { IndicesLogic } from '../../search_indices/indices_logic';
|
||||
import { IndexViewLogic } from '../index_view_logic';
|
||||
|
||||
import { NoConnectorRecordLogic } from './no_connector_record_logic';
|
||||
|
||||
export const NoConnectorRecord: React.FC = () => {
|
||||
const { indexName } = useValues(IndexViewLogic);
|
||||
const { isDeleteLoading } = useValues(IndicesLogic);
|
||||
const { openDeleteModal } = useActions(IndicesLogic);
|
||||
const { makeRequest } = useActions(RecreateCrawlerConnectorApiLogic);
|
||||
const { status } = useValues(RecreateCrawlerConnectorApiLogic);
|
||||
NoConnectorRecordLogic.mount();
|
||||
const buttonsDisabled = status === Status.LOADING || isDeleteLoading;
|
||||
|
||||
return (
|
||||
<>
|
||||
<DeleteIndexModal />
|
||||
|
||||
<EuiPageTemplate.EmptyPrompt
|
||||
iconType="alert"
|
||||
color="danger"
|
||||
title={
|
||||
<h2>
|
||||
{i18n.translate(
|
||||
'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.title',
|
||||
{
|
||||
defaultMessage: "This index's connector configuration has been removed",
|
||||
}
|
||||
)}
|
||||
</h2>
|
||||
}
|
||||
body={
|
||||
<p>
|
||||
{i18n.translate(
|
||||
'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.description',
|
||||
{
|
||||
defaultMessage:
|
||||
'We could not find a connector configuration for this crawler index. The record should be recreated, or the index should be deleted.',
|
||||
}
|
||||
)}
|
||||
</p>
|
||||
}
|
||||
actions={[
|
||||
<EuiButton
|
||||
color="danger"
|
||||
disabled={buttonsDisabled}
|
||||
isLoading={status === Status.LOADING}
|
||||
onClick={() => makeRequest({ indexName })}
|
||||
>
|
||||
{i18n.translate(
|
||||
'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.recreateConnectorRecord',
|
||||
{
|
||||
defaultMessage: 'Recreate connector record',
|
||||
}
|
||||
)}
|
||||
</EuiButton>,
|
||||
<EuiButton
|
||||
color="danger"
|
||||
disabled={buttonsDisabled}
|
||||
isLoading={isDeleteLoading}
|
||||
fill
|
||||
onClick={() => openDeleteModal(indexName)}
|
||||
>
|
||||
{i18n.translate(
|
||||
'xpack.enterpriseSearch.content.searchIndex.noCrawlerConnectorFound.deleteIndex',
|
||||
{
|
||||
defaultMessage: 'Delete index',
|
||||
}
|
||||
)}
|
||||
</EuiButton>,
|
||||
]}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { LogicMounter } from '../../../../__mocks__/kea_logic';
|
||||
|
||||
import { KibanaLogic } from '../../../../shared/kibana';
|
||||
|
||||
import { RecreateCrawlerConnectorApiLogic } from '../../../api/crawler/recreate_crawler_connector_api_logic';
|
||||
import { DeleteIndexApiLogic } from '../../../api/index/delete_index_api_logic';
|
||||
import { SEARCH_INDICES_PATH } from '../../../routes';
|
||||
|
||||
import { NoConnectorRecordLogic } from './no_connector_record_logic';
|
||||
|
||||
describe('NoConnectorRecordLogic', () => {
|
||||
const { mount: deleteMount } = new LogicMounter(DeleteIndexApiLogic);
|
||||
const { mount: recreateMount } = new LogicMounter(RecreateCrawlerConnectorApiLogic);
|
||||
const { mount } = new LogicMounter(NoConnectorRecordLogic);
|
||||
beforeEach(() => {
|
||||
deleteMount();
|
||||
recreateMount();
|
||||
mount();
|
||||
});
|
||||
it('should redirect to search indices on delete', () => {
|
||||
KibanaLogic.values.navigateToUrl = jest.fn();
|
||||
DeleteIndexApiLogic.actions.apiSuccess({} as any);
|
||||
expect(KibanaLogic.values.navigateToUrl).toHaveBeenCalledWith(SEARCH_INDICES_PATH);
|
||||
});
|
||||
it('should fetch index on recreate', () => {
|
||||
NoConnectorRecordLogic.actions.fetchIndex = jest.fn();
|
||||
RecreateCrawlerConnectorApiLogic.actions.apiSuccess({} as any);
|
||||
expect(NoConnectorRecordLogic.actions.fetchIndex).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { kea, MakeLogicType } from 'kea';
|
||||
|
||||
import { KibanaLogic } from '../../../../shared/kibana';
|
||||
|
||||
import {
|
||||
RecreateCrawlerConnectorActions,
|
||||
RecreateCrawlerConnectorApiLogic,
|
||||
} from '../../../api/crawler/recreate_crawler_connector_api_logic';
|
||||
import {
|
||||
DeleteIndexApiActions,
|
||||
DeleteIndexApiLogic,
|
||||
} from '../../../api/index/delete_index_api_logic';
|
||||
import { SEARCH_INDICES_PATH } from '../../../routes';
|
||||
import { IndexViewActions, IndexViewLogic } from '../index_view_logic';
|
||||
|
||||
type NoConnectorRecordActions = RecreateCrawlerConnectorActions['apiSuccess'] & {
|
||||
deleteSuccess: DeleteIndexApiActions['apiSuccess'];
|
||||
fetchIndex: IndexViewActions['fetchIndex'];
|
||||
};
|
||||
|
||||
export const NoConnectorRecordLogic = kea<MakeLogicType<{}, NoConnectorRecordActions>>({
|
||||
connect: {
|
||||
actions: [
|
||||
RecreateCrawlerConnectorApiLogic,
|
||||
['apiSuccess'],
|
||||
IndexViewLogic,
|
||||
['fetchIndex'],
|
||||
DeleteIndexApiLogic,
|
||||
['apiSuccess as deleteSuccess'],
|
||||
],
|
||||
},
|
||||
listeners: ({ actions }) => ({
|
||||
apiSuccess: () => {
|
||||
actions.fetchIndex();
|
||||
},
|
||||
deleteSuccess: () => {
|
||||
KibanaLogic.values.navigateToUrl(SEARCH_INDICES_PATH);
|
||||
},
|
||||
}),
|
||||
path: ['enterprise_search', 'content', 'no_connector_record'],
|
||||
});
|
||||
|
|
@ -38,6 +38,7 @@ import { AutomaticCrawlScheduler } from './crawler/automatic_crawl_scheduler/aut
|
|||
import { CrawlCustomSettingsFlyout } from './crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout';
|
||||
import { CrawlerConfiguration } from './crawler/crawler_configuration/crawler_configuration';
|
||||
import { SearchIndexDomainManagement } from './crawler/domain_management/domain_management';
|
||||
import { NoConnectorRecord } from './crawler/no_connector_record';
|
||||
import { SearchIndexDocuments } from './documents';
|
||||
import { SearchIndexIndexMappings } from './index_mappings';
|
||||
import { IndexNameLogic } from './index_name_logic';
|
||||
|
|
@ -224,12 +225,16 @@ export const SearchIndex: React.FC = () => {
|
|||
rightSideItems: getHeaderActions(index),
|
||||
}}
|
||||
>
|
||||
<>
|
||||
{indexName === index?.name && (
|
||||
<EuiTabbedContent tabs={tabs} selectedTab={selectedTab} onTabClick={onTabClick} />
|
||||
)}
|
||||
{isCrawlerIndex(index) && <CrawlCustomSettingsFlyout />}
|
||||
</>
|
||||
{isCrawlerIndex(index) && !index.connector ? (
|
||||
<NoConnectorRecord />
|
||||
) : (
|
||||
<>
|
||||
{indexName === index?.name && (
|
||||
<EuiTabbedContent tabs={tabs} selectedTab={selectedTab} onTabClick={onTabClick} />
|
||||
)}
|
||||
{isCrawlerIndex(index) && <CrawlCustomSettingsFlyout />}
|
||||
</>
|
||||
)}
|
||||
</EnterpriseSearchContentPageTemplate>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ describe('IndicesLogic', () => {
|
|||
describe('openDeleteModal', () => {
|
||||
it('should set deleteIndexName and set isDeleteModalVisible to true', () => {
|
||||
IndicesLogic.actions.fetchIndexDetails = jest.fn();
|
||||
IndicesLogic.actions.openDeleteModal(connectorIndex);
|
||||
IndicesLogic.actions.openDeleteModal(connectorIndex.name);
|
||||
expect(IndicesLogic.values).toEqual({
|
||||
...DEFAULT_VALUES,
|
||||
deleteModalIndexName: 'connector',
|
||||
|
|
@ -98,7 +98,7 @@ describe('IndicesLogic', () => {
|
|||
});
|
||||
describe('closeDeleteModal', () => {
|
||||
it('should set deleteIndexName to empty and set isDeleteModalVisible to false', () => {
|
||||
IndicesLogic.actions.openDeleteModal(connectorIndex);
|
||||
IndicesLogic.actions.openDeleteModal(connectorIndex.name);
|
||||
IndicesLogic.actions.fetchIndexDetails = jest.fn();
|
||||
IndicesLogic.actions.closeDeleteModal();
|
||||
expect(IndicesLogic.values).toEqual({
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ export interface IndicesActions {
|
|||
}): { meta: Meta; returnHiddenIndices: boolean; searchQuery?: string };
|
||||
makeRequest: typeof FetchIndicesAPILogic.actions.makeRequest;
|
||||
onPaginate(newPageIndex: number): { newPageIndex: number };
|
||||
openDeleteModal(index: ElasticsearchViewIndex): { index: ElasticsearchViewIndex };
|
||||
openDeleteModal(indexName: string): { indexName: string };
|
||||
setIsFirstRequest(): void;
|
||||
}
|
||||
export interface IndicesValues {
|
||||
|
|
@ -102,7 +102,7 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
|
|||
searchQuery,
|
||||
}),
|
||||
onPaginate: (newPageIndex) => ({ newPageIndex }),
|
||||
openDeleteModal: (index) => ({ index }),
|
||||
openDeleteModal: (indexName) => ({ indexName }),
|
||||
setIsFirstRequest: true,
|
||||
},
|
||||
connect: {
|
||||
|
|
@ -137,8 +137,8 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
|
|||
await breakpoint(150);
|
||||
actions.makeRequest(input);
|
||||
},
|
||||
openDeleteModal: ({ index }) => {
|
||||
actions.fetchIndexDetails({ indexName: index.name });
|
||||
openDeleteModal: ({ indexName }) => {
|
||||
actions.fetchIndexDetails({ indexName });
|
||||
},
|
||||
}),
|
||||
path: ['enterprise_search', 'content', 'indices_logic'],
|
||||
|
|
@ -147,7 +147,7 @@ export const IndicesLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
|
|||
'',
|
||||
{
|
||||
closeDeleteModal: () => '',
|
||||
openDeleteModal: (_, { index: { name } }) => name,
|
||||
openDeleteModal: (_, { indexName }) => indexName,
|
||||
},
|
||||
],
|
||||
isDeleteModalVisible: [
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ interface IndicesTableProps {
|
|||
isLoading?: boolean;
|
||||
meta: Meta;
|
||||
onChange: (criteria: CriteriaWithPagination<ElasticsearchViewIndex>) => void;
|
||||
onDelete: (index: ElasticsearchViewIndex) => void;
|
||||
onDelete: (indexName: string) => void;
|
||||
}
|
||||
|
||||
export const IndicesTable: React.FC<IndicesTableProps> = ({
|
||||
|
|
@ -175,7 +175,7 @@ export const IndicesTable: React.FC<IndicesTableProps> = ({
|
|||
},
|
||||
}
|
||||
),
|
||||
onClick: (index) => onDelete(index),
|
||||
onClick: (index) => onDelete(index.name),
|
||||
type: 'icon',
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ export function getIngestionStatus(index?: ElasticsearchIndexWithIngestion): Ing
|
|||
if (!index || isApiIndex(index)) {
|
||||
return IngestionStatus.CONNECTED;
|
||||
}
|
||||
if (isConnectorIndex(index) || isCrawlerIndex(index)) {
|
||||
if (isConnectorIndex(index) || (isCrawlerIndex(index) && index.connector)) {
|
||||
if (
|
||||
index.connector.last_seen &&
|
||||
moment(index.connector.last_seen).isBefore(moment().subtract(30, 'minutes'))
|
||||
|
|
|
|||
|
|
@ -8,18 +8,13 @@
|
|||
import { IScopedClusterClient } from '@kbn/core/server';
|
||||
|
||||
import { CONNECTORS_INDEX, CONNECTORS_VERSION } from '../..';
|
||||
import {
|
||||
ConnectorDocument,
|
||||
ConnectorStatus,
|
||||
FilteringPolicy,
|
||||
FilteringRuleRule,
|
||||
FilteringValidationState,
|
||||
} from '../../../common/types/connectors';
|
||||
import { ConnectorDocument } from '../../../common/types/connectors';
|
||||
import { ErrorCode } from '../../../common/types/error_codes';
|
||||
import {
|
||||
DefaultConnectorsPipelineMeta,
|
||||
setupConnectorsIndices,
|
||||
} from '../../index_management/setup_indices';
|
||||
import { createConnectorDocument } from '../../utils/create_connector_document';
|
||||
|
||||
import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers';
|
||||
import { createIndex } from '../indices/create_index';
|
||||
|
|
@ -85,89 +80,24 @@ export const addConnector = async (
|
|||
const connectorsIndicesMapping = await client.asCurrentUser.indices.getMapping({
|
||||
index: CONNECTORS_INDEX,
|
||||
});
|
||||
const connectorsPipelineMeta: DefaultConnectorsPipelineMeta =
|
||||
const pipeline: DefaultConnectorsPipelineMeta =
|
||||
connectorsIndicesMapping[`${CONNECTORS_INDEX}-v${CONNECTORS_VERSION}`]?.mappings?._meta
|
||||
?.pipeline;
|
||||
|
||||
const currentTimestamp = new Date().toISOString();
|
||||
const document: ConnectorDocument = {
|
||||
api_key_id: null,
|
||||
configuration: {},
|
||||
custom_scheduling: {},
|
||||
description: null,
|
||||
error: null,
|
||||
features: null,
|
||||
filtering: [
|
||||
{
|
||||
active: {
|
||||
advanced_snippet: {
|
||||
created_at: currentTimestamp,
|
||||
updated_at: currentTimestamp,
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: currentTimestamp,
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: FilteringPolicy.INCLUDE,
|
||||
rule: FilteringRuleRule.REGEX,
|
||||
updated_at: currentTimestamp,
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: FilteringValidationState.VALID,
|
||||
},
|
||||
},
|
||||
domain: 'DEFAULT',
|
||||
draft: {
|
||||
advanced_snippet: {
|
||||
created_at: currentTimestamp,
|
||||
updated_at: currentTimestamp,
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: currentTimestamp,
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: FilteringPolicy.INCLUDE,
|
||||
rule: FilteringRuleRule.REGEX,
|
||||
updated_at: currentTimestamp,
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: FilteringValidationState.VALID,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
index_name: input.index_name,
|
||||
is_native: input.is_native,
|
||||
const document = createConnectorDocument({
|
||||
indexName: input.index_name,
|
||||
isNative: input.is_native,
|
||||
language: input.language,
|
||||
last_seen: null,
|
||||
last_sync_error: null,
|
||||
last_sync_status: null,
|
||||
last_synced: null,
|
||||
name: input.index_name.startsWith('search-') ? input.index_name.substring(7) : input.index_name,
|
||||
pipeline: connectorsPipelineMeta
|
||||
pipeline: pipeline
|
||||
? {
|
||||
extract_binary_content: connectorsPipelineMeta.default_extract_binary_content,
|
||||
name: connectorsPipelineMeta.default_name,
|
||||
reduce_whitespace: connectorsPipelineMeta.default_reduce_whitespace,
|
||||
run_ml_inference: connectorsPipelineMeta.default_run_ml_inference,
|
||||
extract_binary_content: pipeline.default_extract_binary_content,
|
||||
name: pipeline.default_name,
|
||||
reduce_whitespace: pipeline.default_reduce_whitespace,
|
||||
run_ml_inference: pipeline.default_run_ml_inference,
|
||||
}
|
||||
: null,
|
||||
scheduling: { enabled: false, interval: '0 0 0 * * ?' },
|
||||
service_type: input.service_type || null,
|
||||
status: ConnectorStatus.CREATED,
|
||||
sync_now: false,
|
||||
};
|
||||
serviceType: input.service_type,
|
||||
});
|
||||
|
||||
return await createConnector(document, client, input.language, !!input.delete_existing_connector);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IScopedClusterClient } from '@kbn/core/server';
|
||||
|
||||
import { CONNECTORS_INDEX } from '../..';
|
||||
import { ConnectorStatus } from '../../../common/types/connectors';
|
||||
|
||||
import { recreateConnectorDocument } from './post_connector';
|
||||
|
||||
describe('recreateConnectorDocument lib function', () => {
|
||||
const mockClient = {
|
||||
asCurrentUser: {
|
||||
index: jest.fn(),
|
||||
},
|
||||
asInternalUser: {},
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should recreate connector document', async () => {
|
||||
mockClient.asCurrentUser.index.mockResolvedValue({ _id: 'connectorId' });
|
||||
|
||||
await recreateConnectorDocument(mockClient as unknown as IScopedClusterClient, 'indexName');
|
||||
expect(mockClient.asCurrentUser.index).toHaveBeenCalledWith({
|
||||
document: {
|
||||
api_key_id: null,
|
||||
configuration: {},
|
||||
custom_scheduling: {},
|
||||
description: null,
|
||||
error: null,
|
||||
features: null,
|
||||
filtering: [
|
||||
{
|
||||
active: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
domain: 'DEFAULT',
|
||||
draft: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
index_name: 'indexName',
|
||||
is_native: false,
|
||||
language: '',
|
||||
last_seen: null,
|
||||
last_sync_error: null,
|
||||
last_sync_status: null,
|
||||
last_synced: null,
|
||||
name: 'indexName',
|
||||
pipeline: null,
|
||||
scheduling: { enabled: false, interval: '0 0 0 * * ?' },
|
||||
service_type: 'elastic-crawler',
|
||||
status: ConnectorStatus.CONFIGURED,
|
||||
sync_now: false,
|
||||
},
|
||||
index: CONNECTORS_INDEX,
|
||||
refresh: 'wait_for',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
|
||||
|
||||
import { CONNECTORS_INDEX } from '../..';
|
||||
|
||||
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../common/constants';
|
||||
import { ConnectorStatus } from '../../../common/types/connectors';
|
||||
|
||||
import { createConnectorDocument } from '../../utils/create_connector_document';
|
||||
|
||||
export const recreateConnectorDocument = async (
|
||||
client: IScopedClusterClient,
|
||||
indexName: string
|
||||
) => {
|
||||
const document = createConnectorDocument({
|
||||
indexName,
|
||||
isNative: false,
|
||||
// The search index has already been created so we don't need the language, which we can't retrieve anymore anyway
|
||||
language: '',
|
||||
pipeline: null,
|
||||
serviceType: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE,
|
||||
});
|
||||
const result = await client.asCurrentUser.index({
|
||||
document: { ...document, status: ConnectorStatus.CONFIGURED },
|
||||
index: CONNECTORS_INDEX,
|
||||
refresh: 'wait_for',
|
||||
});
|
||||
return result._id;
|
||||
};
|
||||
|
|
@ -16,6 +16,7 @@ import { addConnector } from '../../../lib/connectors/add_connector';
|
|||
import { deleteConnectorById } from '../../../lib/connectors/delete_connector';
|
||||
import { fetchConnectorByIndexName } from '../../../lib/connectors/fetch_connectors';
|
||||
import { fetchCrawlerByIndexName } from '../../../lib/crawler/fetch_crawlers';
|
||||
import { recreateConnectorDocument } from '../../../lib/crawler/post_connector';
|
||||
import { updateHtmlExtraction } from '../../../lib/crawler/put_html_extraction';
|
||||
import { deleteIndex } from '../../../lib/indices/delete_index';
|
||||
import { RouteDependencies } from '../../../plugin';
|
||||
|
|
@ -429,6 +430,37 @@ export function registerCrawlerRoutes(routeDependencies: RouteDependencies) {
|
|||
})
|
||||
);
|
||||
|
||||
router.post(
|
||||
{
|
||||
path: '/internal/enterprise_search/indices/{indexName}/crawler/connector',
|
||||
validate: {
|
||||
params: schema.object({
|
||||
indexName: schema.string(),
|
||||
}),
|
||||
},
|
||||
},
|
||||
elasticsearchErrorHandler(log, async (context, request, response) => {
|
||||
const { client } = (await context.core).elasticsearch;
|
||||
const connector = await fetchConnectorByIndexName(client, request.params.indexName);
|
||||
if (connector) {
|
||||
return createError({
|
||||
errorCode: ErrorCode.CONNECTOR_DOCUMENT_ALREADY_EXISTS,
|
||||
message: i18n.translate(
|
||||
'xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError',
|
||||
{
|
||||
defaultMessage: 'A connector for this index already exists',
|
||||
}
|
||||
),
|
||||
response,
|
||||
statusCode: 409,
|
||||
});
|
||||
}
|
||||
|
||||
const connectorId = await recreateConnectorDocument(client, request.params.indexName);
|
||||
return response.ok({ body: { connector_id: connectorId } });
|
||||
})
|
||||
);
|
||||
|
||||
registerCrawlerCrawlRulesRoutes(routeDependencies);
|
||||
registerCrawlerEntryPointRoutes(routeDependencies);
|
||||
registerCrawlerSitemapRoutes(routeDependencies);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,195 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ConnectorStatus } from '../../common/types/connectors';
|
||||
|
||||
import { createConnectorDocument } from './create_connector_document';
|
||||
|
||||
describe('createConnectorDocument', () => {
|
||||
it('should create a connector document', () => {
|
||||
expect(
|
||||
createConnectorDocument({
|
||||
indexName: 'indexName',
|
||||
isNative: false,
|
||||
language: 'fr',
|
||||
pipeline: {
|
||||
extract_binary_content: true,
|
||||
name: 'ent-search-generic-ingestion',
|
||||
reduce_whitespace: true,
|
||||
run_ml_inference: false,
|
||||
},
|
||||
})
|
||||
).toEqual({
|
||||
api_key_id: null,
|
||||
configuration: {},
|
||||
custom_scheduling: {},
|
||||
description: null,
|
||||
error: null,
|
||||
features: null,
|
||||
filtering: [
|
||||
{
|
||||
active: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
domain: 'DEFAULT',
|
||||
draft: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
index_name: 'indexName',
|
||||
is_native: false,
|
||||
language: 'fr',
|
||||
last_seen: null,
|
||||
last_sync_error: null,
|
||||
last_sync_status: null,
|
||||
last_synced: null,
|
||||
name: 'indexName',
|
||||
pipeline: {
|
||||
extract_binary_content: true,
|
||||
name: 'ent-search-generic-ingestion',
|
||||
reduce_whitespace: true,
|
||||
run_ml_inference: false,
|
||||
},
|
||||
scheduling: { enabled: false, interval: '0 0 0 * * ?' },
|
||||
service_type: null,
|
||||
status: ConnectorStatus.CREATED,
|
||||
sync_now: false,
|
||||
});
|
||||
});
|
||||
it('should remove search- from name', () => {
|
||||
expect(
|
||||
createConnectorDocument({
|
||||
indexName: 'search-indexName',
|
||||
isNative: false,
|
||||
language: 'fr',
|
||||
pipeline: {
|
||||
extract_binary_content: true,
|
||||
name: 'ent-search-generic-ingestion',
|
||||
reduce_whitespace: true,
|
||||
run_ml_inference: false,
|
||||
},
|
||||
})
|
||||
).toEqual({
|
||||
api_key_id: null,
|
||||
configuration: {},
|
||||
custom_scheduling: {},
|
||||
description: null,
|
||||
error: null,
|
||||
features: null,
|
||||
filtering: [
|
||||
{
|
||||
active: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
domain: 'DEFAULT',
|
||||
draft: {
|
||||
advanced_snippet: {
|
||||
created_at: expect.any(String),
|
||||
updated_at: expect.any(String),
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: expect.any(String),
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: 'include',
|
||||
rule: 'regex',
|
||||
updated_at: expect.any(String),
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: 'valid',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
index_name: 'search-indexName',
|
||||
is_native: false,
|
||||
language: 'fr',
|
||||
last_seen: null,
|
||||
last_sync_error: null,
|
||||
last_sync_status: null,
|
||||
last_synced: null,
|
||||
name: 'indexName',
|
||||
pipeline: {
|
||||
extract_binary_content: true,
|
||||
name: 'ent-search-generic-ingestion',
|
||||
reduce_whitespace: true,
|
||||
run_ml_inference: false,
|
||||
},
|
||||
scheduling: { enabled: false, interval: '0 0 0 * * ?' },
|
||||
service_type: null,
|
||||
status: ConnectorStatus.CREATED,
|
||||
sync_now: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
ConnectorDocument,
|
||||
ConnectorStatus,
|
||||
FilteringPolicy,
|
||||
FilteringRuleRule,
|
||||
FilteringValidationState,
|
||||
IngestPipelineParams,
|
||||
} from '../../common/types/connectors';
|
||||
|
||||
export function createConnectorDocument({
|
||||
indexName,
|
||||
isNative,
|
||||
pipeline,
|
||||
serviceType,
|
||||
language,
|
||||
}: {
|
||||
indexName: string;
|
||||
isNative: boolean;
|
||||
language: string | null;
|
||||
pipeline?: IngestPipelineParams | null;
|
||||
serviceType?: string | null;
|
||||
}): ConnectorDocument {
|
||||
const currentTimestamp = new Date().toISOString();
|
||||
return {
|
||||
api_key_id: null,
|
||||
configuration: {},
|
||||
custom_scheduling: {},
|
||||
description: null,
|
||||
error: null,
|
||||
features: null,
|
||||
filtering: [
|
||||
{
|
||||
active: {
|
||||
advanced_snippet: {
|
||||
created_at: currentTimestamp,
|
||||
updated_at: currentTimestamp,
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: currentTimestamp,
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: FilteringPolicy.INCLUDE,
|
||||
rule: FilteringRuleRule.REGEX,
|
||||
updated_at: currentTimestamp,
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: FilteringValidationState.VALID,
|
||||
},
|
||||
},
|
||||
domain: 'DEFAULT',
|
||||
draft: {
|
||||
advanced_snippet: {
|
||||
created_at: currentTimestamp,
|
||||
updated_at: currentTimestamp,
|
||||
value: {},
|
||||
},
|
||||
rules: [
|
||||
{
|
||||
created_at: currentTimestamp,
|
||||
field: '_',
|
||||
id: 'DEFAULT',
|
||||
order: 0,
|
||||
policy: FilteringPolicy.INCLUDE,
|
||||
rule: FilteringRuleRule.REGEX,
|
||||
updated_at: currentTimestamp,
|
||||
value: '.*',
|
||||
},
|
||||
],
|
||||
validation: {
|
||||
errors: [],
|
||||
state: FilteringValidationState.VALID,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
index_name: indexName,
|
||||
is_native: isNative,
|
||||
language,
|
||||
last_seen: null,
|
||||
last_sync_error: null,
|
||||
last_sync_status: null,
|
||||
last_synced: null,
|
||||
name: indexName.startsWith('search-') ? indexName.substring(7) : indexName,
|
||||
pipeline,
|
||||
scheduling: { enabled: false, interval: '0 0 0 * * ?' },
|
||||
service_type: serviceType || null,
|
||||
status: ConnectorStatus.CREATED,
|
||||
sync_now: false,
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue