github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.10] [Enterprise Search] Custom crawler scheduling UI - post FF fixes (#164434) (#164588)

Browse source 
# Backport

This will backport the following commits from `main` to `8.10`:
- [[Enterprise Search] Custom crawler scheduling UI - post FF fixes
(#164434)](https://github.com/elastic/kibana/pull/164434)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Jedrzej
Blaszyk","email":"jedrazb@gmail.com"},"sourceCommit":{"committedDate":"2023-08-23T13:25:26Z","message":"[Enterprise
Search] Custom crawler scheduling UI - post FF fixes
(#164434)","sha":"eda140620611d3f8617fec31afe4d2f0b3b892fe","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:EnterpriseSearch","backport:prev-minor","v8.10.0","v8.11.0"],"number":164434,"url":"https://github.com/elastic/kibana/pull/164434","mergeCommit":{"message":"[Enterprise
Search] Custom crawler scheduling UI - post FF fixes
(#164434)","sha":"eda140620611d3f8617fec31afe4d2f0b3b892fe"}},"sourceBranch":"main","suggestedTargetBranches":["8.10"],"targetPullRequestStates":[{"branch":"8.10","label":"v8.10.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/164434","number":164434,"mergeCommit":{"message":"[Enterprise
Search] Custom crawler scheduling UI - post FF fixes
(#164434)","sha":"eda140620611d3f8617fec31afe4d2f0b3b892fe"}}]}]
BACKPORT-->

Co-authored-by: Jedrzej Blaszyk <jedrazb@gmail.com>
This commit is contained in:
Kibana Machine 2023-08-23 10:53:18 -04:00 • committed by GitHub
parent 670f31b248
commit 1f81e89851
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 619 additions and 327 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/types.ts
View file

@ -173,6 +173,7 @@ export interface CrawlScheduleFromServer {
// Client
export interface CrawlerCustomSchedule {
scheduleKey: string;
name: string;
customEntryPointUrls: string[];
customSitemapUrls: string[];
@ -183,6 +184,8 @@ export interface CrawlerCustomSchedule {
selectedSitemapUrls: string[];
interval: string; // interval has crontab syntax
enabled: boolean;
entryPointUrls: string[];
sitemapUrls: string[];
}
export enum CustomCrawlType {

62
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/utils.ts
View file

@ -248,34 +248,39 @@ export const domainConfigServerToClient = (
export const crawlerCustomSchedulingServerToClient = (
customSchedulingFromServer: CrawlerCustomSchedulesServer
): CrawlerCustomSchedule[] =>
Object.entries(customSchedulingFromServer.custom_scheduling).map((scheduleMapping) => {
const {
name,
interval,
configuration_overrides: configurationOverrides,
enabled,
} = scheduleMapping[1];
const {
max_crawl_depth: maxCrawlDepth = 2,
sitemap_discovery_disabled: notIncludeSitemapsInRobotsTxt = false,
domain_allowlist: selectedDomainUrls = [],
sitemap_urls: customSitemapUrls = [],
seed_urls: customEntryPointUrls = [],
} = configurationOverrides;
Object.entries(customSchedulingFromServer.custom_scheduling).map(
([scheduleKey, scheduleMapping]) => {
const {
name,
interval,
configuration_overrides: configurationOverrides,
enabled,
} = scheduleMapping;
const {
max_crawl_depth: maxCrawlDepth = 2,
sitemap_discovery_disabled: notIncludeSitemapsInRobotsTxt = false,
domain_allowlist: selectedDomainUrls = [],
sitemap_urls: customSitemapUrls = [],
seed_urls: customEntryPointUrls = [],
} = configurationOverrides;
return {
name,
interval,
enabled,
maxCrawlDepth,
includeSitemapsInRobotsTxt: !notIncludeSitemapsInRobotsTxt,
selectedDomainUrls,
selectedEntryPointUrls: [],
selectedSitemapUrls: [],
customEntryPointUrls,
customSitemapUrls,
};
});
return {
scheduleKey,
name,
interval,
enabled,
maxCrawlDepth,
includeSitemapsInRobotsTxt: !notIncludeSitemapsInRobotsTxt,
selectedDomainUrls,
selectedEntryPointUrls: [],
selectedSitemapUrls: [],
customEntryPointUrls,
customSitemapUrls,
entryPointUrls: [],
sitemapUrls: [],
};
}
);
export const crawlerCustomSchedulingClientToServer = (
crawlerCustomSchedules: CrawlerCustomSchedule[]
@ -304,8 +309,7 @@ export const crawlerCustomSchedulingClientToServer = (
const customSchedules: CrawlerCustomScheduleMappingClient = crawlerCustomSchedules.reduce(
(map, schedule) => {
const scheduleNameFormatted = schedule.name.replace(/\s+/g, '_').toLowerCase();
map.set(scheduleNameFormatted, mapToServerFormat(schedule));
map.set(schedule.scheduleKey, mapToServerFormat(schedule));
return map;
},
new Map()

8
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout.tsx
View file

@ -33,6 +33,7 @@ import { CrawlCustomSettingsFlyoutCrawlTypeSelection } from './crawl_custom_sett
import { CrawlCustomSettingsFlyoutDomainsPanelWithLogicProps } from './crawl_custom_settings_flyout_domains_panel';
import { CrawlCustomSettingsFlyoutLogic } from './crawl_custom_settings_flyout_logic';
import { CrawlCustomSettingsFlyoutMultipleCrawlDelete } from './crawl_custom_settings_flyout_multi_crawl_delete';
import { CrawlCustomSettingsFlyoutMultiCrawlLogic } from './crawl_custom_settings_flyout_multi_crawl_logic';
import { CrawlCustomSettingsFlyoutMultipleCrawlTabs } from './crawl_custom_settings_flyout_multi_crawl_tabs';
import { CrawlCustomSettingsFlyoutMultiCrawlScheduling } from './crawl_custom_settings_flyout_mutli_crawl';
import { CrawlCustomSettingsFlyoutSeedUrlsPanelWithLogicProps } from './crawl_custom_settings_flyout_seed_urls_panel';
@ -45,6 +46,7 @@ export const CrawlCustomSettingsFlyout: React.FC = () => {
isSingleCrawlType,
selectedDomainUrls,
} = useValues(CrawlCustomSettingsFlyoutLogic);
const { crawlerCustomSchedulingIsValid } = useValues(CrawlCustomSettingsFlyoutMultiCrawlLogic);
const { hideFlyout, startCustomCrawl, saveCustomSchedulingConfiguration } = useActions(
CrawlCustomSettingsFlyoutLogic
);
@ -125,7 +127,11 @@ export const CrawlCustomSettingsFlyout: React.FC = () => {
data-telemetry-id="entSearchContent-crawler-customCrawlSettings-startCrawl"
fill
onClick={submitFunctionLogic}
disabled={isDataLoading || selectedDomainUrls.length === 0}
disabled={
isSingleCrawlType
? isDataLoading || selectedDomainUrls.length === 0
: !crawlerCustomSchedulingIsValid
}
isLoading={isFormSubmitting}
>
{isSingleCrawlType

197
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_domain_logic.test.ts Normal file
View file

@ -0,0 +1,197 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { LogicMounter, mockHttpValues } from '../../../../../__mocks__/kea_logic';
import '../../_mocks_/index_name_logic.mock';
import { nextTick } from '@kbn/test-jest-helpers';
import { itShowsServerErrorAsFlashMessage } from '../../../../../test_helpers';
import { DomainConfig } from '../../../../api/crawler/types';
import { CrawlCustomSettingsFlyoutDomainConfigLogic } from './crawl_custom_settings_flyout_domain_logic';
describe('CrawlCustomSettingsFlyoutDomainConfigLogic', () => {
const { mount } = new LogicMounter(CrawlCustomSettingsFlyoutDomainConfigLogic);
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
mount();
});
it('has expected default values', () => {
expect(CrawlCustomSettingsFlyoutDomainConfigLogic.values).toEqual({
domainConfigMap: {},
domainConfigs: [],
domainUrls: [],
});
});
describe('actions', () => {
describe('fetchDomainConfigData', () => {
it('updates logic with data that has been converted from server to client', async () => {
jest.spyOn(CrawlCustomSettingsFlyoutDomainConfigLogic.actions, 'onRecieveDomainConfigData');
http.get.mockReturnValueOnce(
Promise.resolve({
meta: {
page: {
current: 1,
size: 1,
total_pages: 2,
},
},
results: [
{
id: '1234',
name: 'https://www.elastic.co',
seed_urls: [],
sitemap_urls: [],
},
],
})
);
http.get.mockReturnValueOnce(
Promise.resolve({
meta: {
page: {
current: 2,
size: 1,
total_pages: 2,
},
},
results: [
{
id: '5678',
name: 'https://www.swiftype.com',
seed_urls: [],
sitemap_urls: [],
},
],
})
);
CrawlCustomSettingsFlyoutDomainConfigLogic.actions.fetchDomainConfigData();
await nextTick();
expect(http.get).toHaveBeenNthCalledWith(
1,
'/internal/enterprise_search/indices/index-name/crawler/domain_configs',
{
query: {
'page[current]': 1,
'page[size]': 100,
},
}
);
expect(http.get).toHaveBeenNthCalledWith(
2,
'/internal/enterprise_search/indices/index-name/crawler/domain_configs',
{
query: {
'page[current]': 2,
'page[size]': 1,
},
}
);
expect(
CrawlCustomSettingsFlyoutDomainConfigLogic.actions.onRecieveDomainConfigData
).toHaveBeenCalledWith([
{
id: '1234',
name: 'https://www.elastic.co',
seedUrls: [],
sitemapUrls: [],
},
{
id: '5678',
name: 'https://www.swiftype.com',
seedUrls: [],
sitemapUrls: [],
},
]);
});
itShowsServerErrorAsFlashMessage(http.get, () => {
CrawlCustomSettingsFlyoutDomainConfigLogic.actions.fetchDomainConfigData();
});
});
describe('onRecieveDomainConfigData', () => {
it('saves the data', () => {
mount({
domainConfigs: [],
});
CrawlCustomSettingsFlyoutDomainConfigLogic.actions.onRecieveDomainConfigData([
{
name: 'https://www.elastic.co',
},
] as DomainConfig[]);
expect(CrawlCustomSettingsFlyoutDomainConfigLogic.values.domainConfigs).toEqual([
{
name: 'https://www.elastic.co',
},
]);
});
});
});
describe('selectors', () => {
beforeEach(() => {
mount({
domainConfigs: [
{
name: 'https://www.elastic.co',
sitemapUrls: [
'https://www.elastic.co/sitemap1.xml',
'https://www.elastic.co/sitemap2.xml',
],
seedUrls: ['https://www.elastic.co/', 'https://www.elastic.co/guide'],
},
{
name: 'https://swiftype.com',
sitemapUrls: ['https://swiftype.com/sitemap1.xml', 'https://swiftype.com/sitemap2.xml'],
seedUrls: ['https://swiftype.com/', 'https://swiftype.com/documentation'],
},
],
});
});
describe('domainUrls', () => {
it('contains all the domain urls from the domain config', () => {
expect(CrawlCustomSettingsFlyoutDomainConfigLogic.values.domainUrls).toEqual([
'https://www.elastic.co',
'https://swiftype.com',
]);
});
});
describe('domainConfigMap', () => {
it('contains all the domain urls from the domain config', () => {
expect(CrawlCustomSettingsFlyoutDomainConfigLogic.values.domainConfigMap).toEqual({
'https://www.elastic.co': {
name: 'https://www.elastic.co',
sitemapUrls: [
'https://www.elastic.co/sitemap1.xml',
'https://www.elastic.co/sitemap2.xml',
],
seedUrls: ['https://www.elastic.co/', 'https://www.elastic.co/guide'],
},
'https://swiftype.com': {
name: 'https://swiftype.com',
sitemapUrls: ['https://swiftype.com/sitemap1.xml', 'https://swiftype.com/sitemap2.xml'],
seedUrls: ['https://swiftype.com/', 'https://swiftype.com/documentation'],
},
});
});
});
});
});

103
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_domain_logic.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kea, MakeLogicType } from 'kea';
import { Meta } from '../../../../../../../common/types';
import { flashAPIErrors } from '../../../../../shared/flash_messages';
import { HttpLogic } from '../../../../../shared/http';
import { DomainConfig, DomainConfigFromServer } from '../../../../api/crawler/types';
import { domainConfigServerToClient } from '../../../../api/crawler/utils';
import { IndexNameLogic } from '../../index_name_logic';
export interface CrawlCustomSettingsFlyoutDomainConfigLogicValues {
domainUrls: string[];
domainConfigs: DomainConfig[];
domainConfigMap: {
[key: string]: DomainConfig;
};
}
export const domainConfigsToDomainUrls = (domainConfigs: DomainConfig[]) =>
domainConfigs.map((domainConfig) => domainConfig.name);
export const domainConfigsToDomainConfigMap = (domainConfigs: DomainConfig[]) =>
domainConfigs.reduce(
(acc, domainConfig) => ({ ...acc, [domainConfig.name]: domainConfig }),
{} as { [key: string]: DomainConfig }
);
export interface CrawlCustomSettingsFlyoutDomainConfigLogicActions {
fetchDomainConfigData(): void;
onRecieveDomainConfigData(domainConfigs: DomainConfig[]): { domainConfigs: DomainConfig[] };
}
export const CrawlCustomSettingsFlyoutDomainConfigLogic = kea<
MakeLogicType<
CrawlCustomSettingsFlyoutDomainConfigLogicValues,
CrawlCustomSettingsFlyoutDomainConfigLogicActions
>
>({
path: ['enterprise_search', 'crawler', 'crawl_custom_settings_flyout_domain_logic'],
actions: () => ({
fetchDomainConfigData: true,
onRecieveDomainConfigData: (domainConfigs) => ({ domainConfigs }),
}),
reducers: () => ({
domainConfigs: [
[],
{
onRecieveDomainConfigData: (_, { domainConfigs }) => domainConfigs,
},
],
}),
selectors: () => ({
domainUrls: [
(selectors) => [selectors.domainConfigs],
(domainConfigs: DomainConfig[]) => domainConfigsToDomainUrls(domainConfigs),
],
domainConfigMap: [
(selectors) => [selectors.domainConfigs],
(domainConfigs: DomainConfig[]) => domainConfigsToDomainConfigMap(domainConfigs),
],
}),
listeners: ({ actions }) => ({
fetchDomainConfigData: async () => {
const { http } = HttpLogic.values;
const { indexName } = IndexNameLogic.values;
let domainConfigs: DomainConfig[] = [];
let totalPages: number = 1;
let nextPage: number = 1;
let pageSize: number = 100;
try {
while (nextPage <= totalPages) {
const {
results,
meta: { page },
} = await http.get<{
meta: Meta;
results: DomainConfigFromServer[];
}>(`/internal/enterprise_search/indices/${indexName}/crawler/domain_configs`, {
query: { 'page[current]': nextPage, 'page[size]': pageSize },
});
domainConfigs = [...domainConfigs, ...results.map(domainConfigServerToClient)];
nextPage = page.current + 1;
totalPages = page.total_pages;
pageSize = page.size;
}
actions.onRecieveDomainConfigData(domainConfigs);
} catch (e) {
flashAPIErrors(e);
}
},
}),
});

4
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_domains_panel.tsx
View file

@ -24,6 +24,7 @@ import { i18n } from '@kbn/i18n';
import { SimplifiedSelectable } from '../../../../../shared/simplified_selectable/simplified_selectable';
import { CrawlCustomSettingsFlyoutDomainConfigLogic } from './crawl_custom_settings_flyout_domain_logic';
import { CrawlCustomSettingsFlyoutLogic } from './crawl_custom_settings_flyout_logic';
interface CrawlCustomSettingsFlyoutDomainsPanelProps {
@ -33,7 +34,8 @@ interface CrawlCustomSettingsFlyoutDomainsPanelProps {
}
export const CrawlCustomSettingsFlyoutDomainsPanelWithLogicProps: React.FC = () => {
const { domainUrls, selectedDomainUrls } = useValues(CrawlCustomSettingsFlyoutLogic);
const { selectedDomainUrls } = useValues(CrawlCustomSettingsFlyoutLogic);
const { domainUrls } = useValues(CrawlCustomSettingsFlyoutDomainConfigLogic);
const { onSelectDomainUrls } = useActions(CrawlCustomSettingsFlyoutLogic);
return (

181
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_logic.test.ts
View file

@ -4,58 +4,43 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { LogicMounter, mockHttpValues } from '../../../../../__mocks__/kea_logic';
import '../../_mocks_/index_name_logic.mock';
import { LogicMounter } from '../../../../../__mocks__/kea_logic';
import { nextTick } from '@kbn/test-jest-helpers';
import { itShowsServerErrorAsFlashMessage } from '../../../../../test_helpers';
import { StartSyncApiLogic } from '../../../../api/connector/start_sync_api_logic';
import { DomainConfig } from '../../../../api/crawler/types';
import { CachedFetchIndexApiLogic } from '../../../../api/index/cached_fetch_index_api_logic';
import { DomainConfig, CustomCrawlType } from '../../../../api/crawler/types';
import { IndexNameLogic } from '../../index_name_logic';
import { IndexViewLogic } from '../../index_view_logic';
import { CrawlerLogic } from '../crawler_logic';
import { CrawlCustomSettingsFlyoutDomainConfigLogic } from './crawl_custom_settings_flyout_domain_logic';
import { CrawlCustomSettingsFlyoutLogic } from './crawl_custom_settings_flyout_logic';
import { CrawlCustomSettingsFlyoutMultiCrawlLogic } from './crawl_custom_settings_flyout_multi_crawl_logic';
// Temporarily skipping the tests before FF, the error results from connected kea logic.
// They will be fixed as a separate ticket.
describe.skip('CrawlCustomSettingsFlyoutLogic', () => {
describe('CrawlCustomSettingsFlyoutLogic', () => {
const { mount } = new LogicMounter(CrawlCustomSettingsFlyoutLogic);
const { mount: multiCrawlLogicMount } = new LogicMounter(
CrawlCustomSettingsFlyoutMultiCrawlLogic
);
const { mount: indexViewLogicMount } = new LogicMounter(IndexViewLogic);
const { mount: apiLogicMount } = new LogicMounter(StartSyncApiLogic);
const { mount: fetchIndexMount } = new LogicMounter(CachedFetchIndexApiLogic);
const { mount: indexNameMount } = new LogicMounter(IndexNameLogic);
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
indexNameMount();
apiLogicMount();
fetchIndexMount();
indexViewLogicMount();
multiCrawlLogicMount();
indexNameMount();
mount();
});
it('has expected default values', () => {
expect(CrawlCustomSettingsFlyoutLogic.values).toEqual({
crawlType: CustomCrawlType.ONE_TIME,
customEntryPointUrls: [],
customSitemapUrls: [],
domainConfigMap: {},
domainConfigs: [],
domainUrls: [],
entryPointUrls: [],
includeSitemapsInRobotsTxt: true,
isDataLoading: true,
isFlyoutVisible: false,
isFormSubmitting: false,
isSingleCrawlType: true,
maxCrawlDepth: 2,
selectedDomainUrls: [],
selectedEntryPointUrls: [],
@ -65,96 +50,6 @@ describe.skip('CrawlCustomSettingsFlyoutLogic', () => {
});
describe('actions', () => {
describe('fetchDomainConfigData', () => {
it('updates logic with data that has been converted from server to client', async () => {
jest.spyOn(CrawlCustomSettingsFlyoutLogic.actions, 'onRecieveDomainConfigData');
http.get.mockReturnValueOnce(
Promise.resolve({
meta: {
page: {
current: 1,
size: 1,
total_pages: 2,
},
},
results: [
{
id: '1234',
name: 'https://www.elastic.co',
seed_urls: [],
sitemap_urls: [],
},
],
})
);
http.get.mockReturnValueOnce(
Promise.resolve({
meta: {
page: {
current: 2,
size: 1,
total_pages: 2,
},
},
results: [
{
id: '5678',
name: 'https://www.swiftype.com',
seed_urls: [],
sitemap_urls: [],
},
],
})
);
CrawlCustomSettingsFlyoutLogic.actions.fetchDomainConfigData();
await nextTick();
expect(http.get).toHaveBeenNthCalledWith(
1,
'/internal/enterprise_search/indices/index-name/crawler/domain_configs',
{
query: {
'page[current]': 1,
'page[size]': 100,
},
}
);
expect(http.get).toHaveBeenNthCalledWith(
2,
'/internal/enterprise_search/indices/index-name/crawler/domain_configs',
{
query: {
'page[current]': 2,
'page[size]': 1,
},
}
);
expect(
CrawlCustomSettingsFlyoutLogic.actions.onRecieveDomainConfigData
).toHaveBeenCalledWith([
{
id: '1234',
name: 'https://www.elastic.co',
seedUrls: [],
sitemapUrls: [],
},
{
id: '5678',
name: 'https://www.swiftype.com',
seedUrls: [],
sitemapUrls: [],
},
]);
});
itShowsServerErrorAsFlashMessage(http.get, () => {
CrawlCustomSettingsFlyoutLogic.actions.fetchDomainConfigData();
});
});
describe('hideFlyout', () => {
it('hides the modal', () => {
CrawlCustomSettingsFlyoutLogic.actions.hideFlyout();
@ -443,53 +338,27 @@ describe.skip('CrawlCustomSettingsFlyoutLogic', () => {
describe('selectors', () => {
beforeEach(() => {
jest.clearAllMocks();
mount({
domainConfigs: [
{
name: 'https://www.elastic.co',
sitemapUrls: [
'https://www.elastic.co/sitemap1.xml',
'https://www.elastic.co/sitemap2.xml',
],
seedUrls: ['https://www.elastic.co/', 'https://www.elastic.co/guide'],
},
{
name: 'https://swiftype.com',
sitemapUrls: ['https://swiftype.com/sitemap1.xml', 'https://swiftype.com/sitemap2.xml'],
seedUrls: ['https://swiftype.com/', 'https://swiftype.com/documentation'],
},
],
selectedDomainUrls: ['https://swiftype.com'],
});
});
describe('domainUrls', () => {
it('contains all the domain urls from the domain config', () => {
expect(CrawlCustomSettingsFlyoutLogic.values.domainUrls).toEqual([
'https://www.elastic.co',
'https://swiftype.com',
]);
});
});
describe('domainConfigMap', () => {
it('contains all the domain urls from the domain config', () => {
expect(CrawlCustomSettingsFlyoutLogic.values.domainConfigMap).toEqual({
'https://www.elastic.co': {
name: 'https://www.elastic.co',
sitemapUrls: [
'https://www.elastic.co/sitemap1.xml',
'https://www.elastic.co/sitemap2.xml',
],
seedUrls: ['https://www.elastic.co/', 'https://www.elastic.co/guide'],
},
'https://swiftype.com': {
name: 'https://swiftype.com',
sitemapUrls: ['https://swiftype.com/sitemap1.xml', 'https://swiftype.com/sitemap2.xml'],
seedUrls: ['https://swiftype.com/', 'https://swiftype.com/documentation'],
},
});
});
CrawlCustomSettingsFlyoutDomainConfigLogic.actions.onRecieveDomainConfigData([
{
id: '1',
name: 'https://www.elastic.co',
sitemapUrls: [
'https://www.elastic.co/sitemap1.xml',
'https://www.elastic.co/sitemap2.xml',
],
seedUrls: ['https://www.elastic.co/', 'https://www.elastic.co/guide'],
},
{
id: '2',
name: 'https://swiftype.com',
sitemapUrls: ['https://swiftype.com/sitemap1.xml', 'https://swiftype.com/sitemap2.xml'],
seedUrls: ['https://swiftype.com/', 'https://swiftype.com/documentation'],
},
]);
});
describe('entryPointUrls', () => {

110
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_logic.ts
View file

@ -7,21 +7,13 @@
import { kea, MakeLogicType } from 'kea';
import { Meta } from '../../../../../../../common/types';
import { flashAPIErrors } from '../../../../../shared/flash_messages';
import { HttpLogic } from '../../../../../shared/http';
import {
CustomCrawlType,
DomainConfig,
DomainConfigFromServer,
CrawlerCustomSchedule,
} from '../../../../api/crawler/types';
import { domainConfigServerToClient } from '../../../../api/crawler/utils';
import { IndexNameLogic } from '../../index_name_logic';
import { CustomCrawlType, DomainConfig } from '../../../../api/crawler/types';
import { CrawlerActions, CrawlerLogic, CrawlRequestOverrides } from '../crawler_logic';
import { extractDomainAndEntryPointFromUrl } from '../domain_management/add_domain/utils';
import { CrawlCustomSettingsFlyoutDomainConfigLogic } from './crawl_custom_settings_flyout_domain_logic';
import { CrawlCustomSettingsFlyoutMultiCrawlLogic } from './crawl_custom_settings_flyout_multi_crawl_logic';
export interface CrawlCustomSettingsFlyoutLogicValues {
@ -44,9 +36,6 @@ export interface CrawlCustomSettingsFlyoutLogicValues {
selectedEntryPointUrls: string[];
selectedSitemapUrls: string[];
sitemapUrls: string[];
crawlerConfigurations: CrawlerCustomSchedule[];
multiCrawlerSitemapUrls: string[][];
multiCrawlerEntryPointUrls: string[][];
}
export interface CrawlCustomSettingsFlyoutLogicActions {
@ -89,16 +78,16 @@ export const CrawlCustomSettingsFlyoutLogic = kea<
actions: [
CrawlerLogic,
['startCrawl'],
CrawlCustomSettingsFlyoutDomainConfigLogic,
['fetchDomainConfigData', 'onRecieveDomainConfigData'],
CrawlCustomSettingsFlyoutMultiCrawlLogic,
['fetchCustomScheduling', 'postCustomScheduling'],
],
values: [CrawlCustomSettingsFlyoutMultiCrawlLogic, ['crawlerConfigurations']],
values: [CrawlCustomSettingsFlyoutDomainConfigLogic, ['domainConfigs', 'domainConfigMap']],
},
actions: () => ({
fetchDomainConfigData: true,
saveCustomSchedulingConfiguration: true,
hideFlyout: true,
onRecieveDomainConfigData: (domainConfigs) => ({ domainConfigs }),
onSelectCrawlType: (crawlType) => ({ crawlType }),
onSelectCustomEntryPointUrls: (entryPointUrls) => ({ entryPointUrls }),
onSelectCustomSitemapUrls: (sitemapUrls) => ({ sitemapUrls }),
@ -131,12 +120,6 @@ export const CrawlCustomSettingsFlyoutLogic = kea<
onSelectCustomSitemapUrls: (_, { sitemapUrls }) => sitemapUrls,
},
],
domainConfigs: [
[],
{
onRecieveDomainConfigData: (_, { domainConfigs }) => domainConfigs,
},
],
includeSitemapsInRobotsTxt: [
true,
{
@ -202,20 +185,11 @@ export const CrawlCustomSettingsFlyoutLogic = kea<
],
}),
selectors: () => ({
domainUrls: [
(selectors) => [selectors.domainConfigs],
(domainConfigs: DomainConfig[]) => domainConfigs.map((domainConfig) => domainConfig.name),
],
domainConfigMap: [
(selectors) => [selectors.domainConfigs],
(domainConfigs: DomainConfig[]) =>
domainConfigs.reduce(
(acc, domainConfig) => ({ ...acc, [domainConfig.name]: domainConfig }),
{} as { [key: string]: DomainConfig }
),
],
entryPointUrls: [
(selectors) => [selectors.domainConfigMap, selectors.selectedDomainUrls],
(selectors) => [
CrawlCustomSettingsFlyoutDomainConfigLogic.selectors.domainConfigMap,
selectors.selectedDomainUrls,
],
(domainConfigMap: { [key: string]: DomainConfig }, selectedDomainUrls: string[]): string[] =>
selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].seedUrls
@ -226,72 +200,18 @@ export const CrawlCustomSettingsFlyoutLogic = kea<
(crawlType: string): boolean => crawlType === CustomCrawlType.ONE_TIME,
],
sitemapUrls: [
(selectors) => [selectors.domainConfigMap, selectors.selectedDomainUrls],
(selectors) => [
CrawlCustomSettingsFlyoutDomainConfigLogic.selectors.domainConfigMap,
selectors.selectedDomainUrls,
],
(domainConfigMap: { [key: string]: DomainConfig }, selectedDomainUrls: string[]): string[] =>
selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].sitemapUrls
),
],
multiCrawlerEntryPointUrls: [
(selectors) => [selectors.domainConfigMap, selectors.crawlerConfigurations],
(
domainConfigMap: { [key: string]: DomainConfig },
crawlerConfigs: CrawlerCustomSchedule[]
): string[][] =>
crawlerConfigs.map((c) =>
c.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].seedUrls
)
),
],
multiCrawlerSitemapUrls: [
(selectors) => [selectors.domainConfigMap, selectors.crawlerConfigurations],
(
domainConfigMap: { [key: string]: DomainConfig },
crawlerConfigs: CrawlerCustomSchedule[]
): string[][] =>
crawlerConfigs.map((c) =>
c.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].sitemapUrls
)
),
],
}),
listeners: ({ actions, values }) => ({
fetchDomainConfigData: async () => {
const { http } = HttpLogic.values;
const { indexName } = IndexNameLogic.values;
let domainConfigs: DomainConfig[] = [];
let totalPages: number = 1;
let nextPage: number = 1;
let pageSize: number = 100;
try {
while (nextPage <= totalPages) {
const {
results,
meta: { page },
} = await http.get<{
meta: Meta;
results: DomainConfigFromServer[];
}>(`/internal/enterprise_search/indices/${indexName}/crawler/domain_configs`, {
query: { 'page[current]': nextPage, 'page[size]': pageSize },
});
domainConfigs = [...domainConfigs, ...results.map(domainConfigServerToClient)];
nextPage = page.current + 1;
totalPages = page.total_pages;
pageSize = page.size;
}
actions.onRecieveDomainConfigData(domainConfigs);
} catch (e) {
flashAPIErrors(e);
}
},
showFlyout: () => {
showFlyout: async () => {
actions.fetchDomainConfigData();
actions.fetchCustomScheduling();
},

161
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_multi_crawl_logic.ts
View file

@ -8,14 +8,13 @@
import { kea, MakeLogicType } from 'kea';
import { ConnectorScheduling } from '../../../../../../../common/types/connectors';
import {
CrawlerCustomSchedulesServer,
CrawlerCustomScheduleClient,
} from '../../../../../../../common/types/crawler';
import { CrawlerCustomSchedulesServer } from '../../../../../../../common/types/crawler';
import { CrawlerIndex } from '../../../../../../../common/types/indices';
import { Actions } from '../../../../../shared/api_logic/create_api_logic';
import { flashAPIErrors } from '../../../../../shared/flash_messages';
import { HttpLogic } from '../../../../../shared/http';
import { CrawlerCustomSchedule } from '../../../../api/crawler/types';
import { DomainConfig, CrawlerCustomSchedule } from '../../../../api/crawler/types';
import {
crawlerCustomSchedulingServerToClient,
crawlerCustomSchedulingClientToServer,
@ -24,15 +23,33 @@ import { IndexNameLogic } from '../../index_name_logic';
import { IndexViewLogic } from '../../index_view_logic';
import { filterSeedUrlsByDomainUrls } from './crawl_custom_settings_flyout_logic';
import {
CrawlCustomSettingsFlyoutDomainConfigLogic,
domainConfigsToDomainConfigMap,
} from './crawl_custom_settings_flyout_domain_logic';
export interface CrawlCustomSettingsFlyoutLogicValues {
import { filterSeedUrlsByDomainUrls } from './crawl_custom_settings_flyout_logic';
import {
PostCustomSchedulingApiLogic,
PostCustomSchedulingArgs,
} from './crawl_custom_settings_flyout_schedule_api_logic';
export interface CrawlCustomSettingsFlyoutMultiCrawlLogicValues {
crawlerConfigActiveTab: number;
crawlerConfigurations: CrawlerCustomSchedule[];
crawlerConfigurationsWithDomainData: CrawlerCustomSchedule[];
index: CrawlerIndex;
domainUrls: string[];
domainConfigs: DomainConfig[];
domainConfigMap: {
[key: string]: DomainConfig;
};
crawlerCustomSchedulingIsValid: boolean;
}
export interface CrawlCustomSettingsFlyoutLogicActions {
type PostCustomSchedulingApiValues = Actions<PostCustomSchedulingArgs, {}>;
export interface CrawlCustomSettingsFlyoutMultiCrawlLogicActions {
fetchCustomScheduling(): void;
postCustomScheduling(): void;
onReceiveCrawlerCustomScheduling(crawlerConfigurations: CrawlerCustomSchedule[]): {
@ -79,9 +96,11 @@ export interface CrawlCustomSettingsFlyoutLogicActions {
enabled: boolean;
};
toggleIncludeSitemapsInRobotsTxt(index: number): { index: number };
makePostCustomSchedulingRequest: PostCustomSchedulingApiValues['makeRequest'];
}
const defaulCrawlerConfiguration: CrawlerCustomSchedule = {
scheduleKey: 'crawler_0',
name: 'Crawler 0',
maxCrawlDepth: 2,
customEntryPointUrls: [],
@ -90,16 +109,32 @@ const defaulCrawlerConfiguration: CrawlerCustomSchedule = {
selectedDomainUrls: [],
selectedEntryPointUrls: [],
selectedSitemapUrls: [],
interval: '* * * * *',
interval: '0 0 0 * * ?',
enabled: false,
sitemapUrls: [],
entryPointUrls: [],
};
export const CrawlCustomSettingsFlyoutMultiCrawlLogic = kea<
MakeLogicType<CrawlCustomSettingsFlyoutLogicValues, CrawlCustomSettingsFlyoutLogicActions>
MakeLogicType<
CrawlCustomSettingsFlyoutMultiCrawlLogicValues,
CrawlCustomSettingsFlyoutMultiCrawlLogicActions
>
>({
path: ['enterprise_search', 'crawler', 'crawl_custom_settings_flyout_multi_crawl_logic'],
connect: {
values: [IndexViewLogic, ['index']],
actions: [
PostCustomSchedulingApiLogic,
['makeRequest as makePostCustomSchedulingRequest'],
CrawlCustomSettingsFlyoutDomainConfigLogic,
['onRecieveDomainConfigData'],
],
values: [
IndexViewLogic,
['index'],
CrawlCustomSettingsFlyoutDomainConfigLogic,
['domainConfigs', 'domainConfigMap'],
],
},
actions: () => ({
fetchCustomScheduling: true,
@ -130,15 +165,36 @@ export const CrawlCustomSettingsFlyoutMultiCrawlLogic = kea<
[defaulCrawlerConfiguration],
{
onReceiveCrawlerCustomScheduling: (_, { crawlerConfigurations }) => {
return crawlerConfigurations.map((configuration) => ({
...defaulCrawlerConfiguration,
...configuration,
}));
// Handle case with no custom scheduling returned from server
return crawlerConfigurations.length > 0
? crawlerConfigurations.map((configuration) => ({
...defaulCrawlerConfiguration,
...configuration,
}))
: [defaulCrawlerConfiguration];
},
onAddCustomCrawler: (state, { index }) => {
let newScheduleKey = `crawler_${index}`;
let suffix = index;
// Check if the newScheduleKey already exists in the array
const existingKeys = state.map((crawler) => crawler.scheduleKey);
if (existingKeys.includes(newScheduleKey)) {
// Handle the case where a duplicate scheduleKey is found
while (existingKeys.includes(`${newScheduleKey}_${suffix}`)) {
suffix++;
}
newScheduleKey = `${newScheduleKey}_${suffix}`;
}
return [
...state,
{
...defaulCrawlerConfiguration,
name: `Crawler ${suffix}`,
scheduleKey: newScheduleKey,
},
];
},
onAddCustomCrawler: (state, { index }) => [
...state,
{ ...defaulCrawlerConfiguration, name: `Crawler ${index}` },
],
onDeleteCustomCrawler: (state, { index }) => {
return state.filter((_, i) => i !== index);
},
@ -197,9 +253,70 @@ export const CrawlCustomSettingsFlyoutMultiCrawlLogic = kea<
const { interval } = newSchedule;
return state.map((crawler, i) => (i === index ? { ...crawler, interval } : crawler));
},
onRecieveDomainConfigData: (state, { domainConfigs }) => {
const domainConfigsMap = domainConfigsToDomainConfigMap(domainConfigs);
return state.map((crawler) => {
const entryPointUrls = crawler.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigsMap[selectedDomainUrl].seedUrls
);
const selectedEntryPointUrls = crawler.customEntryPointUrls.filter((entryPointUrl) =>
entryPointUrls.includes(entryPointUrl)
);
const customEntryPointUrls = crawler.customEntryPointUrls.filter(
(entryPointUrl) => !entryPointUrls.includes(entryPointUrl)
);
const sitemapUrls = crawler.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigsMap[selectedDomainUrl].sitemapUrls
);
const selectedSitemapUrls = crawler.customSitemapUrls.filter((sitemapUrl) =>
sitemapUrls.includes(sitemapUrl)
);
const customSitemapUrls = crawler.customSitemapUrls.filter(
(sitemapUrl) => !sitemapUrls.includes(sitemapUrl)
);
return {
...crawler,
entryPointUrls,
selectedEntryPointUrls,
customEntryPointUrls,
sitemapUrls,
selectedSitemapUrls,
customSitemapUrls,
};
});
},
},
],
}),
selectors: () => ({
crawlerConfigurationsWithDomainData: [
(selectors) => [selectors.domainConfigMap, selectors.crawlerConfigurations],
(
domainConfigMap: { [key: string]: DomainConfig },
crawlerConfigs: CrawlerCustomSchedule[]
): CrawlerCustomSchedule[] =>
crawlerConfigs.map((crawlerConfig) => {
const entryPointUrls = crawlerConfig.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].seedUrls
);
const sitemapUrls = crawlerConfig.selectedDomainUrls.flatMap(
(selectedDomainUrl) => domainConfigMap[selectedDomainUrl].sitemapUrls
);
return {
...crawlerConfig,
entryPointUrls,
sitemapUrls,
};
}),
],
crawlerCustomSchedulingIsValid: [
(selectors) => [selectors.crawlerConfigurations],
(crawlerConfigs: CrawlerCustomSchedule[]): boolean =>
crawlerConfigs.every((config) => config.selectedDomainUrls.length > 0),
],
}),
listeners: ({ actions, values }) => ({
fetchCustomScheduling: async () => {
const { http } = HttpLogic.values;
@ -216,15 +333,11 @@ export const CrawlCustomSettingsFlyoutMultiCrawlLogic = kea<
}
},
postCustomScheduling: async () => {
const { http } = HttpLogic.values;
const { indexName } = IndexNameLogic.values;
const { crawlerConfigurations } = values;
const customScheduling = crawlerCustomSchedulingClientToServer(crawlerConfigurations);
try {
await http.post<CrawlerCustomScheduleClient>(
`/internal/enterprise_search/indices/${indexName}/crawler/custom_scheduling`,
{ body: JSON.stringify(Object.fromEntries(customScheduling)) }
);
actions.makePostCustomSchedulingRequest({ indexName, customScheduling });
} catch (e) {
flashAPIErrors(e);
}

2
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_multi_crawl_tabs.tsx
View file

@ -31,7 +31,7 @@ export const CrawlCustomSettingsFlyoutMultipleCrawlTabs: React.FC = () => {
);
const crawlerTabData = crawlerConfigurations.map((_, index) => ({
key: `crawl_${index}`,
key: `crawler_${index}`,
index,
label: `${CRAWLER_TAB_PREFIX} ${index + 1}`,
}));

12
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_mutli_crawl.tsx
View file

@ -13,18 +13,16 @@ import { EuiSpacer } from '@elastic/eui';
import { CrawlCustomSettingsFlyoutCrawlDepthPanel } from './crawl_custom_settings_flyout_crawl_depth_panel';
import { MultiCrawlScheduler } from './crawl_custom_settings_flyout_crawl_scheduler';
import { CrawlCustomSettingsFlyoutDomainConfigLogic } from './crawl_custom_settings_flyout_domain_logic';
import { CrawlCustomSettingsFlyoutDomainsPanel } from './crawl_custom_settings_flyout_domains_panel';
import { CrawlCustomSettingsFlyoutLogic } from './crawl_custom_settings_flyout_logic';
import { CrawlCustomSettingsFlyoutMultiCrawlLogic } from './crawl_custom_settings_flyout_multi_crawl_logic';
import { CrawlCustomSettingsFlyoutSeedUrlsPanel } from './crawl_custom_settings_flyout_seed_urls_panel';
export const CrawlCustomSettingsFlyoutMultiCrawlScheduling: React.FC = () => {
const { domainUrls, multiCrawlerEntryPointUrls, multiCrawlerSitemapUrls } = useValues(
CrawlCustomSettingsFlyoutLogic
);
const { domainUrls } = useValues(CrawlCustomSettingsFlyoutDomainConfigLogic);
const {
crawlerConfigurations,
crawlerConfigurationsWithDomainData,
crawlerConfigActiveTab,
index: crawlerIndex,
} = useValues(CrawlCustomSettingsFlyoutMultiCrawlLogic);
@ -43,7 +41,7 @@ export const CrawlCustomSettingsFlyoutMultiCrawlScheduling: React.FC = () => {
return (
<>
{crawlerConfigurations.map((config, index) => {
{crawlerConfigurationsWithDomainData.map((config, index) => {
if (index === crawlerConfigActiveTab) {
return (
<React.Fragment key={index}>
@ -65,8 +63,6 @@ export const CrawlCustomSettingsFlyoutMultiCrawlScheduling: React.FC = () => {
onSelectEntryPointUrls={(e) => onSelectEntryPointUrls(index, e)}
onSelectSitemapUrls={(e) => onSelectSitemapUrls(index, e)}
toggleIncludeSitemapsInRobotsTxt={() => toggleIncludeSitemapsInRobotsTxt(index)}
entryPointUrls={multiCrawlerEntryPointUrls[index]}
sitemapUrls={multiCrawlerSitemapUrls[index]}
/>
<EuiSpacer />
<MultiCrawlScheduler

44
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_schedule_api_logic.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import {
CrawlerCustomScheduleClient,
CrawlerCustomScheduleMappingClient,
} from '../../../../../../../common/types/crawler';
import { createApiLogic } from '../../../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../../../shared/http';
export interface PostCustomSchedulingArgs {
indexName: string;
customScheduling: CrawlerCustomScheduleMappingClient;
}
export const postCrawlerCustomScheduling = async ({
indexName,
customScheduling,
}: PostCustomSchedulingArgs) => {
const route = `/internal/enterprise_search/indices/${indexName}/crawler/custom_scheduling`;
await HttpLogic.values.http.post<CrawlerCustomScheduleClient>(route, {
body: JSON.stringify(Object.fromEntries(customScheduling)),
});
};
export const PostCustomSchedulingApiLogic = createApiLogic(
['post_crawler_custom_scheduling_api_logic'],
postCrawlerCustomScheduling,
{
showSuccessFlashFn: () =>
i18n.translate(
'xpack.enterpriseSearch.crawler.crawlCustomSettingsFlyout.postCrawlerCustomSchedulingSuccess.message',
{
defaultMessage: 'Successfully saved crawler custom scheduling.',
}
),
}
);

4
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_seed_urls_panel.test.tsx
View file

@ -71,14 +71,14 @@ describe('CrawlCustomSettingsFlyoutSeedUrlsPanel', () => {
selectedDomainUrls: MOCK_VALUES.selectedDomainUrls,
selectedEntryPointUrls: MOCK_VALUES.selectedEntryPointUrls,
selectedSitemapUrls: MOCK_VALUES.selectedSitemapUrls,
entryPointUrls: MOCK_VALUES.entryPointUrls,
sitemapUrls: MOCK_VALUES.sitemapUrls,
}}
onSelectCustomEntryPointUrls={MOCK_ACTIONS.onSelectCustomEntryPointUrls}
onSelectCustomSitemapUrls={MOCK_ACTIONS.onSelectCustomSitemapUrls}
onSelectEntryPointUrls={MOCK_ACTIONS.onSelectEntryPointUrls}
onSelectSitemapUrls={MOCK_ACTIONS.onSelectSitemapUrls}
toggleIncludeSitemapsInRobotsTxt={MOCK_ACTIONS.toggleIncludeSitemapsInRobotsTxt}
entryPointUrls={MOCK_VALUES.entryPointUrls}
sitemapUrls={MOCK_VALUES.sitemapUrls}
/>
);
});

14
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_custom_settings_flyout/crawl_custom_settings_flyout_seed_urls_panel.tsx
View file

@ -41,6 +41,8 @@ type CrawlerCustomScheduleConfig = Pick<
| 'selectedDomainUrls'
| 'selectedEntryPointUrls'
| 'selectedSitemapUrls'
| 'entryPointUrls'
| 'sitemapUrls'
>;
interface CrawlCustomSettingsFlyoutSeedUrlsPanelProps {
@ -50,8 +52,6 @@ interface CrawlCustomSettingsFlyoutSeedUrlsPanelProps {
onSelectEntryPointUrls: (urls: string[]) => void;
onSelectSitemapUrls: (urls: string[]) => void;
toggleIncludeSitemapsInRobotsTxt: () => void;
entryPointUrls: string[];
sitemapUrls: string[];
}
export const CrawlCustomSettingsFlyoutSeedUrlsPanelWithLogicProps: React.FC = () => {
@ -80,6 +80,8 @@ export const CrawlCustomSettingsFlyoutSeedUrlsPanelWithLogicProps: React.FC = ()
selectedDomainUrls,
selectedEntryPointUrls,
selectedSitemapUrls,
entryPointUrls,
sitemapUrls,
};
return (
@ -90,8 +92,6 @@ export const CrawlCustomSettingsFlyoutSeedUrlsPanelWithLogicProps: React.FC = ()
onSelectEntryPointUrls={onSelectEntryPointUrls}
onSelectSitemapUrls={onSelectSitemapUrls}
toggleIncludeSitemapsInRobotsTxt={toggleIncludeSitemapsInRobotsTxt}
entryPointUrls={entryPointUrls}
sitemapUrls={sitemapUrls}
/>
);
};
@ -105,8 +105,6 @@ export const CrawlCustomSettingsFlyoutSeedUrlsPanel: React.FC<
onSelectEntryPointUrls,
onSelectSitemapUrls,
toggleIncludeSitemapsInRobotsTxt,
entryPointUrls,
sitemapUrls,
}) => {
const totalSeedUrls =
scheduleConfig.customEntryPointUrls.length +
@ -187,7 +185,7 @@ export const CrawlCustomSettingsFlyoutSeedUrlsPanel: React.FC<
</EuiPanel>
<SimplifiedSelectable
data-telemetry-id="entSearchContent-crawler-customCrawlSettings-selectDomain"
options={sitemapUrls}
options={scheduleConfig.sitemapUrls}
selectedOptions={scheduleConfig.selectedSitemapUrls}
onChange={onSelectSitemapUrls}
emptyMessage={
@ -229,7 +227,7 @@ export const CrawlCustomSettingsFlyoutSeedUrlsPanel: React.FC<
<EuiSpacer size="s" />
<SimplifiedSelectable
data-telemetry-id="entSearchContent-crawler-customCrawlSettings-selectDomain"
options={entryPointUrls}
options={scheduleConfig.entryPointUrls}
selectedOptions={scheduleConfig.selectedEntryPointUrls}
onChange={onSelectEntryPointUrls}
emptyMessage={

11
x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts
View file

@ -24,3 +24,14 @@ export const fetchCrawlerCustomSchedulingByIndexName = async (
const result = crawlerResult.hits.hits[0]?._source;
return result;
};
export const fetchCrawlerCustomSchedulingKeysByIndexName = async (
client: IScopedClusterClient,
indexName: string
): Promise<string[]> => {
const crawlerCustomSchedules = await fetchCrawlerCustomSchedulingByIndexName(client, indexName);
return crawlerCustomSchedules?.custom_scheduling
? Object.keys(crawlerCustomSchedules.custom_scheduling)
: [];
};

30
x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts
View file

@ -15,6 +15,7 @@ import {
CrawlerCustomScheduleServer,
} from '../../../common/types/crawler';
import { fetchCrawlerCustomSchedulingKeysByIndexName } from './fetch_crawler_multiple_schedules';
import { fetchCrawlerDocumentIdByIndexName } from './fetch_crawlers';
const convertCustomScheduleMappingClientToServer = (
@ -64,12 +65,37 @@ export const postCrawlerCustomScheduling = async (
customSchedules: CrawlerCustomScheduleMappingClient
) => {
const connectorId = await fetchCrawlerDocumentIdByIndexName(client, indexName);
const convertCustomSchedulesServer = convertCustomScheduleMappingClientToServer(customSchedules);
const customSchedulingPayload = convertCustomScheduleMappingClientToServer(customSchedules);
const existingCustomScheduleKeys = await fetchCrawlerCustomSchedulingKeysByIndexName(
client,
indexName
);
const newCustomScheduleKeys = Array.from(customSchedulingPayload.keys());
const scheduleKeysToDelete = existingCustomScheduleKeys.filter(
(key) => !newCustomScheduleKeys.includes(key)
);
// Handle deleted schedules
if (scheduleKeysToDelete.length > 0) {
const scriptSource = scheduleKeysToDelete
.map((scheduleKey) => `ctx._source['custom_scheduling'].remove('${scheduleKey}');`)
.join(' ');
await client.asCurrentUser.update({
index: CONNECTORS_INDEX,
id: connectorId,
script: {
source: scriptSource,
},
});
}
return await client.asCurrentUser.update({
index: CONNECTORS_INDEX,
id: connectorId,
doc: {
custom_scheduling: Object.fromEntries(convertCustomSchedulesServer),
custom_scheduling: Object.fromEntries(customSchedulingPayload),
},
});
};