github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[App Search] Migrate duplicate document handling UX for Crawler domains (#108623)

Browse source
This commit is contained in:
Byron Hulcher 2021-08-17 11:59:51 -04:00 • committed by GitHub
parent d07f7a5d5e
commit 720a609266
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 722 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_requests_table.test.tsx
View file

@ -43,6 +43,9 @@ const values: { domains: CrawlerDomain[]; crawlRequests: CrawlRequest[] } = {
rule: CrawlerRules.regex,
pattern: '.*',
},
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
],
crawlRequests: [

14
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/deduplication_panel.scss Normal file
View file

@ -0,0 +1,14 @@
.deduplicationPanel {
.selectableWrapper {
padding: $euiSize;
border-radius: $euiSize *.675;
border: $euiBorderThin solid $euiColorLightestShade;
}
.showAllFieldsPopoverToggle {
.euiButtonEmpty__content {
padding-left: $euiSizeM;
padding-right: $euiSizeM;
}
}
}

170
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/deduplication_panel.test.tsx Normal file
View file

@ -0,0 +1,170 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { setMockActions, setMockValues } from '../../../../../__mocks__/kea_logic';
import React from 'react';
import { shallow } from 'enzyme';
import { act } from 'react-dom/test-utils';
import {
EuiButton,
EuiButtonEmpty,
EuiContextMenuItem,
EuiPopover,
EuiSelectable,
EuiSelectableList,
EuiSelectableSearch,
EuiSwitch,
} from '@elastic/eui';
import { mountWithIntl, rerender } from '../../../../../test_helpers';
import { DeduplicationPanel } from './deduplication_panel';
const MOCK_ACTIONS = {
submitDeduplicationUpdate: jest.fn(),
};
const MOCK_VALUES = {
domain: {
deduplicationEnabled: true,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
};
describe('DeduplicationPanel', () => {
beforeEach(() => {
jest.clearAllMocks();
setMockActions(MOCK_ACTIONS);
setMockValues(MOCK_VALUES);
});
it('renders an empty component if no domain', () => {
setMockValues({
...MOCK_VALUES,
domain: null,
});
const wrapper = shallow(<DeduplicationPanel />);
expect(wrapper.isEmptyRender()).toBe(true);
});
it('contains a button to reset to defaults', () => {
const wrapper = shallow(<DeduplicationPanel />);
wrapper.find(EuiButton).simulate('click');
expect(MOCK_ACTIONS.submitDeduplicationUpdate).toHaveBeenCalledWith(MOCK_VALUES.domain, {
fields: [],
});
});
it('contains a switch to enable and disable deduplication', () => {
setMockValues({
...MOCK_VALUES,
domain: {
...MOCK_VALUES.domain,
deduplicationEnabled: false,
},
});
const wrapper = shallow(<DeduplicationPanel />);
wrapper.find(EuiSwitch).simulate('change');
expect(MOCK_ACTIONS.submitDeduplicationUpdate).toHaveBeenNthCalledWith(
1,
{
...MOCK_VALUES.domain,
deduplicationEnabled: false,
},
{
enabled: true,
}
);
setMockValues({
...MOCK_VALUES,
domain: {
...MOCK_VALUES.domain,
deduplicationEnabled: true,
},
});
rerender(wrapper);
wrapper.find(EuiSwitch).simulate('change');
expect(MOCK_ACTIONS.submitDeduplicationUpdate).toHaveBeenNthCalledWith(
2,
{
...MOCK_VALUES.domain,
deduplicationEnabled: true,
},
{
enabled: false,
fields: [],
}
);
});
it('contains a popover to switch between displaying all fields or only selected ones', () => {
const fullRender = mountWithIntl(<DeduplicationPanel />);
expect(fullRender.find(EuiButtonEmpty).text()).toEqual('All fields');
expect(fullRender.find(EuiPopover).prop('isOpen')).toEqual(false);
// Open the popover
fullRender.find(EuiButtonEmpty).simulate('click');
rerender(fullRender);
expect(fullRender.find(EuiPopover).prop('isOpen')).toEqual(true);
// Click "Show selected fields"
fullRender.find(EuiContextMenuItem).at(1).simulate('click');
rerender(fullRender);
expect(fullRender.find(EuiButtonEmpty).text()).toEqual('Selected fields');
expect(fullRender.find(EuiPopover).prop('isOpen')).toEqual(false);
// Open the popover and click "show all fields"
fullRender.find(EuiButtonEmpty).simulate('click');
fullRender.find(EuiContextMenuItem).at(0).simulate('click');
rerender(fullRender);
expect(fullRender.find(EuiButtonEmpty).text()).toEqual('All fields');
expect(fullRender.find(EuiPopover).prop('isOpen')).toEqual(false);
// Open the popover then simulate closing the popover
fullRender.find(EuiButtonEmpty).simulate('click');
act(() => {
fullRender.find(EuiPopover).prop('closePopover')();
});
rerender(fullRender);
expect(fullRender.find(EuiPopover).prop('isOpen')).toEqual(false);
});
it('contains a selectable to toggle fields for deduplication', () => {
const wrapper = shallow(<DeduplicationPanel />);
wrapper
.find(EuiSelectable)
.simulate('change', [{ label: 'title' }, { label: 'description', checked: 'on' }]);
expect(MOCK_ACTIONS.submitDeduplicationUpdate).toHaveBeenCalledWith(MOCK_VALUES.domain, {
fields: ['description'],
});
const fullRender = mountWithIntl(<DeduplicationPanel />);
expect(fullRender.find(EuiSelectableSearch)).toHaveLength(1);
expect(fullRender.find(EuiSelectableList)).toHaveLength(1);
});
});

214
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/deduplication_panel.tsx Normal file
View file

@ -0,0 +1,214 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useState } from 'react';
import { useActions, useValues } from 'kea';
import {
EuiButton,
EuiButtonEmpty,
EuiContextMenuItem,
EuiContextMenuPanel,
EuiFlexGroup,
EuiFlexItem,
EuiLink,
EuiPopover,
EuiSelectable,
EuiSpacer,
EuiSwitch,
EuiText,
EuiTitle,
} from '@elastic/eui';
import { EuiSelectableLIOption } from '@elastic/eui/src/components/selectable/selectable_option';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import { DOCS_PREFIX } from '../../../../routes';
import { CrawlerSingleDomainLogic } from '../../crawler_single_domain_logic';
import { getCheckedOptionLabels, getSelectableOptions } from './utils';
import './deduplication_panel.scss';
export const DeduplicationPanel: React.FC = () => {
const { domain } = useValues(CrawlerSingleDomainLogic);
const { submitDeduplicationUpdate } = useActions(CrawlerSingleDomainLogic);
const [showAllFields, setShowAllFields] = useState(true);
const [showAllFieldsPopover, setShowAllFieldsPopover] = useState(false);
if (!domain) {
return null;
}
const { deduplicationEnabled, deduplicationFields } = domain;
const selectableOptions = getSelectableOptions(domain, showAllFields);
return (
<div className="deduplicationPanel">
<EuiFlexGroup direction="row" alignItems="stretch">
<EuiFlexItem>
<EuiTitle size="s">
<h2>
{i18n.translate('xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.title', {
defaultMessage: 'Duplicate document handling',
})}
</h2>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButton
color="warning"
iconType="refresh"
size="s"
onClick={() => submitDeduplicationUpdate(domain, { fields: [] })}
disabled={deduplicationFields.length === 0}
>
{i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.resetToDefaultsButtonLabel',
{
defaultMessage: 'Reset to defaults',
}
)}
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size="m" />
<EuiText size="s" color="subdued">
<p>
<FormattedMessage
id="xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.description"
defaultMessage="The web crawler only indexes unique pages. Choose which fields the crawler should use when
considering which pages are duplicates. Deselect all schema fields to allow duplicate
documents on this domain. {documentationLink}."
values={{
documentationLink: (
<EuiLink
href={`${DOCS_PREFIX}/web-crawler-reference.html#web-crawler-reference-content-deduplication`}
target="_blank"
external
>
{i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.learnMoreMessage',
{
defaultMessage: 'Learn more about content hashing',
}
)}
</EuiLink>
),
}}
/>
</p>
</EuiText>
<EuiSpacer size="l" />
<EuiSwitch
label="Prevent duplicate documents"
checked={deduplicationEnabled}
onChange={() =>
deduplicationEnabled
? submitDeduplicationUpdate(domain, { enabled: false, fields: [] })
: submitDeduplicationUpdate(domain, { enabled: true })
}
/>
<EuiFlexGroup>
<EuiFlexItem>
<div className="selectableWrapper">
<EuiSelectable
options={selectableOptions}
onChange={(options) =>
submitDeduplicationUpdate(domain, {
fields: getCheckedOptionLabels(options as Array<EuiSelectableLIOption<object>>),
})
}
searchable
searchProps={{
disabled: !deduplicationEnabled,
append: (
<EuiPopover
button={
<EuiButtonEmpty
size="xs"
iconType="arrowDown"
iconSide="right"
onClick={() => setShowAllFieldsPopover(!showAllFieldsPopover)}
className="showAllFieldsPopoverToggle"
disabled={!deduplicationEnabled}
>
{showAllFields
? i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.allFieldsLabel',
{
defaultMessage: 'All fields',
}
)
: i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.selectedFieldsLabel',
{
defaultMessage: 'Selected fields',
}
)}
</EuiButtonEmpty>
}
isOpen={showAllFieldsPopover}
closePopover={() => setShowAllFieldsPopover(false)}
panelPaddingSize="none"
anchorPosition="downLeft"
>
<EuiContextMenuPanel
items={[
<EuiContextMenuItem
key="all fields"
icon={showAllFields ? 'check' : 'empty'}
onClick={() => {
setShowAllFields(true);
setShowAllFieldsPopover(false);
}}
>
{i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.deduplicationPanel.showAllFieldsButtonLabel',
{
defaultMessage: 'Show all fields',
}
)}
</EuiContextMenuItem>,
<EuiContextMenuItem
key="selected fields"
icon={showAllFields ? 'empty' : 'check'}
onClick={() => {
setShowAllFields(false);
setShowAllFieldsPopover(false);
}}
>
{i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.crawlerStatusIndicator.showSelectedFieldsButtonLabel',
{
defaultMessage: 'Show only selected fields',
}
)}
</EuiContextMenuItem>,
]}
/>
</EuiPopover>
),
}}
>
{(list, search) => (
<>
{search}
{list}
</>
)}
</EuiSelectable>
</div>
</EuiFlexItem>
</EuiFlexGroup>
</div>
);
};

8
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { DeduplicationPanel } from './deduplication_panel';

69
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/utils.test.ts Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiSelectableLIOption } from '@elastic/eui/src/components/selectable/selectable_option';
import { CrawlerDomain } from '../../types';
import { getCheckedOptionLabels, getSelectableOptions } from './utils';
describe('getCheckedOptionLabels', () => {
it('returns the labels of selected options', () => {
const options = [{ label: 'title' }, { label: 'description', checked: 'on' }] as Array<
EuiSelectableLIOption<object>
>;
expect(getCheckedOptionLabels(options)).toEqual(['description']);
});
});
describe('getSelectableOptions', () => {
it('returns all available fields when we want all fields', () => {
expect(
getSelectableOptions(
{
availableDeduplicationFields: ['title', 'description'],
deduplicationFields: ['title'],
deduplicationEnabled: true,
} as CrawlerDomain,
true
)
).toEqual([
{ label: 'title', checked: 'on' },
{ label: 'description', checked: undefined },
]);
});
it('can returns only selected fields', () => {
expect(
getSelectableOptions(
{
availableDeduplicationFields: ['title', 'description'],
deduplicationFields: ['title'],
deduplicationEnabled: true,
} as CrawlerDomain,
false
)
).toEqual([{ label: 'title', checked: 'on' }]);
});
it('disables all options when deduplication is disabled', () => {
expect(
getSelectableOptions(
{
availableDeduplicationFields: ['title', 'description'],
deduplicationFields: ['title'],
deduplicationEnabled: false,
} as CrawlerDomain,
true
)
).toEqual([
{ label: 'title', checked: 'on', disabled: true },
{ label: 'description', checked: undefined, disabled: true },
]);
});
});

39
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/deduplication_panel/utils.ts Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiSelectableLIOption } from '@elastic/eui/src/components/selectable/selectable_option';
import { CrawlerDomain } from '../../types';
export const getSelectableOptions = (
domain: CrawlerDomain,
showAllFields: boolean
): Array<EuiSelectableLIOption<object>> => {
const { availableDeduplicationFields, deduplicationFields, deduplicationEnabled } = domain;
let selectableOptions: Array<EuiSelectableLIOption<object>>;
if (showAllFields) {
selectableOptions = availableDeduplicationFields.map((field) => ({
label: field,
checked: deduplicationFields.includes(field) ? 'on' : undefined,
}));
} else {
selectableOptions = availableDeduplicationFields
.filter((field) => deduplicationFields.includes(field))
.map((field) => ({ label: field, checked: 'on' }));
}
if (!deduplicationEnabled) {
selectableOptions = selectableOptions.map((option) => ({ ...option, disabled: true }));
}
return selectableOptions;
};
export const getCheckedOptionLabels = (options: Array<EuiSelectableLIOption<object>>): string[] => {
return options.filter((option) => option.checked).map((option) => option.label);
};

10
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/delete_domain_panel.tsx
View file

@ -9,7 +9,7 @@ import React from 'react';
import { useActions, useValues } from 'kea';
import { EuiButton, EuiSpacer, EuiText } from '@elastic/eui';
import { EuiButton, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
@ -27,6 +27,14 @@ export const DeleteDomainPanel: React.FC = ({}) => {
return (
<>
<EuiTitle size="s">
<h2>
{i18n.translate('xpack.enterpriseSearch.appSearch.crawler.deleteDomainPanel.title', {
defaultMessage: 'Delete domain',
})}
</h2>
</EuiTitle>
<EuiSpacer size="m" />
<EuiText size="s">
<p>
<FormattedMessage

6
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/domains_table.test.tsx
View file

@ -30,6 +30,9 @@ const domains: CrawlerDomain[] = [
sitemaps: [],
lastCrawl: '2020-01-01T00:00:00-12:00',
createdOn: '2020-01-01T00:00:00-12:00',
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
{
id: '4567',
@ -39,6 +42,9 @@ const domains: CrawlerDomain[] = [
entryPoints: [],
sitemaps: [],
createdOn: '1970-01-01T00:00:00-12:00',
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
];

7
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/entry_points_table.test.tsx
View file

@ -15,6 +15,8 @@ import { GenericEndpointInlineEditableTable } from '../../../../shared/tables/ge
import { mountWithIntl } from '../../../../test_helpers';
import { CrawlerDomain } from '../types';
import { EntryPointsTable } from './entry_points_table';
describe('EntryPointsTable', () => {
@ -23,7 +25,7 @@ describe('EntryPointsTable', () => {
{ id: '1', value: '/whatever' },
{ id: '2', value: '/foo' },
];
const domain = {
const domain: CrawlerDomain = {
createdOn: '2018-01-01T00:00:00.000Z',
documentCount: 10,
id: '6113e1407a2f2e6f42489794',
@ -31,6 +33,9 @@ describe('EntryPointsTable', () => {
crawlRules: [],
entryPoints,
sitemaps: [],
deduplicationEnabled: true,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
};
beforeEach(() => {

3
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/sitemaps_table.test.tsx
View file

@ -34,6 +34,9 @@ describe('SitemapsTable', () => {
crawlRules: [],
entryPoints: [],
sitemaps,
deduplicationEnabled: true,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
};
beforeEach(() => {

6
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview.test.tsx
View file

@ -47,6 +47,9 @@ const domains: CrawlerDomainFromServer[] = [
rule: CrawlerRules.regex,
pattern: '.*',
},
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
},
{
id: 'y',
@ -57,6 +60,9 @@ const domains: CrawlerDomainFromServer[] = [
sitemaps: [],
entry_points: [],
crawl_rules: [],
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
},
];

6
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_overview_logic.test.ts
View file

@ -52,6 +52,9 @@ const MOCK_SERVER_CRAWLER_DATA: CrawlerDataFromServer = {
sitemaps: [],
entry_points: [],
crawl_rules: [],
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
},
],
};
@ -112,6 +115,9 @@ describe('CrawlerOverviewLogic', () => {
entryPoints: [],
crawlRules: [],
defaultCrawlRule: DEFAULT_CRAWL_RULE,
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
],
};

23
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_single_domain.test.tsx
View file

@ -13,15 +13,13 @@ import React from 'react';
import { shallow } from 'enzyme';
import { EuiCode } from '@elastic/eui';
import { getPageHeaderActions } from '../../../test_helpers';
import { CrawlerStatusBanner } from './components/crawler_status_banner';
import { CrawlerStatusIndicator } from './components/crawler_status_indicator/crawler_status_indicator';
import { DeduplicationPanel } from './components/deduplication_panel';
import { DeleteDomainPanel } from './components/delete_domain_panel';
import { ManageCrawlsPopover } from './components/manage_crawls_popover/manage_crawls_popover';
import { CrawlerOverview } from './crawler_overview';
import { CrawlerSingleDomain } from './crawler_single_domain';
const MOCK_VALUES = {
@ -53,7 +51,6 @@ describe('CrawlerSingleDomain', () => {
const wrapper = shallow(<CrawlerSingleDomain />);
expect(wrapper.find(DeleteDomainPanel)).toHaveLength(1);
expect(wrapper.find(EuiCode).render().text()).toContain('https://elastic.co');
expect(wrapper.prop('pageHeader').pageTitle).toEqual('https://elastic.co');
});
@ -71,20 +68,32 @@ describe('CrawlerSingleDomain', () => {
});
it('contains a crawler status banner', () => {
const wrapper = shallow(<CrawlerOverview />);
const wrapper = shallow(<CrawlerSingleDomain />);
expect(wrapper.find(CrawlerStatusBanner)).toHaveLength(1);
});
it('contains a crawler status indicator', () => {
const wrapper = shallow(<CrawlerOverview />);
const wrapper = shallow(<CrawlerSingleDomain />);
expect(getPageHeaderActions(wrapper).find(CrawlerStatusIndicator)).toHaveLength(1);
});
it('contains a popover to manage crawls', () => {
const wrapper = shallow(<CrawlerOverview />);
const wrapper = shallow(<CrawlerSingleDomain />);
expect(getPageHeaderActions(wrapper).find(ManageCrawlsPopover)).toHaveLength(1);
});
it('contains a panel to manage deduplication settings', () => {
const wrapper = shallow(<CrawlerSingleDomain />);
expect(wrapper.find(DeduplicationPanel)).toHaveLength(1);
});
it('contains a panel to delete the domain', () => {
const wrapper = shallow(<CrawlerSingleDomain />);
expect(wrapper.find(DeleteDomainPanel)).toHaveLength(1);
});
});

20
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_single_domain.tsx
View file

@ -11,9 +11,7 @@ import { useParams } from 'react-router-dom';
import { useActions, useValues } from 'kea';
import { EuiCode, EuiPanel, EuiSpacer, EuiTitle } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { EuiPanel, EuiSpacer } from '@elastic/eui';
import { EngineLogic, getEngineBreadcrumbs } from '../engine';
import { AppSearchPageTemplate } from '../layout';
@ -21,6 +19,7 @@ import { AppSearchPageTemplate } from '../layout';
import { CrawlRulesTable } from './components/crawl_rules_table';
import { CrawlerStatusBanner } from './components/crawler_status_banner';
import { CrawlerStatusIndicator } from './components/crawler_status_indicator/crawler_status_indicator';
import { DeduplicationPanel } from './components/deduplication_panel';
import { DeleteDomainPanel } from './components/delete_domain_panel';
import { EntryPointsTable } from './components/entry_points_table';
import { ManageCrawlsPopover } from './components/manage_crawls_popover/manage_crawls_popover';
@ -76,20 +75,9 @@ export const CrawlerSingleDomain: React.FC = () => {
<EuiSpacer size="xxl" />
</>
)}
<EuiTitle size="s">
<h2>
{i18n.translate(
'xpack.enterpriseSearch.appSearch.crawler.singleDomain.deleteDomainTitle',
{
defaultMessage: 'Delete domain',
}
)}
</h2>
</EuiTitle>
<EuiSpacer size="m" />
<DeleteDomainPanel />
<DeduplicationPanel />
<EuiSpacer size="xl" />
<EuiCode>{JSON.stringify(domain, null, 2)}</EuiCode>
<DeleteDomainPanel />
</AppSearchPageTemplate>
);
};

57
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_single_domain_logic.test.ts
View file

@ -216,5 +216,62 @@ describe('CrawlerSingleDomainLogic', () => {
expect(flashAPIErrors).toHaveBeenCalledWith('error');
});
});
describe('submitDeduplicationUpdate', () => {
it('updates logic with data that has been converted from server to client', async () => {
jest.spyOn(CrawlerSingleDomainLogic.actions, 'onReceiveDomainData');
http.put.mockReturnValueOnce(
Promise.resolve({
id: '507f1f77bcf86cd799439011',
name: 'https://elastic.co',
created_on: 'Mon, 31 Aug 2020 17:00:00 +0000',
document_count: 13,
sitemaps: [],
entry_points: [],
crawl_rules: [],
deduplication_enabled: true,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
})
);
CrawlerSingleDomainLogic.actions.submitDeduplicationUpdate(
{ id: '507f1f77bcf86cd799439011' } as CrawlerDomain,
{ fields: ['title'], enabled: true }
);
await nextTick();
expect(http.put).toHaveBeenCalledWith(
'/api/app_search/engines/some-engine/crawler/domains/507f1f77bcf86cd799439011',
{
body: JSON.stringify({ deduplication_enabled: true, deduplication_fields: ['title'] }),
}
);
expect(CrawlerSingleDomainLogic.actions.onReceiveDomainData).toHaveBeenCalledWith({
id: '507f1f77bcf86cd799439011',
createdOn: 'Mon, 31 Aug 2020 17:00:00 +0000',
url: 'https://elastic.co',
documentCount: 13,
sitemaps: [],
entryPoints: [],
crawlRules: [],
deduplicationEnabled: true,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
});
});
it('displays any errors to the user', async () => {
http.put.mockReturnValueOnce(Promise.reject('error'));
CrawlerSingleDomainLogic.actions.submitDeduplicationUpdate(
{ id: '507f1f77bcf86cd799439011' } as CrawlerDomain,
{ fields: ['title'], enabled: true }
);
await nextTick();
expect(flashAPIErrors).toHaveBeenCalledWith('error');
});
});
});
});

29
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/crawler_single_domain_logic.ts
View file

@ -29,6 +29,10 @@ interface CrawlerSingleDomainActions {
updateCrawlRules(crawlRules: CrawlRule[]): { crawlRules: CrawlRule[] };
updateEntryPoints(entryPoints: EntryPoint[]): { entryPoints: EntryPoint[] };
updateSitemaps(entryPoints: Sitemap[]): { sitemaps: Sitemap[] };
submitDeduplicationUpdate(
domain: CrawlerDomain,
payload: { fields?: string[]; enabled?: boolean }
): { domain: CrawlerDomain; fields: string[]; enabled: boolean };
}
export const CrawlerSingleDomainLogic = kea<
@ -42,6 +46,7 @@ export const CrawlerSingleDomainLogic = kea<
updateCrawlRules: (crawlRules) => ({ crawlRules }),
updateEntryPoints: (entryPoints) => ({ entryPoints }),
updateSitemaps: (sitemaps) => ({ sitemaps }),
submitDeduplicationUpdate: (domain, { fields, enabled }) => ({ domain, fields, enabled }),
},
reducers: {
dataLoading: [
@ -88,6 +93,30 @@ export const CrawlerSingleDomainLogic = kea<
const domainData = crawlerDomainServerToClient(response);
actions.onReceiveDomainData(domainData);
} catch (e) {
flashAPIErrors(e);
}
},
submitDeduplicationUpdate: async ({ domain, fields, enabled }) => {
const { http } = HttpLogic.values;
const { engineName } = EngineLogic.values;
const payload = {
deduplication_enabled: enabled,
deduplication_fields: fields,
};
try {
const response = await http.put(
`/api/app_search/engines/${engineName}/crawler/domains/${domain.id}`,
{
body: JSON.stringify(payload),
}
);
const domainData = crawlerDomainServerToClient(response);
actions.onReceiveDomainData(domainData);
} catch (e) {
flashAPIErrors(e);

6
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/types.ts
View file

@ -98,6 +98,9 @@ export interface CrawlerDomain {
defaultCrawlRule?: CrawlRule;
entryPoints: EntryPoint[];
sitemaps: Sitemap[];
deduplicationEnabled: boolean;
deduplicationFields: string[];
availableDeduplicationFields: string[];
}
export interface CrawlerDomainFromServer {
@ -110,6 +113,9 @@ export interface CrawlerDomainFromServer {
default_crawl_rule?: CrawlRule;
entry_points: EntryPoint[];
sitemaps: Sitemap[];
deduplication_enabled: boolean;
deduplication_fields: string[];
available_deduplication_fields: string[];
}
export interface CrawlerData {

23
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.test.ts
View file

@ -16,6 +16,7 @@ import {
CrawlerStatus,
CrawlerData,
CrawlRequest,
CrawlerDomain,
} from './types';
import {
@ -39,7 +40,7 @@ describe('crawlerDomainServerToClient', () => {
const id = '507f1f77bcf86cd799439011';
const name = 'moviedatabase.com';
const defaultServerPayload = {
const defaultServerPayload: CrawlerDomainFromServer = {
id,
name,
created_on: 'Mon, 31 Aug 2020 17:00:00 +0000',
@ -47,9 +48,12 @@ describe('crawlerDomainServerToClient', () => {
sitemaps: [],
entry_points: [],
crawl_rules: [],
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
};
const defaultClientPayload = {
const defaultClientPayload: CrawlerDomain = {
id,
createdOn: 'Mon, 31 Aug 2020 17:00:00 +0000',
url: name,
@ -57,6 +61,9 @@ describe('crawlerDomainServerToClient', () => {
sitemaps: [],
entryPoints: [],
crawlRules: [],
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
};
expect(crawlerDomainServerToClient(defaultServerPayload)).toStrictEqual(defaultClientPayload);
@ -124,6 +131,9 @@ describe('crawlerDataServerToClient', () => {
entry_points: [],
crawl_rules: [],
default_crawl_rule: DEFAULT_CRAWL_RULE,
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
},
{
id: 'y',
@ -134,6 +144,9 @@ describe('crawlerDataServerToClient', () => {
sitemaps: [],
entry_points: [],
crawl_rules: [],
deduplication_enabled: false,
deduplication_fields: ['title'],
available_deduplication_fields: ['title', 'description'],
},
];
@ -154,6 +167,9 @@ describe('crawlerDataServerToClient', () => {
entryPoints: [],
crawlRules: [],
defaultCrawlRule: DEFAULT_CRAWL_RULE,
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
{
id: 'y',
@ -164,6 +180,9 @@ describe('crawlerDataServerToClient', () => {
sitemaps: [],
entryPoints: [],
crawlRules: [],
deduplicationEnabled: false,
deduplicationFields: ['title'],
availableDeduplicationFields: ['title', 'description'],
},
]);
});

6
x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/utils.ts
View file

@ -29,6 +29,9 @@ export function crawlerDomainServerToClient(payload: CrawlerDomainFromServer): C
crawl_rules: crawlRules,
default_crawl_rule: defaultCrawlRule,
entry_points: entryPoints,
deduplication_enabled: deduplicationEnabled,
deduplication_fields: deduplicationFields,
available_deduplication_fields: availableDeduplicationFields,
} = payload;
const clientPayload: CrawlerDomain = {
@ -39,6 +42,9 @@ export function crawlerDomainServerToClient(payload: CrawlerDomainFromServer): C
crawlRules,
sitemaps,
entryPoints,
deduplicationEnabled,
deduplicationFields,
availableDeduplicationFields,
};
if (lastCrawl) {

4
x-pack/plugins/enterprise_search/public/applications/test_helpers/enzyme_rerender.ts
View file

@ -5,13 +5,13 @@
* 2.0.
*/
import { ShallowWrapper } from 'enzyme';
import { CommonWrapper } from 'enzyme';
/**
* Quick and easy helper for re-rendering a React component in Enzyme
* after (e.g.) updating Kea values
*/
export const rerender = (wrapper: ShallowWrapper) => {
export const rerender = (wrapper: CommonWrapper) => {
wrapper.setProps({}); // Re-renders
wrapper.update(); // Just in case
};

23
x-pack/plugins/enterprise_search/server/routes/app_search/crawler.test.ts
View file

@ -266,7 +266,7 @@ describe('crawler routes', () => {
});
});
it('validates correctly with required params', () => {
it('validates correctly with crawl rules', () => {
const request = {
params: { name: 'some-engine', id: '1234' },
body: {
@ -281,9 +281,24 @@ describe('crawler routes', () => {
mockRouter.shouldValidate(request);
});
it('fails otherwise', () => {
const request = { params: {}, body: {} };
mockRouter.shouldThrow(request);
it('validates correctly with deduplication enabled', () => {
const request = {
params: { name: 'some-engine', id: '1234' },
body: {
deduplication_enabled: true,
},
};
mockRouter.shouldValidate(request);
});
it('validates correctly with deduplication fields', () => {
const request = {
params: { name: 'some-engine', id: '1234' },
body: {
deduplication_fields: ['title', 'description'],
},
};
mockRouter.shouldValidate(request);
});
});

14
x-pack/plugins/enterprise_search/server/routes/app_search/crawler.ts
View file

@ -136,12 +136,16 @@ export function registerCrawlerRoutes({
id: schema.string(),
}),
body: schema.object({
crawl_rules: schema.arrayOf(
schema.object({
order: schema.number(),
id: schema.string(),
})
crawl_rules: schema.maybe(
schema.arrayOf(
schema.object({
order: schema.number(),
id: schema.string(),
})
)
),
deduplication_enabled: schema.maybe(schema.boolean()),
deduplication_fields: schema.maybe(schema.arrayOf(schema.string())),
}),
},
},