github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Enterprise Search] Add a recent jobs table to connector overview (#140111)

Browse source 
* [Enterprise Search] Add sync jobs table to connectors overview

* Fix dangling connector_package reference

* Fix dangling connector_package reference
This commit is contained in:
Sander Philipse 2022-09-06 21:29:18 +02:00 • committed by GitHub
parent a3061a8d67
commit 2f2b4b8fa6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 868 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

15
x-pack/plugins/enterprise_search/common/types/connectors.ts
View file

@ -39,7 +39,7 @@ export interface Connector {
language: string | null;
last_seen: string | null;
last_sync_error: string | null;
last_sync_status: string | null;
last_sync_status: SyncStatus | null;
last_synced: string | null;
name: string;
scheduling: {
@ -52,3 +52,16 @@ export interface Connector {
}
export type ConnectorDocument = Omit<Connector, 'id'>;
export interface ConnectorSyncJob {
completed_at: string | null;
connector?: ConnectorDocument;
connector_id: string;
created_at: string;
deleted_document_count: number;
error: string | null;
index_name: string;
indexed_document_count: number;
status: SyncStatus;
worker_hostname: string;
}

15
x-pack/plugins/enterprise_search/common/types/pagination.ts Normal file
View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface Paginate<T> {
data: T[];
has_more_hits_than_total: boolean;
pageIndex: number;
pageSize: number;
size: number;
total: number;
}

43
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/fetch_sync_jobs_api_logic.test.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { mockHttpValues } from '../../../__mocks__/kea_logic';
import { nextTick } from '@kbn/test-jest-helpers';
import { fetchSyncJobs } from './fetch_sync_jobs_api_logic';
describe('FetchSyncJobs', () => {
const { http } = mockHttpValues;
beforeEach(() => {
jest.clearAllMocks();
});
describe('fetchSyncJobs', () => {
it('calls correct api', async () => {
const promise = Promise.resolve('result');
http.get.mockReturnValue(promise);
const result = fetchSyncJobs({ connectorId: 'connectorId1' });
await nextTick();
expect(http.get).toHaveBeenCalledWith(
'/internal/enterprise_search/connectors/connectorId1/sync_jobs',
{ query: { page: 0, size: 10 } }
);
await expect(result).resolves.toEqual('result');
});
it('appends query if specified', async () => {
const promise = Promise.resolve('result');
http.get.mockReturnValue(promise);
const result = fetchSyncJobs({ connectorId: 'connectorId1', page: 10, size: 20 });
await nextTick();
expect(http.get).toHaveBeenCalledWith(
'/internal/enterprise_search/connectors/connectorId1/sync_jobs',
{ query: { page: 10, size: 20 } }
);
await expect(result).resolves.toEqual('result');
});
});
});

31
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/fetch_sync_jobs_api_logic.ts Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ConnectorSyncJob } from '../../../../../common/types/connectors';
import { Paginate } from '../../../../../common/types/pagination';
import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
import { HttpLogic } from '../../../shared/http';
export interface FetchSyncJobsArgs {
connectorId: string;
page?: number;
size?: number;
}
export type FetchSyncJobsResponse = Paginate<ConnectorSyncJob>;
export const fetchSyncJobs = async ({ connectorId, page = 0, size = 10 }: FetchSyncJobsArgs) => {
const route = `/internal/enterprise_search/connectors/${connectorId}/sync_jobs`;
const query = { page, size };
return await HttpLogic.values.http.get<Paginate<ConnectorSyncJob>>(route, { query });
};
export const FetchSyncJobsApiLogic = createApiLogic(
['enterprise_search_content', 'fetch_sync_api_logic'],
fetchSyncJobs
);

2
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.test.ts
View file

@ -26,6 +26,7 @@ import { IndexViewLogic } from './index_view_logic';
// We can't test fetchTimeOutId because this will get set whenever the logic is created
// And the timeoutId is non-deterministic. We use expect.object.containing throughout this test file
const DEFAULT_VALUES = {
connectorId: null,
data: undefined,
index: undefined,
indexName: '',
@ -41,6 +42,7 @@ const DEFAULT_VALUES = {
const CONNECTOR_VALUES = {
...DEFAULT_VALUES,
connectorId: connectorIndex.connector.id,
data: connectorIndex,
index: indexToViewIndex(connectorIndex),
ingestionMethod: IngestionMethod.CONNECTOR,

8
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts
View file

@ -29,6 +29,7 @@ import {
getLastUpdated,
indexToViewIndex,
isConnectorIndex,
isConnectorViewIndex,
isCrawlerIndex,
} from '../../utils/indices';
@ -61,6 +62,7 @@ export interface IndexViewActions {
}
export interface IndexViewValues {
connectorId: string | null;
data: typeof FetchIndexApiLogic.values.data;
fetchIndexTimeoutId: NodeJS.Timeout | null;
index: ElasticsearchViewIndex | undefined;
@ -170,7 +172,7 @@ export const IndexViewLogic = kea<MakeLogicType<IndexViewValues, IndexViewAction
},
startSync: () => {
if (isConnectorIndex(values.data)) {
actions.makeStartSyncRequest({ connectorId: values.data?.connector?.id });
actions.makeStartSyncRequest({ connectorId: values.data.connector.id });
}
},
startSyncApiError: (e) => flashAPIErrors(e),
@ -215,6 +217,10 @@ export const IndexViewLogic = kea<MakeLogicType<IndexViewValues, IndexViewAction
],
},
selectors: ({ selectors }) => ({
connectorId: [
() => [selectors.index],
(index) => (isConnectorViewIndex(index) ? index.connector.id : null),
],
index: [() => [selectors.data], (data) => (data ? indexToViewIndex(data) : undefined)],
ingestionMethod: [() => [selectors.data], (data) => getIngestionMethod(data)],
ingestionStatus: [() => [selectors.data], (data) => getIngestionStatus(data)],

3
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/overview.tsx
View file

@ -21,6 +21,7 @@ import { CrawlRequestsPanel } from './crawler/crawl_requests_panel/crawl_request
import { CrawlerTotalStats } from './crawler_total_stats';
import { GenerateApiKeyPanel } from './generate_api_key_panel';
import { OverviewLogic } from './overview.logic';
import { SyncJobs } from './sync_jobs';
import { TotalStats } from './total_stats';
export const SearchIndexOverview: React.FC = () => {
@ -67,6 +68,8 @@ export const SearchIndexOverview: React.FC = () => {
<>
<EuiSpacer />
<ConnectorOverviewPanels />
<EuiSpacer />
<SyncJobs />
</>
)}
</>

101
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs.tsx Normal file
View file

@ -0,0 +1,101 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useEffect } from 'react';
import { useActions, useValues } from 'kea';
import { EuiBadge, EuiBasicTable, EuiBasicTableColumn } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { SyncStatus } from '../../../../../common/types/connectors';
import { FormattedDateTime } from '../../../shared/formatted_date_time';
import { durationToText } from '../../utils/duration_to_text';
import { syncStatusToColor, syncStatusToText } from '../../utils/sync_status_to_text';
import { IndexViewLogic } from './index_view_logic';
import { SyncJobsViewLogic, SyncJobView } from './sync_jobs_view_logic';
export const SyncJobs: React.FC = () => {
const { connectorId } = useValues(IndexViewLogic);
const { syncJobs, syncJobsLoading, syncJobsPagination } = useValues(SyncJobsViewLogic);
const { fetchSyncJobs } = useActions(SyncJobsViewLogic);
useEffect(() => {
if (connectorId) {
fetchSyncJobs({
connectorId,
page: syncJobsPagination.pageIndex ?? 0,
size: syncJobsPagination.pageSize ?? 10,
});
}
}, [connectorId]);
const columns: Array<EuiBasicTableColumn<SyncJobView>> = [
{
field: 'lastSync',
name: i18n.translate('xpack.enterpriseSearch.content.syncJobs.lastSync.columnTitle', {
defaultMessage: 'Last sync',
}),
render: (lastSync: string) => <FormattedDateTime date={new Date(lastSync)} />,
sortable: true,
truncateText: true,
width: '25%',
},
{
field: 'duration',
name: i18n.translate('xpack.enterpriseSearch.content.syncJobs.syncDuration.columnTitle', {
defaultMessage: 'Sync duration',
}),
render: (duration: moment.Duration) => durationToText(duration),
sortable: true,
truncateText: true,
width: '25%',
},
{
field: 'docsCount',
name: i18n.translate('xpack.enterpriseSearch.content.searchIndices.docsCount.columnTitle', {
defaultMessage: 'Docs count',
}),
sortable: true,
truncateText: true,
width: '25%',
},
{
field: 'status',
name: i18n.translate('xpack.enterpriseSearch.content.searchIndices.syncStatus.columnTitle', {
defaultMessage: 'Status',
}),
render: (syncStatus: SyncStatus) => (
<EuiBadge color={syncStatusToColor(syncStatus)}>{syncStatusToText(syncStatus)}</EuiBadge>
),
truncateText: true,
width: '25%',
},
];
return (
<EuiBasicTable
items={syncJobs}
columns={columns}
onChange={({ page: { index } }: { page: { index: number } }) => {
if (connectorId) {
fetchSyncJobs({ connectorId, page: index, size: syncJobsPagination.pageSize });
}
}}
pagination={{
...syncJobsPagination,
totalItemCount: syncJobsPagination.total,
}}
tableLayout="fixed"
loading={syncJobsLoading}
/>
);
};

181
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs_view_logic.test.ts Normal file
View file

@ -0,0 +1,181 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { LogicMounter, mockFlashMessageHelpers } from '../../../__mocks__/kea_logic';
import moment from 'moment';
import { nextTick } from '@kbn/test-jest-helpers';
import { HttpError, Status } from '../../../../../common/types/api';
import { SyncStatus } from '../../../../../common/types/connectors';
import { FetchSyncJobsApiLogic } from '../../api/connector/fetch_sync_jobs_api_logic';
import { IndexViewLogic } from './index_view_logic';
import { SyncJobView, SyncJobsViewLogic } from './sync_jobs_view_logic';
// We can't test fetchTimeOutId because this will get set whenever the logic is created
// And the timeoutId is non-deterministic. We use expect.object.containing throughout this test file
const DEFAULT_VALUES = {
connectorId: null,
syncJobs: [],
syncJobsData: undefined,
syncJobsLoading: true,
syncJobsPagination: {
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 10,
size: 0,
total: 0,
},
syncJobsStatus: Status.IDLE,
};
describe('SyncJobsViewLogic', () => {
const { mount: indexViewLogicMount } = new LogicMounter(IndexViewLogic);
const { mount: fetchSyncJobsMount } = new LogicMounter(FetchSyncJobsApiLogic);
const { mount } = new LogicMounter(SyncJobsViewLogic);
beforeEach(() => {
indexViewLogicMount();
fetchSyncJobsMount();
mount();
});
it('has expected default values', () => {
expect(SyncJobsViewLogic.values).toEqual(DEFAULT_VALUES);
});
describe('actions', () => {
describe('FetchIndexApiLogic.apiSuccess', () => {
const syncJob = {
completed_at: '2022-09-05T15:59:39.816+00:00',
connector_id: 'we2284IBjobuR2-lAuXh',
created_at: '2022-09-05T14:59:39.816+00:00',
deleted_document_count: 20,
error: null,
index_name: 'indexName',
indexed_document_count: 50,
status: SyncStatus.COMPLETED,
worker_hostname: 'hostname_fake',
};
const syncJobView: SyncJobView = {
docsCount: 30,
duration: moment.duration(1, 'hour'),
lastSync: syncJob.completed_at,
status: SyncStatus.COMPLETED,
};
it('should update values', async () => {
FetchSyncJobsApiLogic.actions.apiSuccess({
data: [syncJob],
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
});
await nextTick();
expect(SyncJobsViewLogic.values).toEqual({
...DEFAULT_VALUES,
syncJobs: [syncJobView],
syncJobsData: {
data: [syncJob],
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
},
syncJobsLoading: false,
syncJobsPagination: {
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
},
syncJobsStatus: Status.SUCCESS,
});
});
it('should update values for incomplete job', async () => {
FetchSyncJobsApiLogic.actions.apiSuccess({
data: [
{
...syncJob,
completed_at: null,
deleted_document_count: 0,
status: SyncStatus.IN_PROGRESS,
},
],
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
});
await nextTick();
expect(SyncJobsViewLogic.values).toEqual({
...DEFAULT_VALUES,
syncJobs: [
{
docsCount: 50,
duration: undefined,
lastSync: syncJob.created_at,
status: SyncStatus.IN_PROGRESS,
},
],
syncJobsData: {
data: [
{
...syncJob,
completed_at: null,
deleted_document_count: 0,
status: SyncStatus.IN_PROGRESS,
},
],
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
},
syncJobsLoading: false,
syncJobsPagination: {
has_more_hits_than_total: false,
pageIndex: 3,
pageSize: 20,
size: 20,
total: 50,
},
syncJobsStatus: Status.SUCCESS,
});
});
});
});
describe('listeners', () => {
it('calls clearFlashMessages on fetchSyncJobs', async () => {
SyncJobsViewLogic.actions.fetchSyncJobs({ connectorId: 'connectorId' });
await nextTick();
expect(mockFlashMessageHelpers.clearFlashMessages).toHaveBeenCalledTimes(1);
});
it('calls flashAPIErrors on apiError', async () => {
SyncJobsViewLogic.actions.fetchSyncJobsError({} as HttpError);
await nextTick();
expect(mockFlashMessageHelpers.flashAPIErrors).toHaveBeenCalledTimes(1);
expect(mockFlashMessageHelpers.flashAPIErrors).toHaveBeenCalledWith({});
expect(SyncJobsViewLogic.values).toEqual({
...DEFAULT_VALUES,
syncJobsLoading: false,
syncJobsStatus: Status.ERROR,
});
});
});
});

107
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/sync_jobs_view_logic.ts Normal file
View file

@ -0,0 +1,107 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { kea, MakeLogicType } from 'kea';
import moment from 'moment';
import { Status } from '../../../../../common/types/api';
import { ConnectorSyncJob, SyncStatus } from '../../../../../common/types/connectors';
import { Paginate } from '../../../../../common/types/pagination';
import { Actions } from '../../../shared/api_logic/create_api_logic';
import { clearFlashMessages, flashAPIErrors } from '../../../shared/flash_messages';
import {
FetchSyncJobsApiLogic,
FetchSyncJobsArgs,
FetchSyncJobsResponse,
} from '../../api/connector/fetch_sync_jobs_api_logic';
import { IndexViewLogic } from './index_view_logic';
export interface SyncJobView {
docsCount: number;
duration: moment.Duration;
lastSync: string;
status: SyncStatus;
}
export interface IndexViewActions {
fetchSyncJobs: Actions<FetchSyncJobsArgs, FetchSyncJobsResponse>['makeRequest'];
fetchSyncJobsError: Actions<FetchSyncJobsArgs, FetchSyncJobsResponse>['apiError'];
}
export interface IndexViewValues {
connectorId: string | null;
syncJobs: SyncJobView[];
syncJobsData: Paginate<ConnectorSyncJob> | null;
syncJobsLoading: boolean;
syncJobsPagination: Paginate<undefined>;
syncJobsStatus: Status;
}
export const SyncJobsViewLogic = kea<MakeLogicType<IndexViewValues, IndexViewActions>>({
actions: {},
connect: {
actions: [
FetchSyncJobsApiLogic,
[
'apiError as fetchSyncJobsError',
'apiReset as resetFetchSyncJobsIndexApi',
'apiSuccess as fetchSyncJobsApiSuccess',
'makeRequest as fetchSyncJobs',
],
],
values: [
IndexViewLogic,
['connectorId'],
FetchSyncJobsApiLogic,
['data as syncJobsData', 'status as syncJobsStatus'],
],
},
listeners: () => ({
fetchSyncJobs: () => clearFlashMessages(),
fetchSyncJobsError: (e) => flashAPIErrors(e),
}),
path: ['enterprise_search', 'content', 'sync_jobs_view_logic'],
selectors: ({ selectors }) => ({
syncJobs: [
() => [selectors.syncJobsData],
(data?: Paginate<ConnectorSyncJob>) =>
data?.data.map((syncJob) => {
return {
docsCount: syncJob.deleted_document_count
? syncJob.indexed_document_count - syncJob.deleted_document_count
: syncJob.indexed_document_count,
duration: syncJob.completed_at
? moment.duration(moment(syncJob.completed_at).diff(moment(syncJob.created_at)))
: undefined,
lastSync: syncJob.completed_at ?? syncJob.created_at,
status: syncJob.status,
};
}) ?? [],
],
syncJobsLoading: [
() => [selectors.syncJobsStatus],
(status: Status) => status === Status.IDLE || status === Status.LOADING,
],
syncJobsPagination: [
() => [selectors.syncJobsData],
(data?: Paginate<ConnectorSyncJob>) =>
data
? { ...data, data: undefined }
: {
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 10,
size: 0,
total: 0,
},
],
}),
});

19
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/duration_to_text.test.ts Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
import { durationToText } from './duration_to_text';
describe('durationToText', () => {
it('should correctly turn duration into text', () => {
expect(durationToText(moment.duration(11005, 'seconds'))).toEqual('3h 3m 25s');
});
it('should return -- for undefined', () => {
expect(durationToText(undefined)).toEqual('--');
});
});

19
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/duration_to_text.ts Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
export function durationToText(input?: moment.Duration): string {
if (input) {
const hours = input.hours();
const minutes = input.minutes();
const seconds = input.seconds();
return `${hours}h ${minutes}m ${seconds}s`;
} else {
return '--';
}
}

34
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/sync_status_to_text.test.ts Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SyncStatus } from '../../../../common/types/connectors';
import { syncStatusToColor, syncStatusToText } from './sync_status_to_text';
describe('syncStatusToText', () => {
it('should return correct value for completed', () => {
expect(syncStatusToText(SyncStatus.COMPLETED)).toEqual('Sync complete');
});
it('should return correct value for error', () => {
expect(syncStatusToText(SyncStatus.ERROR)).toEqual('Sync failure');
});
it('should return correct value for in progress', () => {
expect(syncStatusToText(SyncStatus.IN_PROGRESS)).toEqual('Sync in progress');
});
});
describe('syncStatusToColor', () => {
it('should return correct value for completed', () => {
expect(syncStatusToColor(SyncStatus.COMPLETED)).toEqual('success');
});
it('should return correct value for error', () => {
expect(syncStatusToColor(SyncStatus.ERROR)).toEqual('danger');
});
it('should return correct value for in progress', () => {
expect(syncStatusToColor(SyncStatus.IN_PROGRESS)).toEqual('warning');
});
});

38
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/sync_status_to_text.ts Normal file
View file

@ -0,0 +1,38 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
import { SyncStatus } from '../../../../common/types/connectors';
export function syncStatusToText(status: SyncStatus): string {
switch (status) {
case SyncStatus.COMPLETED:
return i18n.translate('xpack.enterpriseSearch.content.syncStatus.completed', {
defaultMessage: 'Sync complete',
});
case SyncStatus.ERROR:
return i18n.translate('xpack.enterpriseSearch.content.syncStatus.error', {
defaultMessage: 'Sync failure',
});
case SyncStatus.IN_PROGRESS:
return i18n.translate('xpack.enterpriseSearch.content.syncStatus.inProgress', {
defaultMessage: 'Sync in progress',
});
}
}
export function syncStatusToColor(status: SyncStatus): string {
switch (status) {
case SyncStatus.COMPLETED:
return 'success';
case SyncStatus.ERROR:
return 'danger';
case SyncStatus.IN_PROGRESS:
return 'warning';
}
}

143
x-pack/plugins/enterprise_search/server/lib/connectors/fetch_sync_jobs.test.ts Normal file
View file

@ -0,0 +1,143 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { setupConnectorsIndices } from '../../index_management/setup_indices';
import { fetchSyncJobsByConnectorId } from './fetch_sync_jobs';
jest.mock('../../index_management/setup_indices', () => ({
setupConnectorsIndices: jest.fn(),
}));
describe('fetchSyncJobs lib', () => {
const mockClient = {
asCurrentUser: {
get: jest.fn(),
search: jest.fn(),
},
asInternalUser: {},
};
beforeEach(() => {
jest.clearAllMocks();
});
describe('fetch sync jobs by connector id', () => {
it('should fetch sync jobs by connector id', async () => {
mockClient.asCurrentUser.search.mockImplementationOnce(() =>
Promise.resolve({ hits: { hits: ['result1', 'result2'] }, total: 2 })
);
await expect(fetchSyncJobsByConnectorId(mockClient as any, 'id', 0, 10)).resolves.toEqual({
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 10,
size: 0,
total: 0,
});
expect(mockClient.asCurrentUser.search).toHaveBeenCalledWith({
from: 0,
index: '.elastic-connectors-sync-jobs',
query: {
term: {
connector_id: 'id',
},
},
size: 10,
sort: {
created_at: {
order: 'desc',
},
},
});
});
it('should return empty result if size is 0', async () => {
await expect(fetchSyncJobsByConnectorId(mockClient as any, 'id', 0, 0)).resolves.toEqual({
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 0,
size: 0,
total: 0,
});
expect(mockClient.asCurrentUser.search).not.toHaveBeenCalled();
});
it('should call setup connectors on index not found error', async () => {
mockClient.asCurrentUser.search.mockImplementationOnce(() =>
Promise.reject({
meta: {
body: {
error: {
type: 'index_not_found_exception',
},
},
},
})
);
await expect(fetchSyncJobsByConnectorId(mockClient as any, 'id', 0, 10)).resolves.toEqual({
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 10,
size: 0,
total: 0,
});
expect(mockClient.asCurrentUser.search).toHaveBeenCalledWith({
from: 0,
index: '.elastic-connectors-sync-jobs',
query: {
term: {
connector_id: 'id',
},
},
size: 10,
sort: {
created_at: {
order: 'desc',
},
},
});
expect(setupConnectorsIndices as jest.Mock).toHaveBeenCalledWith(mockClient.asCurrentUser);
});
it('should not call setup connectors on other errors', async () => {
mockClient.asCurrentUser.search.mockImplementationOnce(() =>
Promise.reject({
meta: {
body: {
error: {
type: 'other error',
},
},
},
})
);
await expect(fetchSyncJobsByConnectorId(mockClient as any, 'id', 0, 10)).resolves.toEqual({
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: 10,
size: 0,
total: 0,
});
expect(mockClient.asCurrentUser.search).toHaveBeenCalledWith({
from: 0,
index: '.elastic-connectors-sync-jobs',
query: {
term: {
connector_id: 'id',
},
},
size: 10,
sort: {
created_at: {
order: 'desc',
},
},
});
expect(setupConnectorsIndices as jest.Mock).not.toHaveBeenCalled();
});
});
});

85
x-pack/plugins/enterprise_search/server/lib/connectors/fetch_sync_jobs.ts Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types';
import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
import { CONNECTORS_JOBS_INDEX } from '../..';
import { ConnectorSyncJob } from '../../../common/types/connectors';
import { Paginate } from '../../../common/types/pagination';
import { isNotNullish } from '../../../common/utils/is_not_nullish';
import { setupConnectorsIndices } from '../../index_management/setup_indices';
import { isIndexNotFoundException } from '../../utils/identify_exceptions';
export const fetchSyncJobsByConnectorId = async (
client: IScopedClusterClient,
connectorId: string,
pageIndex: number,
size: number
): Promise<Paginate<ConnectorSyncJob>> => {
try {
if (size === 0) {
// prevent some divide by zero errors below
return {
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: size,
size: 0,
total: 0,
};
}
const result = await client.asCurrentUser.search<ConnectorSyncJob>({
from: pageIndex * size,
index: CONNECTORS_JOBS_INDEX,
query: {
term: {
connector_id: connectorId,
},
},
size,
// @ts-ignore Elasticsearch-js has the wrong internal typing for this field
sort: { created_at: { order: 'desc' } },
});
const total = totalToPaginateTotal(result.hits.total);
// If we get fewer results than the target page, make sure we return correct page we're on
const resultPageIndex = Math.min(pageIndex, Math.trunc(total.total / size));
const data = result.hits.hits.map((hit) => hit._source).filter(isNotNullish) ?? [];
return {
data,
pageIndex: resultPageIndex,
pageSize: size,
size: data.length,
...total,
};
} catch (error) {
if (isIndexNotFoundException(error)) {
await setupConnectorsIndices(client.asCurrentUser);
}
return {
data: [],
has_more_hits_than_total: false,
pageIndex: 0,
pageSize: size,
size: 0,
total: 0,
};
}
};
function totalToPaginateTotal(input: number | SearchTotalHits | undefined): {
has_more_hits_than_total: boolean;
total: number;
} {
if (typeof input === 'number') {
return { has_more_hits_than_total: false, total: input };
}
return input
? { has_more_hits_than_total: input.relation === 'gte' ? true : false, total: input.value }
: { has_more_hits_than_total: false, total: 0 };
}

26
x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts
View file

@ -10,6 +10,7 @@ import { i18n } from '@kbn/i18n';
import { ErrorCode } from '../../../common/types/error_codes';
import { addConnector } from '../../lib/connectors/add_connector';
import { fetchSyncJobsByConnectorId } from '../../lib/connectors/fetch_sync_jobs';
import { startConnectorSync } from '../../lib/connectors/start_sync';
import { updateConnectorConfiguration } from '../../lib/connectors/update_connector_configuration';
import { updateConnectorScheduling } from '../../lib/connectors/update_connector_scheduling';
@ -111,4 +112,29 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) {
return response.ok();
})
);
router.get(
{
path: '/internal/enterprise_search/connectors/{connectorId}/sync_jobs',
validate: {
params: schema.object({
connectorId: schema.string(),
}),
query: schema.object({
page: schema.number({ defaultValue: 0, min: 0 }),
size: schema.number({ defaultValue: 10, min: 0 }),
}),
},
},
elasticsearchErrorHandler(log, async (context, request, response) => {
const { client } = (await context.core).elasticsearch;
const result = await fetchSyncJobsByConnectorId(
client,
request.params.connectorId,
request.query.page,
request.query.size
);
return response.ok({ body: result });
})
);
}