github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Enterprise Search] Add connector crawler check (#139298)

Browse source 
* Differentiate crawler connectors from regular connectors
* Return crawler type when fetching connector-crawler index
This commit is contained in:
Nav 2022-08-26 16:27:46 +01:00 • committed by GitHub
parent 6e3aa86de8
commit ea929a0eb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 236 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/enterprise_search/common/constants.ts
View file

@ -98,3 +98,5 @@ export const ENTERPRISE_SEARCH_ELASTICSEARCH_URL = '/app/enterprise_search/elast
export const WORKPLACE_SEARCH_URL = '/app/enterprise_search/workplace_search';
export const ENTERPRISE_SEARCH_DOCUMENTS_DEFAULT_DOC_COUNT = 25;
export const ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE = 'elastic-crawler';

10
x-pack/plugins/enterprise_search/common/types/indices.ts
View file

@ -36,13 +36,11 @@ export interface ElasticsearchIndex {
export interface ConnectorIndex extends ElasticsearchIndex {
connector: Connector;
}
export interface CrawlerIndex extends ElasticsearchIndex {
crawler: Crawler;
connector?: Connector;
}
export interface ConnectorIndex extends ElasticsearchIndex {
connector: Connector;
}
export interface ElasticsearchIndexWithPrivileges extends ElasticsearchIndex {
alias: boolean;
privileges: {
@ -51,8 +49,4 @@ export interface ElasticsearchIndexWithPrivileges extends ElasticsearchIndex {
};
}
export interface CrawlerIndex extends ElasticsearchIndex {
crawler: Crawler;
}
export type ElasticsearchIndexWithIngestion = ElasticsearchIndex | ConnectorIndex | CrawlerIndex;

37
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/search_indices.mock.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../../common/constants';
import { ConnectorStatus, SyncStatus } from '../../../../common/types/connectors';
import { ElasticsearchIndexWithIngestion } from '../../../../common/types/indices';
@ -68,4 +70,39 @@ export const indices: ElasticsearchIndexWithIngestion[] = [
store: { size_in_bytes: '8024' },
},
},
{
connector: {
api_key_id: null,
configuration: { foo: { label: 'bar', value: 'barbar' } },
id: '4',
index_name: 'connector-crawler',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: SyncStatus.COMPLETED,
last_synced: null,
name: 'connector-crawler',
scheduling: {
enabled: false,
interval: '',
},
service_type: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE,
status: ConnectorStatus.CONFIGURED,
sync_now: false,
},
count: 1,
crawler: {
id: '5',
index_name: 'crawler',
},
hidden: false,
name: 'connector-crawler',
total: {
docs: {
count: 1,
deleted: 0,
},
store: { size_in_bytes: '8024' },
},
},
];

47
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/__mocks__/view_index.mock.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../../common/constants';
import { SyncStatus, ConnectorStatus } from '../../../../common/types/connectors';
import {
@ -83,5 +85,48 @@ export const crawlerIndex: CrawlerViewIndex = {
store: { size_in_bytes: '8024' },
},
};
export const connectorCrawlerIndex: CrawlerViewIndex = {
connector: {
api_key_id: null,
configuration: { foo: { label: 'bar', value: 'barbar' } },
id: '4',
index_name: 'connector-crawler',
language: 'en',
last_seen: null,
last_sync_error: null,
last_sync_status: SyncStatus.COMPLETED,
last_synced: null,
name: 'connector-crawler',
scheduling: {
enabled: false,
interval: '',
},
service_type: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE,
status: ConnectorStatus.CONFIGURED,
sync_now: false,
},
count: 1,
crawler: {
id: '5',
index_name: 'crawler',
},
hidden: false,
ingestionMethod: IngestionMethod.CRAWLER,
ingestionStatus: IngestionStatus.INCOMPLETE,
lastUpdated: null,
name: 'connector-crawler',
total: {
docs: {
count: 1,
deleted: 0,
},
store: { size_in_bytes: '8024' },
},
};
export const elasticsearchViewIndices = [apiIndex, connectorIndex, crawlerIndex];
export const elasticsearchViewIndices = [
apiIndex,
connectorIndex,
crawlerIndex,
connectorCrawlerIndex,
];

100
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.test.ts
View file

@ -5,7 +5,12 @@
* 2.0.
*/
import { connectorIndex, crawlerIndex, apiIndex } from '../__mocks__/view_index.mock';
import {
connectorIndex,
crawlerIndex,
connectorCrawlerIndex,
apiIndex,
} from '../__mocks__/view_index.mock';
import moment from 'moment';
@ -17,6 +22,12 @@ import {
getIngestionStatus,
getLastUpdated,
indexToViewIndex,
isConnectorIndex,
isCrawlerIndex,
isApiIndex,
isConnectorViewIndex,
isCrawlerViewIndex,
isApiViewIndex,
} from './indices';
describe('Indices util functions', () => {
@ -27,6 +38,9 @@ describe('Indices util functions', () => {
it('should return correct ingestion method for crawler', () => {
expect(getIngestionMethod(crawlerIndex)).toEqual(IngestionMethod.CRAWLER);
});
it('should return correct ingestion method for connector-crawler', () => {
expect(getIngestionMethod(connectorCrawlerIndex)).toEqual(IngestionMethod.CRAWLER);
});
it('should return correct ingestion method for API', () => {
expect(getIngestionMethod(apiIndex)).toEqual(IngestionMethod.API);
});
@ -116,4 +130,88 @@ describe('Indices util functions', () => {
});
});
});
describe('isConnectorIndex', () => {
it('should return true for connector indices', () => {
expect(isConnectorIndex(connectorIndex)).toEqual(true);
});
it('should return false for connector-crawler indices', () => {
expect(isConnectorIndex(connectorCrawlerIndex)).toEqual(false);
});
it('should return false for crawler indices', () => {
expect(isConnectorIndex(crawlerIndex)).toEqual(false);
});
it('should return false for API indices', () => {
expect(isConnectorIndex(apiIndex)).toEqual(false);
});
});
describe('isCrawlerIndex', () => {
it('should return true for crawler indices', () => {
expect(isCrawlerIndex(crawlerIndex)).toEqual(true);
});
it('should return true for connector-crawler indices', () => {
expect(isCrawlerIndex(connectorCrawlerIndex)).toEqual(true);
});
it('should return false for connector and API indices', () => {
expect(isCrawlerIndex(connectorIndex)).toEqual(false);
});
it('should return false for API indices', () => {
expect(isCrawlerIndex(apiIndex)).toEqual(false);
});
});
describe('isApiIndex', () => {
it('should return true for API indices', () => {
expect(isApiIndex(apiIndex)).toEqual(true);
});
it('should return false for crawler indices', () => {
expect(isApiIndex(crawlerIndex)).toEqual(false);
});
it('should return false for connector-crawler indices', () => {
expect(isApiIndex(connectorCrawlerIndex)).toEqual(false);
});
it('should return false for connector and API indices', () => {
expect(isApiIndex(connectorIndex)).toEqual(false);
});
});
describe('isConnectorViewIndex', () => {
it('should return true for connector indices', () => {
expect(isConnectorViewIndex(connectorIndex)).toEqual(true);
});
it('should return false for connector-crawler indices', () => {
expect(isConnectorViewIndex(connectorCrawlerIndex)).toEqual(false);
});
it('should return false for crawler indices', () => {
expect(isConnectorViewIndex(crawlerIndex)).toEqual(false);
});
it('should return false for API indices', () => {
expect(isConnectorViewIndex(apiIndex)).toEqual(false);
});
});
describe('isCrawlerViewIndex', () => {
it('should return true for crawler indices', () => {
expect(isCrawlerViewIndex(crawlerIndex)).toEqual(true);
});
it('should return true for connector-crawler indices', () => {
expect(isCrawlerViewIndex(connectorCrawlerIndex)).toEqual(true);
});
it('should return false for connector and API indices', () => {
expect(isCrawlerViewIndex(connectorIndex)).toEqual(false);
});
it('should return false for API indices', () => {
expect(isCrawlerViewIndex(apiIndex)).toEqual(false);
});
});
describe('isApiViewIndex', () => {
it('should return true for API indices', () => {
expect(isApiViewIndex(apiIndex)).toEqual(true);
});
it('should return false for crawler indices', () => {
expect(isApiViewIndex(crawlerIndex)).toEqual(false);
});
it('should return false for connector-crawler indices', () => {
expect(isApiViewIndex(connectorCrawlerIndex)).toEqual(false);
});
it('should return false for connector and API indices', () => {
expect(isApiViewIndex(connectorIndex)).toEqual(false);
});
});
});

13
x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/indices.ts
View file

@ -9,6 +9,7 @@ import moment from 'moment';
import { i18n } from '@kbn/i18n';
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../../common/constants';
import { SyncStatus, ConnectorStatus } from '../../../../common/types/connectors';
import {
ConnectorIndex,
@ -28,7 +29,11 @@ import {
export function isConnectorIndex(
index: ElasticsearchIndexWithIngestion | undefined
): index is ConnectorIndex {
return !!(index as ConnectorIndex)?.connector;
const connectorIndex = index as ConnectorIndex;
return (
!!connectorIndex?.connector &&
connectorIndex.connector.service_type !== ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE
);
}
export function isCrawlerIndex(
@ -45,7 +50,11 @@ export function isApiIndex(index: ElasticsearchIndexWithIngestion | undefined):
}
export function isConnectorViewIndex(index: ElasticsearchViewIndex): index is ConnectorViewIndex {
return !!(index as ConnectorViewIndex)?.connector;
const connectorViewIndex = index as ConnectorViewIndex;
return (
!!connectorViewIndex?.connector &&
connectorViewIndex.connector.service_type !== ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE
);
}
export function isCrawlerViewIndex(index: ElasticsearchViewIndex): index is CrawlerViewIndex {

38
x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.test.ts
View file

@ -8,6 +8,8 @@
import { ByteSizeValue } from '@kbn/config-schema';
import { IScopedClusterClient } from '@kbn/core/server';
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../common/constants';
import { fetchConnectorByIndexName } from '../connectors/fetch_connectors';
import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers';
@ -103,13 +105,16 @@ describe('fetchIndex lib function', () => {
})
);
(fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({ doc: 'doc' })
Promise.resolve({
doc: 'doc',
service_type: 'some-service-type',
})
);
mockClient.asCurrentUser.indices.stats.mockImplementation(() => Promise.resolve(statsResponse));
await expect(
fetchIndex(mockClient as unknown as IScopedClusterClient, 'index_name')
).resolves.toEqual({ ...result, connector: { doc: 'doc' } });
).resolves.toEqual({ ...result, connector: { doc: 'doc', service_type: 'some-service-type' } });
});
it('should return data and stats for index and crawler if crawler is present', async () => {
@ -137,6 +142,35 @@ describe('fetchIndex lib function', () => {
},
});
});
it('should return data and stats for index and crawler if a crawler registered as a connector is present', async () => {
mockClient.asCurrentUser.indices.get.mockImplementation(() =>
Promise.resolve({
index_name: { aliases: [], data: 'full index' },
})
);
(fetchCrawlerByIndexName as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
id: '1234',
})
);
(fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
doc: 'doc',
service_type: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE,
})
);
mockClient.asCurrentUser.indices.stats.mockImplementation(() => Promise.resolve(statsResponse));
await expect(
fetchIndex(mockClient as unknown as IScopedClusterClient, 'index_name')
).resolves.toEqual({
...result,
connector: { doc: 'doc', service_type: ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE },
crawler: { id: '1234' },
});
});
it('should throw a 404 error if the index cannot be fonud', async () => {
mockClient.asCurrentUser.indices.get.mockImplementation(() => Promise.resolve({}));
(fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() =>

5
x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.ts
View file

@ -7,6 +7,7 @@
import { IScopedClusterClient } from '@kbn/core/server';
import { ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE } from '../../../common/constants';
import { ElasticsearchIndexWithIngestion } from '../../../common/types/indices';
import { fetchConnectorByIndexName } from '../connectors/fetch_connectors';
import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers';
@ -33,7 +34,7 @@ export const fetchIndex = async (
};
const connector = await fetchConnectorByIndexName(client, index);
if (connector) {
if (connector && connector.service_type !== ENTERPRISE_SEARCH_CONNECTOR_CRAWLER_SERVICE_TYPE) {
return {
...indexResult,
connector,
@ -42,7 +43,7 @@ export const fetchIndex = async (
const crawler = await fetchCrawlerByIndexName(client, index);
if (crawler) {
return { ...indexResult, crawler };
return { ...indexResult, connector, crawler };
}
return indexResult;