[Serverless Search] Indexing API - Select index (#160454)

## Summary

Adds fetching indices and showing a combox to select and search for
indices to the indexing API page.

## Notes:
- Doc links are currently placeholders and will need to be set (at some
point?)

### Screenshots
<img width="1840" alt="image"
src="1b60f0a3-5098-43a0-a741-3f7ed18e5107">

---------

Co-authored-by: Sander Philipse <sander.philipse@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Rodney Norris 2023-06-29 11:06:44 -05:00 committed by GitHub
parent eab826842a
commit 0aa94eda79
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 463 additions and 8 deletions

View file

@ -11,3 +11,12 @@ export interface CreateAPIKeyArgs {
name: string;
role_descriptors?: Record<string, any>;
}
export interface IndexData {
name: string;
count: number;
}
export interface FetchIndicesResult {
indices: IndexData[];
}

View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export function isNotNullish<T>(value: T | null | undefined): value is T {
return value !== null && value !== undefined;
}

View file

@ -5,23 +5,33 @@
* 2.0.
*/
import React, { useState } from 'react';
import React, { useMemo, useState } from 'react';
import {
EuiCallOut,
EuiComboBox,
EuiComboBoxOptionOption,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiLink,
EuiPageTemplate,
EuiSpacer,
EuiStat,
EuiText,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { useQuery } from '@tanstack/react-query';
import { IndexData, FetchIndicesResult } from '../../../common/types';
import { FETCH_INDICES_PATH } from '../routes';
import { API_KEY_PLACEHOLDER, ELASTICSEARCH_URL_PLACEHOLDER } from '../constants';
import { useKibanaServices } from '../hooks/use_kibana';
import { CodeBox } from './code_box';
import { javascriptDefinition } from './languages/javascript';
import { languageDefinitions } from './languages/languages';
import { LanguageDefinition } from './languages/types';
import { LanguageDefinition, LanguageDefinitionSnippetArguments } from './languages/types';
import { OverviewPanel } from './overview_panels/overview_panel';
import { LanguageClientPanel } from './overview_panels/language_client_panel';
@ -48,9 +58,88 @@ const NoIndicesContent = () => (
</>
);
interface IndicesContentProps {
indices: IndexData[];
isLoading: boolean;
onChange: (selectedOptions: Array<EuiComboBoxOptionOption<IndexData>>) => void;
selectedIndex?: IndexData;
setSearchValue: (searchValue?: string) => void;
}
const IndicesContent = ({
indices,
isLoading,
onChange,
selectedIndex,
setSearchValue,
}: IndicesContentProps) => {
const toOption = (index: IndexData) => ({ label: index.name, value: index });
const options: Array<EuiComboBoxOptionOption<IndexData>> = indices.map(toOption);
return (
<>
<EuiSpacer />
<EuiFlexGroup>
<EuiFlexItem>
<EuiFormRow
fullWidth
label={i18n.translate(
'xpack.serverlessSearch.content.indexingApi.index.comboBox.title',
{ defaultMessage: 'Index' }
)}
>
<EuiComboBox
async
fullWidth
isLoading={isLoading}
singleSelection={{ asPlainText: true }}
onChange={onChange}
onSearchChange={setSearchValue}
options={options}
selectedOptions={selectedIndex ? [toOption(selectedIndex)] : undefined}
/>
</EuiFormRow>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiStat
// TODO: format count based on locale
title={selectedIndex ? selectedIndex.count.toLocaleString() : '--'}
titleColor="primary"
description={i18n.translate(
'xpack.serverlessSearch.content.indexingApi.index.documentCount.description',
{ defaultMessage: 'Documents' }
)}
/>
</EuiFlexItem>
</EuiFlexGroup>
</>
);
};
export const ElasticsearchIndexingApi = () => {
const { cloud, http } = useKibanaServices();
const [selectedLanguage, setSelectedLanguage] =
useState<LanguageDefinition>(javascriptDefinition);
const [indexSearchQuery, setIndexSearchQuery] = useState<string | undefined>(undefined);
const [selectedIndex, setSelectedIndex] = useState<IndexData | undefined>(undefined);
const elasticsearchURL = useMemo(() => {
return cloud?.elasticsearchUrl ?? ELASTICSEARCH_URL_PLACEHOLDER;
}, [cloud]);
const { data, isLoading, isError } = useQuery({
queryKey: ['indices', { searchQuery: indexSearchQuery }],
queryFn: async () => {
const query = {
search_query: indexSearchQuery || null,
};
const result = await http.get<FetchIndicesResult>(FETCH_INDICES_PATH, { query });
return result;
},
});
const codeSnippetArguments: LanguageDefinitionSnippetArguments = {
url: elasticsearchURL,
apiKey: API_KEY_PLACEHOLDER,
indexName: selectedIndex?.name,
};
const showNoIndices = !isLoading && data?.indices?.length === 0 && indexSearchQuery === undefined;
return (
<EuiPageTemplate offset={0} grow restrictWidth data-test-subj="svlSearchIndexingApiPage">
@ -67,6 +156,17 @@ export const ElasticsearchIndexingApi = () => {
)}
bottomBorder="extended"
/>
{isError && (
<EuiPageTemplate.Section>
<EuiCallOut
color="danger"
title={i18n.translate(
'xpack.serverlessSearch.content.indexingApi.fetchIndices.error.title',
{ defaultMessage: 'Error fetching indices' }
)}
/>
</EuiPageTemplate.Section>
)}
<EuiPageTemplate.Section color="subdued" bottomBorder="extended">
<OverviewPanel
title={i18n.translate('xpack.serverlessSearch.content.indexingApi.clientPanel.title', {
@ -106,16 +206,41 @@ export const ElasticsearchIndexingApi = () => {
</EuiFlexGroup>
<EuiSpacer />
<CodeBox
code="ingestData"
codeArgs={{ url: '', apiKey: '' }}
code="ingestDataIndex"
codeArgs={codeSnippetArguments}
languages={languageDefinitions}
selectedLanguage={selectedLanguage}
setSelectedLanguage={setSelectedLanguage}
/>
</>
}
links={
showNoIndices
? undefined
: [
{
label: i18n.translate(
'xpack.serverlessSearch.content.indexingApi.ingestDocsLink',
{ defaultMessage: 'Ingestion documentation' }
),
href: '#', // TODO: get doc links ?
},
]
}
>
<NoIndicesContent />
{showNoIndices ? (
<NoIndicesContent />
) : (
<IndicesContent
isLoading={isLoading}
indices={data?.indices ?? []}
onChange={(options) => {
setSelectedIndex(options?.[0]?.value);
}}
setSearchValue={setIndexSearchQuery}
selectedIndex={selectedIndex}
/>
)}
</OverviewPanel>
</EuiPageTemplate.Section>
</EuiPageTemplate>

View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { INDEX_NAME_PLACEHOLDER } from '../../constants';
import { LanguageDefinition } from './types';
export const consoleDefinition: Partial<LanguageDefinition> = {
@ -29,4 +30,8 @@ export const consoleDefinition: Partial<LanguageDefinition> = {
{"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268}
{ "index" : { "_index" : "books" } }
{"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}`,
ingestDataIndex: ({ indexName }) => `POST _bulk?pretty
{ "index" : { "_index" : "${indexName ?? INDEX_NAME_PLACEHOLDER}" } }
{"name": "foo", "title": "bar"}
`,
};

View file

@ -43,6 +43,13 @@ export API_KEY="${apiKey}"`,
{ "index" : { "_index" : "books" } }
{"name": "The Handmaid'"'"'s Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}
'`,
ingestDataIndex: ({ apiKey, url, indexName }) => `curl -X POST ${url}/_bulk?pretty \\
-H "Authorization: ApiKey ${apiKey}" \\
-H "Content-Type: application/json" \\
-d'
{ "index" : { "_index" : "${indexName ?? 'index_name'}" } }
{"name": "foo", "title": "bar" }
`,
installClient: `# if cURL is not already installed on your system
# then install it with the package manager of your choice

View file

@ -59,6 +59,30 @@ bytes: 293,
aborted: false
}
*/`,
ingestDataIndex: ({
apiKey,
url,
indexName,
}) => `const { Client } = require('@elastic/elasticsearch');
const client = new Client({
node: '${url}',
auth: {
apiKey: '${apiKey}'
}
});
const dataset = [
{'name': 'foo', 'title': 'bar'},
];
// Index with the bulk helper
const result = await client.helpers.bulk({
datasource: dataset,
onDocument (doc) {
return { index: { _index: '${indexName ?? 'index_name'}' }};
}
});
console.log(result);
`,
installClient: 'npm install @elastic/elasticsearch@8',
name: i18n.translate('xpack.serverlessSearch.languages.javascript', {
defaultMessage: 'JavaScript / Node.js',

View file

@ -7,6 +7,7 @@
import { i18n } from '@kbn/i18n';
import { docLinks } from '../../../../common/doc_links';
import { INDEX_NAME_PLACEHOLDER } from '../../constants';
import { LanguageDefinition, Languages } from './types';
export const rubyDefinition: LanguageDefinition = {
@ -31,6 +32,18 @@ export const rubyDefinition: LanguageDefinition = {
{ index: { _index: 'books', data: {name: "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} } }
]
client.bulk(body: documents)`,
ingestDataIndex: ({ apiKey, url, indexName }) => `client = ElasticsearchServerless::Client.new(
api_key: '${apiKey}',
url: '${url}'
)
documents = [
{ index: { _index: '${
indexName ?? INDEX_NAME_PLACEHOLDER
}', data: {name: "foo", "title": "bar"} } },
]
client.bulk(body: documents)
`,
installClient: `# Requires Ruby version 3.0 or higher
# From the project's root directory:$ gem build elasticsearch-serverless.gemspec

View file

@ -21,6 +21,7 @@ export enum Languages {
export interface LanguageDefinitionSnippetArguments {
url: string;
apiKey: string;
indexName?: string;
}
type CodeSnippet = string | ((args: LanguageDefinitionSnippetArguments) => string);
@ -33,6 +34,7 @@ export interface LanguageDefinition {
iconType: string;
id: Languages;
ingestData: CodeSnippet;
ingestDataIndex: CodeSnippet;
installClient: string;
languageStyling?: string;
name: string;

View file

@ -23,6 +23,7 @@ import React, { useMemo, useState } from 'react';
import { docLinks } from '../../../common/doc_links';
import { PLUGIN_ID } from '../../../common';
import { useKibanaServices } from '../hooks/use_kibana';
import { API_KEY_PLACEHOLDER, ELASTICSEARCH_URL_PLACEHOLDER } from '../constants';
import { CodeBox } from './code_box';
import { javascriptDefinition } from './languages/javascript';
import { languageDefinitions } from './languages/languages';
@ -35,9 +36,6 @@ import { SelectClientPanel } from './overview_panels/select_client';
import { ApiKeyPanel } from './api_key/api_key';
import { LanguageClientPanel } from './overview_panels/language_client_panel';
const ELASTICSEARCH_URL_PLACEHOLDER = 'https://your_deployment_url';
const API_KEY_PLACEHOLDER = 'your_api_key';
export const ElasticsearchOverview = () => {
const [selectedLanguage, setSelectedLanguage] =
useState<LanguageDefinition>(javascriptDefinition);

View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const API_KEY_PLACEHOLDER = 'your_api_key';
export const ELASTICSEARCH_URL_PLACEHOLDER = 'https://your_deployment_url';
export const INDEX_NAME_PLACEHOLDER = 'index_name';

View file

@ -9,3 +9,4 @@ export const MANAGEMENT_API_KEYS = '/app/management/security/api_keys';
// Server Routes
export const CREATE_API_KEY_PATH = '/internal/security/api_key';
export const FETCH_INDICES_PATH = '/internal/serverless_search/indices';

View file

@ -0,0 +1,122 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ByteSizeValue } from '@kbn/config-schema';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { fetchIndices } from './fetch_indices';
describe('fetch indices lib functions', () => {
const mockClient = {
indices: {
get: jest.fn(),
stats: jest.fn(),
},
security: {
hasPrivileges: jest.fn(),
},
asInternalUser: {},
};
const regularIndexResponse = {
'search-regular-index': {
aliases: {},
},
};
const regularIndexStatsResponse = {
indices: {
'search-regular-index': {
health: 'green',
size: new ByteSizeValue(108000).toString(),
status: 'open',
total: {
docs: {
count: 100,
deleted: 0,
},
store: {
size_in_bytes: 108000,
},
},
uuid: '83a81e7e-5955-4255-b008-5d6961203f57',
},
},
};
beforeEach(() => {
jest.clearAllMocks();
});
describe('fetchIndices', () => {
it('should return regular index', async () => {
mockClient.indices.get.mockImplementation(() => ({
...regularIndexResponse,
}));
mockClient.indices.stats.mockImplementation(() => regularIndexStatsResponse);
await expect(
fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search')
).resolves.toEqual([{ count: 100, name: 'search-regular-index' }]);
expect(mockClient.indices.get).toHaveBeenCalledWith({
expand_wildcards: ['open'],
features: ['aliases', 'settings'],
index: '*search*',
});
expect(mockClient.indices.stats).toHaveBeenCalledWith({
index: ['search-regular-index'],
metric: ['docs'],
});
});
it('should not return hidden indices', async () => {
mockClient.indices.get.mockImplementation(() => ({
...regularIndexResponse,
['search-regular-index']: {
...regularIndexResponse['search-regular-index'],
...{ settings: { index: { hidden: 'true' } } },
},
}));
mockClient.indices.stats.mockImplementation(() => regularIndexStatsResponse);
await expect(
fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20)
).resolves.toEqual([]);
expect(mockClient.indices.get).toHaveBeenCalledWith({
expand_wildcards: ['open'],
features: ['aliases', 'settings'],
index: '*',
});
expect(mockClient.indices.stats).not.toHaveBeenCalled();
});
it('should handle index missing in stats call', async () => {
const missingStatsResponse = {
indices: {
some_other_index: { ...regularIndexStatsResponse.indices['search-regular-index'] },
},
};
mockClient.indices.get.mockImplementationOnce(() => regularIndexResponse);
mockClient.indices.stats.mockImplementationOnce(() => missingStatsResponse);
// simulates when an index has been deleted after get indices call
// deleted index won't be present in the indices stats call response
await expect(
fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search')
).resolves.toEqual([{ count: 0, name: 'search-regular-index' }]);
});
it('should return empty array when no index found', async () => {
mockClient.indices.get.mockImplementationOnce(() => ({}));
await expect(
fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search')
).resolves.toEqual([]);
expect(mockClient.indices.stats).not.toHaveBeenCalled();
});
});
});

View file

@ -0,0 +1,60 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { isNotNullish } from '../../../common/utils/is_not_nullish';
import { isHidden, isClosed } from '../../utils/index_utils';
export async function fetchIndices(
client: ElasticsearchClient,
from: number,
size: number,
searchQuery?: string
) {
const indexPattern = searchQuery ? `*${searchQuery}*` : '*';
const indexMatches = await client.indices.get({
expand_wildcards: ['open'],
// for better performance only compute settings of indices but not mappings
features: ['aliases', 'settings'],
index: indexPattern,
});
const indexNames = Object.keys(indexMatches).filter(
(indexName) =>
indexMatches[indexName] &&
!isHidden(indexMatches[indexName]) &&
!isClosed(indexMatches[indexName])
);
const indexNameSlice = indexNames.slice(from, from + size).filter(isNotNullish);
if (indexNameSlice.length === 0) {
return [];
}
const indexCounts = await fetchIndexCounts(client, indexNameSlice);
return indexNameSlice.map((name) => ({
name,
count: indexCounts[name]?.total?.docs?.count ?? 0,
}));
}
const fetchIndexCounts = async (
client: ElasticsearchClient,
indicesNames: string[]
): Promise<Record<string, IndicesStatsIndicesStats | undefined>> => {
if (indicesNames.length === 0) {
return {};
}
const indexCounts: Record<string, IndicesStatsIndicesStats | undefined> = {};
// batch calls in batches of 100 to prevent loading too much onto ES
for (let i = 0; i < indicesNames.length; i += 100) {
const stats = await client.indices.stats({
index: indicesNames.slice(i, i + 100),
metric: ['docs'],
});
Object.assign(indexCounts, stats.indices);
}
return indexCounts;
};

View file

@ -8,6 +8,7 @@
import { IRouter, Logger, PluginInitializerContext, Plugin, CoreSetup } from '@kbn/core/server';
import { SecurityPluginStart } from '@kbn/security-plugin/server';
import { registerApiKeyRoutes } from './routes/api_key_routes';
import { registerIndicesRoutes } from './routes/indices_routes';
import { ServerlessSearchConfig } from './config';
import { ServerlessSearchPluginSetup, ServerlessSearchPluginStart } from './types';
@ -41,6 +42,7 @@ export class ServerlessSearchPlugin
const dependencies = { logger: this.logger, router, security: this.security };
registerApiKeyRoutes(dependencies);
registerIndicesRoutes(dependencies);
});
return {};
}

View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { schema } from '@kbn/config-schema';
import { fetchIndices } from '../lib/indices/fetch_indices';
import { RouteDependencies } from '../plugin';
export const registerIndicesRoutes = ({ router, security }: RouteDependencies) => {
router.get(
{
path: '/internal/serverless_search/indices',
validate: {
query: schema.object({
from: schema.number({ defaultValue: 0, min: 0 }),
search_query: schema.maybe(schema.string()),
size: schema.number({ defaultValue: 20, min: 0 }),
}),
},
},
async (context, request, response) => {
const client = (await context.core).elasticsearch.client.asCurrentUser;
const user = security.authc.getCurrentUser(request);
if (!user) {
return response.customError({
statusCode: 502,
body: 'Could not retrieve current user, security plugin is not ready',
});
}
const { from, size, search_query: searchQuery } = request.query;
const indices = await fetchIndices(client, from, size, searchQuery);
return response.ok({
body: {
indices,
},
headers: { 'content-type': 'application/json' },
});
}
);
};

View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
export function isHidden(index: IndicesIndexState): boolean {
return index.settings?.index?.hidden === true || index.settings?.index?.hidden === 'true';
}
export function isClosed(index: IndicesIndexState): boolean {
return (
index.settings?.index?.verified_before_close === true ||
index.settings?.index?.verified_before_close === 'true'
);
}

View file

@ -27,5 +27,6 @@
"@kbn/security-plugin",
"@kbn/cloud-plugin",
"@kbn/share-plugin",
"@kbn/core-elasticsearch-server",
]
}