[Fleet] Replace dataviews suggestions in KQL searchboxes with internal ones (#172190)

Fixes https://github.com/elastic/kibana/issues/169760
Related to https://github.com/elastic/kibana/issues/171425

## Summary

[Fleet] Replace dataviews suggestions in KQL searchboxes with internal
ones. Now using Fleet mappings to create the suggestions fields instead
of fetching them through dataView plugin.
This is done for two reasons:

- Solves [permission
problems](https://github.com/elastic/kibana/issues/169760) when the user
doesn't have privileges to read Fleet indices
- Allows us to search only those mappings that we want to expose,
instead of all of them

Only weird thing is that the [querystring
component](1f8c816901/src/plugins/unified_search/public/query_string_input/query_string_input.tsx (L161))
has a cap to show max 50 suggestions. Since for agents suggestions we
are showing some more fields, so the ones starting with `u` are not
visible anymore. I though I had a bug in the way I was creating the
`fieldsMap` but in reality there's no way to show more suggestions than
50 (without touching the original component, which I would gladly
avoid).


### Screenshots
There should be no visible difference with the current suggestions. 
<details>
  <summary>Agents</summary>

![Screenshot 2023-12-01 at 10 49
55](af73476c-3de2-40c1-93fc-c6a1c28a8a8a)

![Screenshot 2023-12-01 at 10 49
48](5db8b30f-ff9e-4542-a590-f77285dbeef6)
  
</details>

<details>
  <summary>Agent policies</summary>

![Screenshot 2023-12-01 at 10 50
09](69756149-6769-48a9-9a34-de482e4e37fc)

</details>

<details>
  <summary>Enrollment keys</summary>

![Screenshot 2023-12-01 at 10 50
18](e542550a-9721-4f5c-a05b-32829dd8fcee)


</details>

### Testing
1. With a normal user, navigate to the "agents", "agent policies" and
"enrollment keys" tabs and click on the searchboxes. The suggestions
should be visible as normal
2. Create a user with role Fleet "all", Integrations "all". Log in and
check the above searchboxes, the suggestions should be visible as
normal. Previously they weren't.

### Checklist
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Cristina Amico 2023-12-04 09:19:26 +01:00 committed by GitHub
parent e1b585cf76
commit ad663136c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 329 additions and 149 deletions

View file

@ -63,7 +63,7 @@ pageLoadAssetSize:
files: 22673
filesManagement: 18683
fileUpload: 25664
fleet: 158438
fleet: 174609
globalSearch: 29696
globalSearchBar: 50403
globalSearchProviders: 25554

View file

@ -55,3 +55,5 @@ export const FLEET_SERVER_INDICES = [
export const DATA_TIERS = ['data_hot'];
export const FLEET_ENROLLMENT_API_PREFIX = 'fleet-enrollment-api-keys';
export * from './mappings';

View file

@ -6,7 +6,7 @@
*/
/**
* ATTENTION: New mappings for Fleet are defined in the ElasticSearch repo.
* ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo.
*
* The following mappings declared here closely mirror them
* But they are only used to perform validation on the endpoints using ListWithKuery
@ -305,7 +305,6 @@ export const AGENT_MAPPINGS = {
fields: {
keyword: {
type: 'keyword',
ignore_above: 16,
},
},
},
@ -331,7 +330,6 @@ export const AGENT_MAPPINGS = {
fields: {
keyword: {
type: 'keyword',
ignore_above: 1024,
},
},
},

View file

@ -6,12 +6,20 @@
*/
import React from 'react';
import { act } from '@testing-library/react';
import type { FieldSpec } from '@kbn/data-plugin/common';
import { createFleetTestRendererMock } from '../../../mock';
import { SearchBar, filterAndConvertFields } from './search_bar';
import {
AGENTS_PREFIX,
FLEET_ENROLLMENT_API_PREFIX,
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
AGENT_POLICY_SAVED_OBJECT_TYPE,
} from '../constants';
import { SearchBar, getFieldSpecs } from './search_bar';
const fields = [
{
@ -36,39 +44,6 @@ const fields = [
},
] as FieldSpec[];
const allFields = [
{
name: 'test-index._id',
type: 'string',
esTypes: ['_id'],
},
{
name: 'test-index.api_key',
type: 'string',
esTypes: ['keyword'],
},
{
name: 'test-index.name',
type: 'string',
esTypes: ['keyword'],
},
{
name: 'another-index.version',
type: 'string',
esTypes: ['keyword'],
},
{
name: 'test2-index.name',
type: 'string',
esTypes: ['keyword'],
},
{
name: 'fleet-agents.actions',
type: 'string',
esTypes: ['keyword'],
},
] as FieldSpec[];
jest.mock('../hooks', () => {
return {
...jest.requireActual('../hooks'),
@ -87,23 +62,6 @@ jest.mock('../hooks', () => {
},
data: {
dataViews: {
getFieldsForWildcard: jest.fn().mockResolvedValue([
{
name: '_id',
type: 'string',
esTypes: ['_id'],
},
{
name: 'api_key',
type: 'string',
esTypes: ['keyword'],
},
{
name: 'name',
type: 'string',
esTypes: ['keyword'],
},
]),
create: jest.fn().mockResolvedValue({
fields,
}),
@ -197,6 +155,9 @@ describe('SearchBar', () => {
);
it('renders the search box', async () => {
await act(async () => {
result.queryByTestId('queryInput');
});
const textArea = result.queryByTestId('queryInput');
expect(textArea).not.toBeNull();
expect(textArea?.getAttribute('placeholder')).toEqual('Filter your data using KQL syntax');
@ -207,60 +168,243 @@ describe('SearchBar', () => {
});
});
describe('filterAndConvertFields', () => {
it('leaves the fields names unchanged and does not hide any fields if fieldPrefix is not passed', async () => {
expect(filterAndConvertFields(fields, '.test-index')).toEqual({
_id: { esTypes: ['_id'], name: '_id', type: 'string' },
api_key: { esTypes: ['keyword'], name: 'api_key', type: 'string' },
name: { esTypes: ['keyword'], name: 'name', type: 'string' },
version: { esTypes: ['keyword'], name: 'version', type: 'string' },
});
describe('getFieldSpecs', () => {
it('returns fieldSpecs for fleet-agents', () => {
expect(getFieldSpecs(`.${AGENTS_PREFIX}`)).toHaveLength(66);
});
it('returns getFieldSpecs for fleet-enrollment-api-keys', () => {
const indexPattern = `.${FLEET_ENROLLMENT_API_PREFIX}`;
expect(getFieldSpecs(indexPattern)).toHaveLength(8);
expect(getFieldSpecs(indexPattern)).toEqual([
{
aggregatable: true,
esTypes: ['boolean'],
name: 'active',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'api_key',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'api_key_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['date'],
name: 'created_at',
searchable: true,
type: 'date',
},
{
aggregatable: true,
esTypes: ['date'],
name: 'expire_at',
searchable: true,
type: 'date',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'name',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'policy_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['date'],
name: 'updated_at',
searchable: true,
type: 'date',
},
]);
});
it('filters out the fields from other indices if indexPattern === .kibana-ingest', async () => {
expect(filterAndConvertFields(allFields, '.kibana_ingest', 'test-index')).toEqual({
'test-index._id': { esTypes: ['_id'], name: 'test-index._id', type: 'string' },
'test-index.api_key': { esTypes: ['keyword'], name: 'test-index.api_key', type: 'string' },
'test-index.name': { esTypes: ['keyword'], name: 'test-index.name', type: 'string' },
});
it('returns getFieldSpecs for fleet-agent-policy', () => {
const indexPattern = `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`;
expect(getFieldSpecs(indexPattern)).toHaveLength(23);
expect(getFieldSpecs(indexPattern)).toEqual([
{
aggregatable: true,
esTypes: ['keyword'],
name: 'agent_features.name',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['boolean'],
name: 'agent_features.enabled',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'data_output_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['text'],
name: 'description',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'download_source_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'fleet_server_host_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['integer'],
name: 'inactivity_timeout',
searchable: true,
type: 'number',
},
{
aggregatable: true,
esTypes: ['boolean'],
name: 'is_default',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['boolean'],
name: 'is_default_fleet_server',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['boolean'],
name: 'is_managed',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'is_preconfigured',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['boolean'],
name: 'is_protected',
searchable: true,
type: 'boolean',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'monitoring_enabled',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['false'],
name: 'monitoring_enabled.index',
searchable: true,
type: 'false',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'monitoring_output_id',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'name',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'namespace',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['integer'],
name: 'revision',
searchable: true,
type: 'number',
},
{
aggregatable: true,
esTypes: ['version'],
name: 'schema_version',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'status',
searchable: true,
type: 'string',
},
{
aggregatable: true,
esTypes: ['integer'],
name: 'unenroll_timeout',
searchable: true,
type: 'number',
},
{
aggregatable: true,
esTypes: ['date'],
name: 'updated_at',
searchable: true,
type: 'date',
},
{
aggregatable: true,
esTypes: ['keyword'],
name: 'updated_by',
searchable: true,
type: 'string',
},
]);
});
expect(getFieldSpecs(`.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`)).toHaveLength(18);
it('returns fields unchanged if fieldPrefix and indexPattern are not passed', async () => {
expect(filterAndConvertFields(allFields, undefined, undefined)).toEqual({
'another-index.version': {
esTypes: ['keyword'],
name: 'another-index.version',
type: 'string',
},
'fleet-agents.actions': {
esTypes: ['keyword'],
name: 'fleet-agents.actions',
type: 'string',
},
'test-index._id': {
esTypes: ['_id'],
name: 'test-index._id',
type: 'string',
},
'test-index.api_key': {
esTypes: ['keyword'],
name: 'test-index.api_key',
type: 'string',
},
'test-index.name': {
esTypes: ['keyword'],
name: 'test-index.name',
type: 'string',
},
'test2-index.name': {
esTypes: ['keyword'],
name: 'test2-index.name',
type: 'string',
},
});
});
it('returns empty object if fields is empty', async () => {
expect(filterAndConvertFields([], '.kibana_ingest', 'test-index')).toEqual({});
it('returns empty array if indexPattern is not one of the previous', async () => {
expect(getFieldSpecs('.kibana_ingest')).toEqual([]);
});
});

View file

@ -16,9 +16,19 @@ import type { DataView } from '@kbn/data-views-plugin/public';
import { i18n } from '@kbn/i18n';
import { useStartServices } from '../hooks';
import { INDEX_NAME, AGENTS_PREFIX } from '../constants';
const HIDDEN_FIELDS = [`${AGENTS_PREFIX}.actions`, '_id', '_index'];
import {
INDEX_NAME,
AGENTS_PREFIX,
FLEET_ENROLLMENT_API_PREFIX,
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
AGENT_POLICY_SAVED_OBJECT_TYPE,
} from '../constants';
import {
AGENT_POLICY_MAPPINGS,
PACKAGE_POLICIES_MAPPINGS,
AGENT_MAPPINGS,
ENROLLMENT_API_KEY_MAPPINGS,
} from '../../../../common/constants';
const NoWrapQueryStringInput = styled(QueryStringInput)`
.kbnQueryBar__textarea {
@ -35,39 +45,69 @@ interface Props {
dataTestSubj?: string;
}
/** Exported for testing only **/
export const filterAndConvertFields = (
fields: FieldSpec[],
indexPattern?: string,
fieldPrefix?: string
) => {
if (!fields) return {};
let filteredFields: FieldSpec[] = [];
if (fieldPrefix) {
// exclude fields from different indices
if (indexPattern === INDEX_NAME) {
filteredFields = fields.filter((field) => field.name.startsWith(fieldPrefix));
} else {
// filter out fields that have names to be hidden
filteredFields = fields.filter((field) => {
for (const hiddenField of HIDDEN_FIELDS) {
if (field.name.includes(hiddenField)) {
return false;
}
}
return true;
});
}
} else {
filteredFields = fields;
const getMappings = (indexPattern: string) => {
switch (indexPattern) {
case `.${AGENTS_PREFIX}`:
return AGENT_MAPPINGS;
case `.${AGENT_POLICY_SAVED_OBJECT_TYPE}`:
return AGENT_POLICY_MAPPINGS;
case `.${PACKAGE_POLICY_SAVED_OBJECT_TYPE}`:
return PACKAGE_POLICIES_MAPPINGS;
case `.${FLEET_ENROLLMENT_API_PREFIX}`:
return ENROLLMENT_API_KEY_MAPPINGS;
default:
return {};
}
};
const fieldsMap = filteredFields.reduce((acc: Record<string, FieldSpec>, curr: FieldSpec) => {
acc[curr.name] = curr;
return acc;
}, {});
return fieldsMap;
const getType = (type: string) => {
switch (type) {
case 'keyword':
return 'string';
case 'text':
return 'string';
case 'version':
return 'string';
case 'integer':
return 'number';
case 'double':
return 'number';
default:
return type;
}
};
const concatKeys = (obj: any, parentKey = '') => {
let result: string[] = [];
for (const key in obj) {
if (typeof obj[key] === 'object') {
result = result.concat(concatKeys(obj[key], `${parentKey}${key}.`));
} else {
result.push(`${parentKey}${key}:${obj[key]}`);
}
}
return result;
};
/** Exported for testing only **/
export const getFieldSpecs = (indexPattern: string) => {
const mapping = getMappings(indexPattern);
// @ts-ignore-next-line
const rawFields = concatKeys(mapping?.properties) || [];
const fields = rawFields
.map((field) => field.replaceAll(/.properties/g, ''))
.map((field) => field.replace(/.type/g, ''))
.map((field) => field.split(':'));
const fieldSpecs: FieldSpec[] = fields.map((field) => {
return {
name: field[0],
type: getType(field[1]),
searchable: true,
aggregatable: true,
esTypes: [field[1]],
};
});
return fieldSpecs;
};
export const SearchBar: React.FunctionComponent<Props> = ({
@ -108,16 +148,11 @@ export const SearchBar: React.FunctionComponent<Props> = ({
useEffect(() => {
const fetchFields = async () => {
try {
const fields: FieldSpec[] = await data.dataViews.getFieldsForWildcard({
pattern: indexPattern,
});
const fieldsMap = filterAndConvertFields(fields, indexPattern, fieldPrefix);
// Refetch only if fieldsMap is empty
const skipFetchField = !!fieldsMap;
const fieldSpecs = getFieldSpecs(indexPattern);
const fieldsMap = data.dataViews.fieldArrayToMap(fieldSpecs);
const newDataView = await data.dataViews.create(
{ title: indexPattern, fields: fieldsMap },
skipFetchField
true
);
setDataView(newDataView);
} catch (err) {

View file

@ -260,6 +260,7 @@ export const AgentPolicyListPage: React.FunctionComponent<{}> = () => {
});
setSearch(newSearch);
}}
indexPattern={`.${AGENT_POLICY_SAVED_OBJECT_TYPE}`}
fieldPrefix={AGENT_POLICY_SAVED_OBJECT_TYPE}
dataTestSubj="agentPolicyList.queryInput"
/>

View file

@ -102,4 +102,4 @@ export {
} from './fleet_es_assets';
export { FILE_STORAGE_DATA_AGENT_INDEX } from './fleet_es_assets';
export { FILE_STORAGE_METADATA_AGENT_INDEX } from './fleet_es_assets';
export * from './mappings';
export * from '../../common/constants/mappings';