Revert "[RAM] [PERF] Remove endpoint browserFields" (#157441)

Reverts elastic/kibana#156869

We need to revert because after talking to @kobelb, we are introducing a
new bug where user always need to be a super user to access the fields
from the alert index since only only super user can access the kibana
index.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Xavier Mouligneau 2023-05-12 12:24:14 -04:00 committed by GitHub
parent 41667399cf
commit 982cb49cb5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 435 additions and 213 deletions

View file

@ -314,7 +314,7 @@ export interface BrowserField {
category: string;
description?: string | null;
example?: string | number | null;
fields: Record<string, Partial<BrowserField>>;
fields: Readonly<Record<string, Partial<BrowserField>>>;
format?: string;
indexes: string[];
name: string;

View file

@ -20,6 +20,7 @@ const createAlertsClientMock = () => {
bulkUpdateCases: jest.fn(),
find: jest.fn(),
getFeatureIdsByRegistrationContexts: jest.fn(),
getBrowserFields: jest.fn(),
getAlertSummary: jest.fn(),
ensureAllAlertsAuthorizedRead: jest.fn(),
removeCaseIdFromAlerts: jest.fn(),

View file

@ -38,7 +38,9 @@ import {
} from '@kbn/alerting-plugin/server';
import { Logger, ElasticsearchClient, EcsEvent } from '@kbn/core/server';
import { AuditLogger } from '@kbn/security-plugin/server';
import { IndexPatternsFetcher } from '@kbn/data-plugin/server';
import { isEmpty } from 'lodash';
import { BrowserFields } from '../../common';
import { alertAuditEvent, operationAlertAuditActionMap } from './audit_events';
import {
ALERT_WORKFLOW_STATUS,
@ -49,6 +51,7 @@ import {
import { ParsedTechnicalFields } from '../../common/parse_technical_fields';
import { Dataset, IRuleDataService } from '../rule_data_plugin_service';
import { getAuthzFilter, getSpacesFilter } from '../lib';
import { fieldDescriptorToBrowserFieldMapper } from './browser_fields';
// TODO: Fix typings https://github.com/elastic/kibana/issues/101776
type NonNullableProps<Obj extends {}, Props extends keyof Obj> = Omit<Obj, Props> & {
@ -1071,4 +1074,23 @@ export class AlertsClient {
throw Boom.failedDependency(errMessage);
}
}
public async getBrowserFields({
indices,
metaFields,
allowNoIndex,
}: {
indices: string[];
metaFields: string[];
allowNoIndex: boolean;
}): Promise<BrowserFields> {
const indexPatternsFetcherAsInternalUser = new IndexPatternsFetcher(this.esClient);
const { fields } = await indexPatternsFetcherAsInternalUser.getFieldsForWildcard({
pattern: indices,
metaFields,
fieldCapsOptions: { allow_no_indices: allowNoIndex },
});
return fieldDescriptorToBrowserFieldMapper(fields);
}
}

View file

@ -0,0 +1,46 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FieldDescriptor } from '@kbn/data-views-plugin/server';
import { BrowserFields, BrowserField } from '../../../common';
const getFieldCategory = (fieldCapability: FieldDescriptor) => {
const name = fieldCapability.name.split('.');
if (name.length === 1) {
return 'base';
}
return name[0];
};
const browserFieldFactory = (
fieldCapability: FieldDescriptor,
category: string
): Readonly<Record<string, Partial<BrowserField>>> => {
return {
[fieldCapability.name]: {
...fieldCapability,
category,
},
};
};
export const fieldDescriptorToBrowserFieldMapper = (fields: FieldDescriptor[]): BrowserFields => {
return fields.reduce((browserFields: BrowserFields, field: FieldDescriptor) => {
const category = getFieldCategory(field);
const browserField = browserFieldFactory(field, category);
if (browserFields[category]) {
browserFields[category] = { fields: { ...browserFields[category].fields, ...browserField } };
} else {
browserFields[category] = { fields: browserField };
}
return browserFields;
}, {});
};

View file

@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { BASE_RAC_ALERTS_API_PATH } from '../../common/constants';
import { getBrowserFieldsByFeatureId } from './get_browser_fields_by_feature_id';
import { requestContextMock } from './__mocks__/request_context';
import { getO11yBrowserFields } from './__mocks__/request_responses';
import { requestMock, serverMock } from './__mocks__/server';
describe('getBrowserFieldsByFeatureId', () => {
let server: ReturnType<typeof serverMock.create>;
let { clients, context } = requestContextMock.createTools();
const path = `${BASE_RAC_ALERTS_API_PATH}/browser_fields`;
beforeEach(async () => {
server = serverMock.create();
({ clients, context } = requestContextMock.createTools());
});
describe('when racClient returns o11y indices', () => {
beforeEach(() => {
clients.rac.getAuthorizedAlertsIndices.mockResolvedValue([
'.alerts-observability.logs.alerts-default',
]);
getBrowserFieldsByFeatureId(server.router);
});
test('route registered', async () => {
const response = await server.inject(getO11yBrowserFields(), context);
expect(response.status).toEqual(200);
});
test('rejects invalid featureId type', async () => {
await expect(
server.inject(
requestMock.create({
method: 'get',
path,
query: { featureIds: undefined },
}),
context
)
).rejects.toThrowErrorMatchingInlineSnapshot(
`"Request was rejected with message: 'Invalid value \\"undefined\\" supplied to \\"featureIds\\"'"`
);
});
test('returns error status if rac client "getAuthorizedAlertsIndices" fails', async () => {
clients.rac.getAuthorizedAlertsIndices.mockRejectedValue(new Error('Unable to get index'));
const response = await server.inject(getO11yBrowserFields(), context);
expect(response.status).toEqual(500);
expect(response.body).toEqual({
attributes: { success: false },
message: 'Unable to get index',
});
});
});
});

View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IRouter } from '@kbn/core/server';
import { transformError } from '@kbn/securitysolution-es-utils';
import * as t from 'io-ts';
import { BrowserFields } from '../../common';
import { RacRequestHandlerContext } from '../types';
import { BASE_RAC_ALERTS_API_PATH } from '../../common/constants';
import { buildRouteValidation } from './utils/route_validation';
export const getBrowserFieldsByFeatureId = (router: IRouter<RacRequestHandlerContext>) => {
router.get(
{
path: `${BASE_RAC_ALERTS_API_PATH}/browser_fields`,
validate: {
query: buildRouteValidation(
t.exact(
t.type({
featureIds: t.union([t.string, t.array(t.string)]),
})
)
),
},
options: {
tags: ['access:rac'],
},
},
async (context, request, response) => {
try {
const racContext = await context.rac;
const alertsClient = await racContext.getAlertsClient();
const { featureIds = [] } = request.query;
const indices = await alertsClient.getAuthorizedAlertsIndices(
Array.isArray(featureIds) ? featureIds : [featureIds]
);
const o11yIndices =
indices?.filter((index) => index.startsWith('.alerts-observability')) ?? [];
if (o11yIndices.length === 0) {
return response.notFound({
body: {
message: `No alerts-observability indices found for featureIds [${featureIds}]`,
attributes: { success: false },
},
});
}
const browserFields: BrowserFields = await alertsClient.getBrowserFields({
indices: o11yIndices,
metaFields: ['_id', '_index'],
allowNoIndex: true,
});
return response.ok({
body: browserFields,
});
} catch (error) {
const formatedError = transformError(error);
const contentType = {
'content-type': 'application/json',
};
const defaultedHeaders = {
...contentType,
};
return response.customError({
headers: defaultedHeaders,
statusCode: formatedError.statusCode,
body: {
message: formatedError.message,
attributes: {
success: false,
},
},
});
}
}
);
};

View file

@ -13,6 +13,7 @@ import { getAlertsIndexRoute } from './get_alert_index';
import { bulkUpdateAlertsRoute } from './bulk_update_alerts';
import { findAlertsByQueryRoute } from './find';
import { getFeatureIdsByRegistrationContexts } from './get_feature_ids_by_registration_contexts';
import { getBrowserFieldsByFeatureId } from './get_browser_fields_by_feature_id';
import { getAlertSummaryRoute } from './get_alert_summary';
export function defineRoutes(router: IRouter<RacRequestHandlerContext>) {
@ -22,5 +23,6 @@ export function defineRoutes(router: IRouter<RacRequestHandlerContext>) {
bulkUpdateAlertsRoute(router);
findAlertsByQueryRoute(router);
getFeatureIdsByRegistrationContexts(router);
getBrowserFieldsByFeatureId(router);
getAlertSummaryRoute(router);
}

View file

@ -9,37 +9,24 @@ import { DataView } from '@kbn/data-views-plugin/common';
import { useKibana } from '@kbn/kibana-react-plugin/public';
import { BASE_RAC_ALERTS_API_PATH } from '@kbn/rule-registry-plugin/common';
import type { ValidFeatureId } from '@kbn/rule-data-utils';
import type { HttpStart } from '@kbn/core/public';
import useAsync from 'react-use/lib/useAsync';
import type { AsyncState } from 'react-use/lib/useAsync';
import { DataPublicPluginStart } from '@kbn/data-plugin/public';
import { TriggersAndActionsUiServices } from '../..';
export const loadAlertDataView = async ({
http,
dataService,
featureIds,
}: {
http: HttpStart;
dataService: DataPublicPluginStart;
featureIds: ValidFeatureId[];
}): Promise<DataView> => {
const features = featureIds.sort().join(',');
const { index_name: indexNames } = await http.get<{ index_name: string[] }>(
`${BASE_RAC_ALERTS_API_PATH}/index`,
{
query: { features },
}
);
return dataService.dataViews.create({ title: indexNames.join(','), allowNoIndex: true });
};
export function useAlertDataView(featureIds: ValidFeatureId[]): AsyncState<DataView> {
const { http, data: dataService } = useKibana<TriggersAndActionsUiServices>().services;
const features = featureIds.sort().join(',');
const dataView = useAsync(async () => {
return loadAlertDataView({ http, dataService, featureIds });
}, [featureIds]);
const { index_name: indexNames } = await http.get<{ index_name: string[] }>(
`${BASE_RAC_ALERTS_API_PATH}/index`,
{
query: { features },
}
);
return dataService.dataViews.create({ title: indexNames.join(','), allowNoIndex: true });
}, [features]);
return dataView;
}

View file

@ -0,0 +1,78 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { renderHook } from '@testing-library/react-hooks';
import { useFetchBrowserFieldCapabilities } from './use_fetch_browser_fields_capabilities';
import { useKibana } from '../../../../common/lib/kibana';
import { BrowserFields } from '@kbn/rule-registry-plugin/common';
import { AlertsField } from '../../../../types';
jest.mock('../../../../common/lib/kibana');
const browserFields: BrowserFields = {
kibana: {
fields: {
[AlertsField.uuid]: {
category: 'kibana',
name: AlertsField.uuid,
},
[AlertsField.name]: {
category: 'kibana',
name: AlertsField.name,
},
[AlertsField.reason]: {
category: 'kibana',
name: AlertsField.reason,
},
},
},
};
describe('useFetchBrowserFieldCapabilities', () => {
let httpMock: jest.Mock;
beforeEach(() => {
httpMock = useKibana().services.http.get as jest.Mock;
httpMock.mockReturnValue({ fakeCategory: {} });
});
afterEach(() => {
httpMock.mockReset();
});
it('should not fetch for siem', () => {
const { result } = renderHook(() => useFetchBrowserFieldCapabilities({ featureIds: ['siem'] }));
expect(httpMock).toHaveBeenCalledTimes(0);
expect(result.current).toEqual([undefined, {}]);
});
it('should call the api only once', async () => {
const { result, waitForNextUpdate, rerender } = renderHook(() =>
useFetchBrowserFieldCapabilities({ featureIds: ['apm'] })
);
await waitForNextUpdate();
expect(httpMock).toHaveBeenCalledTimes(1);
expect(result.current).toEqual([false, { fakeCategory: {} }]);
rerender();
expect(httpMock).toHaveBeenCalledTimes(1);
expect(result.current).toEqual([false, { fakeCategory: {} }]);
});
it('should not fetch if browserFields have been provided', async () => {
const { result } = renderHook(() =>
useFetchBrowserFieldCapabilities({ featureIds: ['apm'], initialBrowserFields: browserFields })
);
expect(httpMock).toHaveBeenCalledTimes(0);
expect(result.current).toEqual([undefined, browserFields]);
});
});

View file

@ -1,126 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { renderHook } from '@testing-library/react-hooks';
import { createStartServicesMock } from '../../../../common/lib/kibana/kibana_react.mock';
import { useFetchBrowserFieldCapabilities } from './use_fetch_browser_fields_capabilities';
import { BrowserFields } from '@kbn/rule-registry-plugin/common';
import { AlertsField } from '../../../../types';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
const mockUseKibanaReturnValue = createStartServicesMock();
jest.mock('@kbn/kibana-react-plugin/public', () => ({
__esModule: true,
useKibana: jest.fn(() => ({
services: mockUseKibanaReturnValue,
})),
}));
const queryClient = new QueryClient({
defaultOptions: {
queries: {
retry: false,
cacheTime: 0,
},
},
});
const browserFields: BrowserFields = {
kibana: {
fields: {
[AlertsField.uuid]: {
category: 'kibana',
name: AlertsField.uuid,
},
[AlertsField.name]: {
category: 'kibana',
name: AlertsField.name,
},
[AlertsField.reason]: {
category: 'kibana',
name: AlertsField.reason,
},
},
},
};
describe('useFetchBrowserFieldCapabilities', () => {
const MOCKED_DATA_VIEW = {
id: 'fakeId',
fields: [{ name: 'fakeField' }],
};
const wrapper: React.FunctionComponent = ({ children }) => (
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
);
beforeEach(() => {
mockUseKibanaReturnValue.http.get = jest.fn().mockReturnValue({
index_name: [
'.alerts-observability.uptime.alerts-*',
'.alerts-observability.metrics.alerts-*',
'.alerts-observability.logs.alerts-*',
'.alerts-observability.apm.alerts-*',
],
});
mockUseKibanaReturnValue.data.dataViews.create = jest.fn().mockReturnValue(MOCKED_DATA_VIEW);
});
afterEach(() => {
jest.clearAllMocks();
});
it('should not fetch for siem', () => {
const { result } = renderHook(
() =>
useFetchBrowserFieldCapabilities({
featureIds: ['siem'],
}),
{ wrapper }
);
expect(mockUseKibanaReturnValue.http.get).toHaveBeenCalledTimes(0);
expect(result.current).toEqual([undefined, {}]);
});
it('should call the api only once', async () => {
const { result, waitFor, rerender } = renderHook(
() => useFetchBrowserFieldCapabilities({ featureIds: ['apm'] }),
{ wrapper }
);
await waitFor(() => result.current[0] === false);
expect(mockUseKibanaReturnValue.http.get).toHaveBeenCalledTimes(1);
expect(result.current).toEqual([
false,
{ base: { fields: { fakeField: { name: 'fakeField' } } } },
]);
rerender();
expect(mockUseKibanaReturnValue.http.get).toHaveBeenCalledTimes(1);
expect(result.current).toEqual([
false,
{ base: { fields: { fakeField: { name: 'fakeField' } } } },
]);
});
it('should not fetch if browserFields have been provided', () => {
const { result } = renderHook(
() =>
useFetchBrowserFieldCapabilities({
featureIds: ['apm'],
initialBrowserFields: browserFields,
}),
{ wrapper }
);
expect(mockUseKibanaReturnValue.http.get).toHaveBeenCalledTimes(0);
expect(result.current).toEqual([undefined, browserFields]);
});
});

View file

@ -6,15 +6,11 @@
*/
import type { ValidFeatureId } from '@kbn/rule-data-utils';
import { BrowserField, BrowserFields } from '@kbn/rule-registry-plugin/common';
import type { DataViewFieldBase } from '@kbn/es-query';
import { useQuery } from '@tanstack/react-query';
import { useMemo, useRef } from 'react';
import { loadAlertDataView } from '../../../hooks/use_alert_data_view';
import { BASE_RAC_ALERTS_API_PATH, BrowserFields } from '@kbn/rule-registry-plugin/common';
import { useCallback, useEffect, useState } from 'react';
import type { Alerts } from '../../../../types';
import { useKibana } from '../../../../common/lib/kibana';
import { ERROR_FETCH_BROWSER_FIELDS } from './translations';
import { getCategory } from '../../field_browser/helpers';
export interface FetchAlertsArgs {
featureIds: ValidFeatureId[];
@ -29,75 +25,56 @@ export type UseFetchAlerts = ({ featureIds }: FetchAlertsArgs) => [boolean, Fetc
const INVALID_FEATURE_ID = 'siem';
/**
* HOT Code path where the fields can be 16087 in length or larger. This is
* VERY mutatious on purpose to improve the performance of the transform.
*/
const getDataViewStateFromIndexFields = (
_title: string,
fields: DataViewFieldBase[],
_includeUnmapped: boolean = false
): BrowserFields => {
// Adds two dangerous casts to allow for mutations within this function
type DangerCastForMutation = Record<string, {}>;
return fields.reduce<BrowserFields>((browserFields, field) => {
// mutate browserFields
const category = getCategory(field.name);
if (browserFields[category] == null) {
(browserFields as DangerCastForMutation)[category] = {};
}
if (browserFields[category].fields == null) {
browserFields[category].fields = {};
}
(browserFields[category].fields as Record<string, BrowserField>)[field.name] =
field as unknown as BrowserField;
return browserFields;
}, {});
};
export const useFetchBrowserFieldCapabilities = ({
featureIds,
initialBrowserFields,
}: FetchAlertsArgs): [boolean | undefined, BrowserFields] => {
const {
http,
data: dataService,
notifications: { toasts },
} = useKibana().services;
const enabled = !initialBrowserFields && !featureIds.includes(INVALID_FEATURE_ID);
const [isLoading, setIsLoading] = useState<boolean | undefined>(undefined);
const [browserFields, setBrowserFields] = useState<BrowserFields>(
() => initialBrowserFields ?? {}
);
const { data: dataView, isFetching } = useQuery({
queryKey: ['fetchBrowserFields', featureIds],
queryFn: () => {
return loadAlertDataView({ http, dataService, featureIds });
},
onError: () => {
const getBrowserFieldInfo = useCallback(async () => {
if (!http) return Promise.resolve({});
try {
return await http.get<BrowserFields>(`${BASE_RAC_ALERTS_API_PATH}/browser_fields`, {
query: { featureIds },
});
} catch (e) {
toasts.addDanger(ERROR_FETCH_BROWSER_FIELDS);
},
enabled,
keepPreviousData: true,
cacheTime: 0,
refetchOnWindowFocus: false,
});
const loading = useRef<boolean | undefined>(isFetching);
const browserFields = useMemo(() => {
if (!enabled && initialBrowserFields) {
loading.current = undefined;
return initialBrowserFields;
}
if (!dataView) {
loading.current = undefined;
return {};
}
loading.current = isFetching;
return getDataViewStateFromIndexFields(
dataView.id ?? '',
dataView.fields != null ? dataView.fields : []
);
}, [enabled, initialBrowserFields, dataView, isFetching]);
}, [featureIds, http, toasts]);
return [loading.current, browserFields];
useEffect(() => {
if (initialBrowserFields) {
// Event if initial browser fields is empty, assign it
// because client may be doing it to hide Fields Browser
setBrowserFields(initialBrowserFields);
return;
}
if (isLoading !== undefined || featureIds.includes(INVALID_FEATURE_ID)) {
return;
}
setIsLoading(true);
const callApi = async () => {
const browserFieldsInfo = await getBrowserFieldInfo();
setBrowserFields(browserFieldsInfo);
setIsLoading(false);
};
callApi();
}, [getBrowserFieldInfo, isLoading, featureIds, initialBrowserFields]);
return [isLoading, browserFields];
};

View file

@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { superUser, obsOnlySpacesAll, secOnlyRead } from '../../../common/lib/authentication/users';
import type { User } from '../../../common/lib/authentication/types';
import { FtrProviderContext } from '../../../common/ftr_provider_context';
import { getSpaceUrlPrefix } from '../../../common/lib/authentication/spaces';
// eslint-disable-next-line import/no-default-export
export default ({ getService }: FtrProviderContext) => {
const supertestWithoutAuth = getService('supertestWithoutAuth');
const esArchiver = getService('esArchiver');
const SPACE1 = 'space1';
const TEST_URL = '/internal/rac/alerts/browser_fields';
const getBrowserFieldsByFeatureId = async (
user: User,
featureIds: string[],
expectedStatusCode: number = 200
) => {
const resp = await supertestWithoutAuth
.get(`${getSpaceUrlPrefix(SPACE1)}${TEST_URL}`)
.query({ featureIds })
.auth(user.username, user.password)
.set('kbn-xsrf', 'true')
.expect(expectedStatusCode);
return resp.body;
};
describe('Alert - Get browser fields by featureId', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/rule_registry/alerts');
});
describe('Users:', () => {
it(`${obsOnlySpacesAll.username} should be able to get browser fields for o11y featureIds`, async () => {
const browserFields = await getBrowserFieldsByFeatureId(obsOnlySpacesAll, [
'apm',
'infrastructure',
'logs',
'uptime',
]);
expect(Object.keys(browserFields)).to.eql(['base', 'event', 'kibana', 'message']);
});
it(`${superUser.username} should be able to get browser fields for o11y featureIds`, async () => {
const browserFields = await getBrowserFieldsByFeatureId(superUser, [
'apm',
'infrastructure',
'logs',
'uptime',
]);
expect(Object.keys(browserFields)).to.eql([
'base',
'agent',
'anomaly',
'ecs',
'error',
'event',
'kibana',
'message',
'monitor',
'observer',
'tls',
'url',
]);
});
it(`${superUser.username} should NOT be able to get browser fields for siem featureId`, async () => {
await getBrowserFieldsByFeatureId(superUser, ['siem'], 404);
});
it(`${secOnlyRead.username} should NOT be able to get browser fields for siem featureId`, async () => {
await getBrowserFieldsByFeatureId(secOnlyRead, ['siem'], 404);
});
});
});
};

View file

@ -29,6 +29,7 @@ export default ({ loadTestFile, getService }: FtrProviderContext): void => {
loadTestFile(require.resolve('./get_alerts_index'));
loadTestFile(require.resolve('./find_alerts'));
loadTestFile(require.resolve('./search_strategy'));
loadTestFile(require.resolve('./get_browser_fields_by_feature_id'));
loadTestFile(require.resolve('./get_alert_summary'));
});
};