[SecuritySolution][PrivMon] Rewrite dashboard queries to use FORK (#223212)

## Summary


### What is included?
* Improves the auth dashboard to display system events
* Add data view index patterns as visualisations index
* Move ESQL query generation to a shared folder
* Parse ESQL query and validate if fields exist in the dataview
* Rewrite the ESQL query if a FORK command has missing fields
* Add a visualisation warning message when there is no valid FORK branch

![Screenshot 2025-06-20 at 07 22
47](https://github.com/user-attachments/assets/3ff85561-33b6-4f40-8037-4e983d6e4057)


### Pros
* To be able to render parts of the query depending on whether indices
or fields exist in the cluster
* The queries become much easier to read, maintain and fix

### Cons
* We need to test the performance
* FORK is in tech preview
* The commands we can use in a fork are limited to “WHERE, LIMIT, SORT,
EVAL, STATS, DISSECT”

### How to test it?
* Open the dashboard without privmon data, some of the visualisations
should display the warning message
* Add privmon data, the visualisation should display the data
(https://github.com/elastic/security-documents-generator/pull/163)
* Check if the visualisation displays the correct data.
* To test if the FORK rewrite logic is working, I update the queries on
my local environment to use a non-existent field and update the page.


### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
This commit is contained in:
Pablo Machado 2025-06-24 10:39:03 +02:00 committed by GitHub
parent 0404a6d965
commit 85ba63638b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 598 additions and 150 deletions

View file

@ -6,8 +6,8 @@
*/
import { useIntervalForHeatmap } from './pad_heatmap_interval_hooks';
import { getPrivilegedMonitorUsersJoin } from '../../../../helpers';
import type { AnomalyBand } from '../pad_anomaly_bands';
import { getPrivilegedMonitorUsersJoin } from '../../../../queries/helpers';
const getHiddenBandsFilters = (anomalyBands: AnomalyBand[]) => {
const hiddenBands = anomalyBands.filter((each) => each.hidden);

View file

@ -138,38 +138,14 @@ describe('columns', () => {
expect(screen.getByText('Console')).toBeInTheDocument();
});
it('renders type column as "Direct" for /api/v1/authn*', () => {
it('renders type column', () => {
const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('/api/v1/authn/something', baseRecord)}</>, {
render(<>{col.render?.('Direct', baseRecord)}</>, {
wrapper: TestProviders,
});
expect(screen.getByText('Direct')).toBeInTheDocument();
});
it('renders type column as "Federated" for /oauth2/v1/authorize', () => {
const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('/oauth2/v1/authorize', baseRecord)}</>, { wrapper: TestProviders });
expect(screen.getByText('Federated')).toBeInTheDocument();
});
it('renders type column as "Federated" for /oauth2/v1/token', () => {
const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('/oauth2/v1/token', baseRecord)}</>, { wrapper: TestProviders });
expect(screen.getByText('Federated')).toBeInTheDocument();
});
it('renders type column as "Federated" for string containing /sso/saml', () => {
const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('/some/path/sso/saml', baseRecord)}</>, { wrapper: TestProviders });
expect(screen.getByText('Federated')).toBeInTheDocument();
});
it('renders type column as original value for unmatched string', () => {
const col = columns[4] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('/api/v1/authn', baseRecord)}</>, { wrapper: TestProviders });
expect(screen.getByText('Direct')).toBeInTheDocument();
});
it('renders result column with badge', () => {
const col = columns[5] as EuiTableFieldDataColumnType<TableItemType>;
render(<>{col.render?.('success', baseRecord)}</>, { wrapper: TestProviders });

View file

@ -221,27 +221,20 @@ export const buildAuthenticationsColumns = (
},
},
{
field: 'url',
field: 'type',
name: (
<FormattedMessage
id="xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type"
defaultMessage="Type"
/>
),
render: (url?: string) => {
if (!url) {
return getEmptyTagValue();
}
const type = getLoginTypeFromUrl(url);
render: (type?: string) => {
if (!type) {
return getEmptyTagValue();
}
return type;
},
truncateText: true,
},
// TODO Add the column depending on this ticket output https://github.com/elastic/security-team/issues/12713
// {
@ -294,25 +287,3 @@ const getResultColor = (value: string) => {
}
return 'default';
};
// TODO Verify if we can improve this logic https://github.com/elastic/security-team/issues/12713
const getLoginTypeFromUrl = (url: string) => {
if (url.startsWith('/api/v1/authn')) {
return i18n.translate(
'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type.direct',
{ defaultMessage: 'Direct' }
);
}
if (
url.startsWith('/oauth2/v1/authorize') ||
url.startsWith('/oauth2/v1/token') ||
url.includes('/sso/saml')
) {
return i18n.translate(
'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.columns.type.federated',
{ defaultMessage: 'Federated' }
);
}
return undefined;
};

View file

@ -1,49 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getPrivilegedMonitorUsersJoin } from '../../helpers';
export const getGrantedRightsEsqlSource = (namespace: string) => {
return `FROM logs-* METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| WHERE (host.os.type == "linux"
AND event.type == "start"
AND event.action IN ("exec", "exec_event", "start", "ProcessRollup2", "executed", "process_started")
AND (
process.name IN ("usermod", "adduser") OR
(process.name == "gpasswd" AND process.args IN ("-a", "--add", "-M", "--members"))
)) OR (
host.os.type=="windows"
AND event.action=="added-member-to-group"
) OR (
okta.event_type IN ("group.user_membership.add", "user.account.privilege.grant")
)
| EVAL okta_privilege = MV_FIRST(okta.target.display_name)
| EVAL group_name = COALESCE(group.name, user.target.group.name, okta_privilege)
| EVAL host_ip = COALESCE(host.ip, source.ip)
| EVAL target_user = COALESCE(user.target.name, user.target.full_name, winlog.event_data.TargetUserName)
| EVAL privileged_user = COALESCE(source.user.name, user.name)
| KEEP @timestamp, privileged_user, process.args, target_user, group_name, host_ip, _id, _index`;
};
export const getAccountSwitchesEsqlSource = (namespace: string) => {
return `FROM logs-* METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| WHERE process.command_line.caseless RLIKE "(su|sudo su|sudo -i|sudo -s|ssh [^@]+@[^\s]+)"
| RENAME process.command_line.caseless AS command_process, process.group_leader.user.name AS target_user, process.parent.real_group.name AS group_name, process.real_user.name as privileged_user, host.ip AS host_ip
| KEEP @timestamp, privileged_user, host_ip, target_user, group_name, command_process, _id, _index`;
};
export const getAuthenticationsEsqlSource = (namespace: string) => {
return `FROM logs-okta.system-* METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| RENAME source.ip AS host_ip, okta.target.display_name as destination, client.user.name as privileged_user, event.module as source, okta.debug_context.debug_data.url as url, okta.outcome.result as result
| WHERE privileged_user IS NOT NULL
| EVAL event_combined = COALESCE(event.action, okta.event_type, event.category)
| WHERE to_lower(event_combined) RLIKE ".*?(authn|authentication|sso|mfa|token\.grant|authorize\.code|session\.start|unauth_app_access_attempt|evaluate_sign_on|verify_push).*?"
| KEEP @timestamp, privileged_user, source, url, host_ip, result, destination, okta.authentication_context*, _id, _index`;
};

View file

@ -8,6 +8,7 @@
import React, { useMemo } from 'react';
import { FormattedMessage } from '@kbn/i18n-react';
import { useExpandableFlyoutApi } from '@kbn/expandable-flyout';
import type { DataViewSpec } from '@kbn/data-views-plugin/public';
import { useSpaceId } from '../../../../../common/hooks/use_space_id';
import {
generateListESQLQuery,
@ -20,16 +21,14 @@ import {
buildAuthenticationsColumns,
} from './columns';
import { getLensAttributes } from './get_lens_attributes';
import {
getAccountSwitchesEsqlSource,
getAuthenticationsEsqlSource,
getGrantedRightsEsqlSource,
} from './esql_source_query';
import {
ACCOUNT_SWITCH_STACK_BY,
AUTHENTICATIONS_STACK_BY,
GRANTED_RIGHTS_STACK_BY,
} from './constants';
import { getAuthenticationsEsqlSource } from '../../queries/authentications_esql_query';
import { getAccountSwitchesEsqlSource } from '../../queries/account_switches_esql_query';
import { getGrantedRightsEsqlSource } from '../../queries/granted_rights_esql_query';
const toggleOptionsConfig = {
[VisualizationToggleOptions.GRANTED_RIGHTS]: {
@ -50,13 +49,24 @@ const toggleOptionsConfig = {
};
export const usePrivilegedUserActivityParams = (
selectedToggleOption: VisualizationToggleOptions
selectedToggleOption: VisualizationToggleOptions,
sourcererDataView: DataViewSpec
) => {
const spaceId = useSpaceId();
const indexPattern = sourcererDataView?.title ?? '';
const fields = sourcererDataView?.fields;
const esqlSource = useMemo(
() =>
spaceId ? toggleOptionsConfig[selectedToggleOption].generateEsqlSource(spaceId) : undefined,
[selectedToggleOption, spaceId]
spaceId && indexPattern && fields
? toggleOptionsConfig[selectedToggleOption].generateEsqlSource(
spaceId,
indexPattern,
fields
)
: undefined,
[selectedToggleOption, spaceId, indexPattern, fields]
);
const generateTableQuery = useMemo(
@ -75,11 +85,17 @@ export const usePrivilegedUserActivityParams = (
[selectedToggleOption, openRightPanel]
);
const hasLoadedDependencies = useMemo(
() => Boolean(spaceId && indexPattern && fields),
[spaceId, indexPattern, fields]
);
return {
getLensAttributes,
generateVisualizationQuery,
generateTableQuery,
columns,
hasLoadedDependencies,
};
};

View file

@ -28,19 +28,33 @@ jest.mock('../../../../../common/hooks/use_space_id', () => ({
useSpaceId: jest.fn().mockReturnValue('default'),
}));
const mockedSourcererDataView = {
title: 'test-*',
fields: {},
};
describe('UserActivityPrivilegedUsersPanel', () => {
it('renders panel title', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
expect(screen.getByText('Privileged user activity')).toBeInTheDocument();
});
it('renders the toggle button group', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
expect(screen.getByRole('group', { name: /ABOUT_CONTROL_LEGEND/i })).toBeInTheDocument();
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
expect(
screen.getByRole('group', { name: /Select a visualization to display/i })
).toBeInTheDocument();
});
it('renders the stack by select with options', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
expect(screen.getByText('Stack by')).toBeInTheDocument();
expect(screen.getByRole('option', { name: 'Privileged user' })).toBeInTheDocument();
expect(screen.getByRole('option', { name: 'Target user' })).toBeInTheDocument();
@ -48,19 +62,30 @@ describe('UserActivityPrivilegedUsersPanel', () => {
});
it('renders the EsqlDashboardPanel', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
// select a visualization that doesn't require dataview fields
fireEvent.click(screen.getByTestId('account_switches'));
expect(screen.getByTestId('esql-dashboard-panel')).toBeInTheDocument();
});
it('changes stack by option when select changes', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
const select = screen.getByRole('combobox');
fireEvent.change(select, { target: { value: 'group_name' } });
expect((select as HTMLSelectElement).value).toBe('group_name');
});
it('renders the "View all events by privileged users" link', () => {
render(<UserActivityPrivilegedUsersPanel />, { wrapper: TestProviders });
render(<UserActivityPrivilegedUsersPanel sourcererDataView={mockedSourcererDataView} />, {
wrapper: TestProviders,
});
// select a visualization that doesn't require dataview fields
fireEvent.click(screen.getByTestId('account_switches'));
expect(screen.getByText('View all events')).toBeInTheDocument();
});
});

View file

@ -7,6 +7,7 @@
import {
EuiButtonGroup,
EuiCallOut,
EuiFlexGroup,
EuiFlexItem,
EuiPanel,
@ -17,6 +18,7 @@ import React, { useCallback, useState } from 'react';
import { i18n } from '@kbn/i18n';
import { useNavigation } from '@kbn/security-solution-navigation';
import { FormattedMessage } from '@kbn/i18n-react';
import type { DataViewSpec } from '@kbn/data-views-plugin/public';
import { useGlobalTime } from '../../../../../common/containers/use_global_time';
import { useQueryToggle } from '../../../../../common/containers/query_toggle';
import { LinkAnchor } from '../../../../../common/components/links';
@ -27,21 +29,33 @@ import { usePrivilegedUserActivityParams, useStackByOptions, useToggleOptions }
import type { TableItemType } from './types';
import { VisualizationToggleOptions } from './types';
const PICK_VISUALIZATION_LEGEND = i18n.translate(
'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.pickVisualizationLegend',
{ defaultMessage: 'Select a visualization to display' }
);
const TITLE = i18n.translate(
'xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.title',
{ defaultMessage: 'Privileged user activity' }
);
export const UserActivityPrivilegedUsersPanel: React.FC = () => {
export const UserActivityPrivilegedUsersPanel: React.FC<{
sourcererDataView: DataViewSpec;
}> = ({ sourcererDataView }) => {
const { toggleStatus, setToggleStatus } = useQueryToggle(PRIVILEGED_USER_ACTIVITY_QUERY_ID);
const { from, to } = useGlobalTime();
const [selectedToggleOption, setToggleOption] = useState<VisualizationToggleOptions>(
VisualizationToggleOptions.GRANTED_RIGHTS
);
const { getLensAttributes, columns, generateVisualizationQuery, generateTableQuery } =
usePrivilegedUserActivityParams(selectedToggleOption);
const stackByOptions = useStackByOptions(selectedToggleOption);
const {
getLensAttributes,
columns,
generateVisualizationQuery,
generateTableQuery,
hasLoadedDependencies,
} = usePrivilegedUserActivityParams(selectedToggleOption, sourcererDataView);
const stackByOptions = useStackByOptions(selectedToggleOption);
const setSelectedChartOptionCallback = useCallback(
(event: React.ChangeEvent<HTMLSelectElement>) => {
setSelectedStackByOption(
@ -92,7 +106,7 @@ export const UserActivityPrivilegedUsersPanel: React.FC = () => {
setToggleOption(id as VisualizationToggleOptions);
setSelectedStackByOption(defaultStackByOption);
}}
legend={'ABOUT_CONTROL_LEGEND'}
legend={PICK_VISUALIZATION_LEGEND}
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
@ -109,7 +123,7 @@ export const UserActivityPrivilegedUsersPanel: React.FC = () => {
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size="m" />
{generateVisualizationQuery && generateTableQuery && (
{generateVisualizationQuery && generateTableQuery ? (
<EsqlDashboardPanel<TableItemType>
title={TITLE}
stackByField={selectedStackByOption.value}
@ -121,6 +135,25 @@ export const UserActivityPrivilegedUsersPanel: React.FC = () => {
pageSize={PAGE_SIZE}
showInspectTable={true}
/>
) : (
// If dependencies are loaded but the query generation functions are not available, show an error message
hasLoadedDependencies && (
<EuiCallOut
title={
<FormattedMessage
id="xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.missingMappings.errorTitle"
defaultMessage="There was a problem rendering the visualization"
/>
}
color="warning"
iconType="error"
>
<FormattedMessage
id="xpack.securitySolution.entityAnalytics.privilegedUserMonitoring.userActivity.missingMappings.errorMessage"
defaultMessage="The required fields are not present in the data view."
/>
</EuiCallOut>
)
)}
</>
)}

View file

@ -19,7 +19,7 @@ import type { RiskLevelsTableItem, RiskLevelsPrivilegedUsersQueryResult } from '
import { RiskScoreLevel } from '../../../severity/common';
import type { RiskSeverity } from '../../../../../../common/search_strategy';
import { esqlResponseToRecords } from '../../../../../common/utils/esql';
import { getRiskLevelsPrivilegedUsersQueryBody } from './esql_query';
import { getRiskLevelsPrivilegedUsersQueryBody } from '../../queries/risk_level_esql_query';
export const useRiskLevelsPrivilegedUserQuery = ({
skip,

View file

@ -23,7 +23,7 @@ import { HeaderSection } from '../../../../../common/components/header_section';
import { InspectButtonContainer } from '../../../../../common/components/inspect';
import { SEVERITY_UI_SORT_ORDER } from '../../../../common';
import { useRiskScoreFillColor } from '../../../risk_score_donut_chart/use_risk_score_fill_color';
import { DONUT_CHART_HEIGHT, RISK_LEVELS_PRIVILEGED_USERS_QUERY_ID } from './esql_query';
import { RISK_LEVELS_PRIVILEGED_USERS_QUERY_ID } from '../../queries/risk_level_esql_query';
import { useRiskLevelsPrivilegedUserQuery, useRiskLevelsTableColumns } from './hooks';
const TITLE = i18n.translate(
@ -31,6 +31,8 @@ const TITLE = i18n.translate(
{ defaultMessage: 'Risk levels of privileged users' }
);
export const DONUT_CHART_HEIGHT = 160;
export const RiskLevelsPrivilegedUsersPanel: React.FC<{ spaceId: string }> = ({ spaceId }) => {
const fillColor = useRiskScoreFillColor();
const { toggleStatus, setToggleStatus } = useQueryToggle(RISK_LEVELS_PRIVILEGED_USERS_QUERY_ID);

View file

@ -1,15 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getPrivilegedMonitorUsersIndex } from '../../../../common/entity_analytics/privilege_monitoring/constants';
export const getPrivilegedMonitorUsersJoin = (
namespace: string
) => `| RENAME @timestamp AS event_timestamp
| LOOKUP JOIN ${getPrivilegedMonitorUsersIndex(namespace)} ON user.name
| RENAME event_timestamp AS @timestamp
| WHERE user.is_privileged == true`;

View file

@ -8,6 +8,7 @@
import { EuiButton, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui';
import React, { useCallback, useState } from 'react';
import { FormattedMessage } from '@kbn/i18n-react';
import type { DataViewSpec } from '@kbn/data-views-plugin/public';
import { useSpaceId } from '../../../common/hooks/use_space_id';
import { RiskLevelsPrivilegedUsersPanel } from './components/risk_level_panel';
import { UserActivityPrivilegedUsersPanel } from './components/privileged_user_activity';
@ -20,9 +21,11 @@ export interface OnboardingCallout {
export const PrivilegedUserMonitoring = ({
callout,
onManageUserClicked,
sourcererDataView,
}: {
callout?: OnboardingCallout;
onManageUserClicked: () => void;
sourcererDataView: DataViewSpec;
}) => {
const spaceId = useSpaceId();
const [dismissCallout, setDismissCallout] = useState(false);
@ -85,7 +88,7 @@ export const PrivilegedUserMonitoring = ({
</EuiFlexItem>
{spaceId && <PrivilegedAccessDetectionsPanel spaceId={spaceId} />}
<EuiFlexItem>
<UserActivityPrivilegedUsersPanel />
<UserActivityPrivilegedUsersPanel sourcererDataView={sourcererDataView} />
</EuiFlexItem>
<EuiFlexItem>
<EuiPanel hasShadow={false} hasBorder={true}>

View file

@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataViewFieldMap } from '@kbn/data-views-plugin/common';
import { getPrivilegedMonitorUsersJoin } from './helpers';
export const getAccountSwitchesEsqlSource = (
namespace: string,
indexPattern: string,
fields: DataViewFieldMap
) => {
return `FROM ${indexPattern} METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| WHERE to_lower(process.command_line) RLIKE "(su|sudo su|sudo -i|sudo -s|ssh [^@]+@[^\s]+)"
| RENAME to_lower(process.command_line) AS command_process, process.group_leader.user.name AS target_user, process.parent.real_group.name AS group_name, process.real_user.name as privileged_user, host.ip AS host_ip
| KEEP @timestamp, privileged_user, host_ip, target_user, group_name, command_process, _id, _index`;
};

View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataViewFieldMap } from '@kbn/data-views-plugin/common';
import { getPrivilegedMonitorUsersJoin, removeInvalidForkBranchesFromESQL } from './helpers';
// TODO add test cases for okta type column logic
// '/api/v1/authn/something' ===> Direct
// /oauth2/v1/authorize' ===> Federated
// /oauth2/v1/token' ===> Federated
// /some/path/sso/saml' ===> Federated
// /api/v1/authn' ===> Direct
// TODO Verify if we can improve the type field logic https://github.com/elastic/security-team/issues/12713
export const getAuthenticationsEsqlSource = (
namespace: string,
indexPattern: string,
fields: DataViewFieldMap
) =>
removeInvalidForkBranchesFromESQL(
fields,
`FROM ${indexPattern} METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| WHERE user.name IS NOT NULL
| FORK
(
WHERE event.dataset == "okta.system"
| EVAL event_combined = COALESCE(event.action, okta.event_type, event.category)
| WHERE to_lower(event_combined) RLIKE ".*?(authn|authentication|sso|mfa|token.grant|authorize.code|session.start|unauth_app_access_attempt|evaluate_sign_on|verify_push).*?"
| EVAL result = okta.outcome.result
| EVAL destination = okta.target.display_name
| EVAL source = event.module
| EVAL host_ip = source.ip
| EVAL url = okta.debug_context.debug_data.url
| EVAL type = CASE(
STARTS_WITH(url, "/api/v1/authn"), "Direct",
STARTS_WITH(url, "/oauth2/v1/authorize") OR STARTS_WITH(url, "/oauth2/v1/token") OR
LOCATE(url, "/sso/saml") > 0, "Federated",
null)
)
(
WHERE event.dataset != "okta.system" AND event.category == "authentication"
| EVAL result = event.outcome
| EVAL source = host.os.name
| EVAL type = "Direct"
| EVAL destination = host.name
| EVAL host_ip = host.ip
)
| RENAME user.name as privileged_user
| KEEP @timestamp, privileged_user, source, host_ip, result, destination, _id, _index, event.outcome, type`
);

View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataViewFieldMap } from '@kbn/data-views-plugin/common';
import { getPrivilegedMonitorUsersJoin, removeInvalidForkBranchesFromESQL } from './helpers';
export const getGrantedRightsEsqlSource = (
namespace: string,
indexPattern: string,
fields: DataViewFieldMap
) =>
removeInvalidForkBranchesFromESQL(
fields,
`FROM ${indexPattern} METADATA _id, _index
${getPrivilegedMonitorUsersJoin(namespace)}
| FORK
(
WHERE event.dataset == "okta.system" AND okta.event_type IN ("group.user_membership.add", "user.account.privilege.grant")
| EVAL group_name = MV_FIRST(okta.target.display_name)
| EVAL host_ip = source.ip
| EVAL target_user = user.target.full_name
| EVAL privileged_user = COALESCE(source.user.name, user.name)
)
(
WHERE (host.os.type == "linux"
AND event.type == "start"
AND event.action IN ("exec", "exec_event", "start", "ProcessRollup2", "executed", "process_started")
AND (
process.name IN ("usermod", "adduser") OR
(process.name == "gpasswd" AND process.args IN ("-a", "--add", "-M", "--members"))
)) OR (
host.os.type=="windows" AND event.action=="added-member-to-group"
)
| EVAL group_name = COALESCE(group.name, user.target.group.name)
| EVAL host_ip = host.ip
| EVAL target_user = COALESCE(user.target.name, user.target.full_name, winlog.event_data.TargetUserName)
| EVAL privileged_user = user.name
)
| KEEP @timestamp, privileged_user, target_user, group_name, host_ip, _id, _index`
);

View file

@ -0,0 +1,212 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { DataViewFieldMap } from '@kbn/data-views-plugin/common';
import { getPrivilegedMonitorUsersJoin, removeInvalidForkBranchesFromESQL } from './helpers';
describe('getPrivilegedMonitorUsersJoin', () => {
it('should return the correct ESQL join string with the given namespace', () => {
const namespace = 'default';
const result = getPrivilegedMonitorUsersJoin(namespace);
expect(result).toMatchInlineSnapshot(`
"| RENAME @timestamp AS event_timestamp
| LOOKUP JOIN .entity_analytics.monitoring.users-default ON user.name
| RENAME event_timestamp AS @timestamp
| WHERE user.is_privileged == true"
`);
});
});
describe('removeInvalidForkBranchesFromESQL', () => {
const fields: DataViewFieldMap = {
foo: {
name: 'foo',
type: 'number',
esTypes: ['long'],
count: 10,
searchable: true,
aggregatable: true,
},
bar: {
name: 'bar',
type: 'number',
esTypes: ['long'],
count: 10,
searchable: true,
aggregatable: true,
},
};
beforeEach(() => {
jest.clearAllMocks();
});
it('should return the original esql if there is no fork command', () => {
const esql = 'FROM test-index | EVAL new_field=foo+bar';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toBe(esql);
});
it('should throw if fork command has less than two arguments', () => {
const esql = 'FROM test-index | FORK (WHERE foo IS NULL)';
expect(() => removeInvalidForkBranchesFromESQL(fields, esql)).toThrow(
'Invalid ESQL query: FORK command must have at least two arguments'
);
});
it('should throw if there are more than one fork command in the query', () => {
const esql =
'FROM test-index | FORK (WHERE foo IS NULL) (WHERE bar IS NULL) | FORK (WHERE foo IS NULL) (WHERE bar IS NULL)';
expect(() => removeInvalidForkBranchesFromESQL(fields, esql)).toThrow(
'removeInvalidForkBranchesFromESQL does not support Multiple FORK commands'
);
});
it('should return undefined if all branches are invalid', () => {
const esql = 'FROM test-index | FORK (WHERE not_a_field IS NULL) (WHERE not_a_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toBeUndefined();
});
it('should remove fork and insert valid branch into root if only one valid branch exists', () => {
const esql = 'FROM test-index | FORK (WHERE foo IS NULL) (WHERE not_a_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| WHERE foo IS NULL"
`);
});
it('should remove invalid branches and return FORK query if multiple valid branches exist', () => {
const esql =
'FROM test-index | FORK (WHERE foo IS NULL) (WHERE bar IS NULL) (WHERE not_a_field IS NULL)';
const result = removeInvalidForkBranchesFromESQL(fields, esql);
expect(result).toMatchInlineSnapshot(`
"FROM test-index
| FORK
(WHERE foo IS NULL)
(WHERE bar IS NULL)"
`);
});
it('should return the original esql if all branches are valid', () => {
const esql = 'FROM test-index | FORK (WHERE foo IS NULL) (WHERE bar IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toBe(esql);
});
it('should remove fork if the invalid field is present inside a SORT command', () => {
const esql = 'FROM test-index | FORK (WHERE foo IS NULL) (SORT not_a_field)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| WHERE foo IS NULL"
`);
});
// Fix The ESQL walker doesn't enter the sort "order" node for some reason
// This scenario will cause an error if the query sorts by a invalid field that was not present anywhere else
// it('should remove fork if the invalid field is present inside a SORT command with order', () => {
// const esql = 'FROM test-index | FORK (SORT foo) (SORT not_a_field ASC)';
// expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
// "FROM test-index
// | WHERE foo IS NULL"
// `);
// });
it('should remove fork if the invalid field is present inside a WHERE command', () => {
const esql = 'FROM test-index | FORK (WHERE foo IS NULL) (WHERE not_a_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| WHERE foo IS NULL"
`);
});
it('should remove fork if the invalid field is present inside a STATS command', () => {
const esql = 'FROM test-index | FORK (STATS AVG(foo)) (STATS AVG(not_a_field))';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| STATS AVG(foo)"
`);
});
it('should remove fork if the invalid field is present inside a DISSECT command', () => {
const esql =
'FROM test-index | FORK (DISSECT foo "%{a}-%{b}") (DISSECT not_a_field "%{a}-%{b}")';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| DISSECT foo \\"%{a}-%{b}\\""
`);
});
it('should not remove fork if an EVAL commands created a new field inside a branch', () => {
const esql =
'FROM test-index | FORK (WHERE foo IS NULL) (EVAL new_field = foo | WHERE new_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an previous EVAL commands created a new field with an complex expression', () => {
const esql = `FROM test-index | FORK
(WHERE foo IS NULL)
(EVAL new_field =
CASE(
STARTS_WITH(foo, "/api/v1/authn"), "Direct",
STARTS_WITH(bar, "/oauth2/v1/authorize") OR STARTS_WITH(bar, "/oauth2/v1/token") OR LOCATE(bar, "/sso/saml") > 0, "Federated",
null)
| WHERE new_field IS NULL)`;
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should remove fork if an EVAL command uses an invalid field inside a complex expression', () => {
const esql = `FROM test-index | FORK
(WHERE foo IS NULL)
(EVAL new_field =
CASE(
STARTS_WITH(foo, "/api/v1/authn"), "Direct",
STARTS_WITH(bar, "/oauth2/v1/authorize") OR STARTS_WITH(bar, "/oauth2/v1/token") OR LOCATE(invalid_Field, "/sso/saml") > 0, "Federated",
null)
)`;
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toMatchInlineSnapshot(`
"FROM test-index
| WHERE foo IS NULL"
`);
});
it('should not remove fork if an previous EVAL commands created a new field', () => {
const esql =
'FROM test-index | EVAL new_field = foo | FORK (WHERE foo IS NULL) (WHERE new_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an previous EVAL commands created a new field with multiple assignments', () => {
const esql =
'FROM test-index | EVAL new_field1 = foo, new_field2 = bar | FORK (WHERE foo IS NULL) (WHERE new_field2 IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an field was renamed', () => {
const esql =
'FROM test-index | RENAME foo as new_field | FORK (WHERE foo IS NULL) (WHERE new_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an field was renamed with a multiple renamed assignments', () => {
const esql =
'FROM test-index | RENAME foo as new_field1, foo as new_field2 | FORK (WHERE foo IS NULL) (WHERE new_field2 IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an field was renamed with new syntax', () => {
const esql =
'FROM test-index | RENAME new_field = foo | FORK (WHERE foo IS NULL) (WHERE new_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
it('should not remove fork if an field was renamed with new syntax and expression', () => {
const esql =
'FROM test-index | RENAME new_field = foo + 1 | FORK (WHERE foo IS NULL) (WHERE new_field IS NULL)';
expect(removeInvalidForkBranchesFromESQL(fields, esql)).toEqual(esql);
});
});

View file

@ -0,0 +1,155 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ESQLAstQueryExpression, ESQLCommand } from '@kbn/esql-ast';
import { Walker, BasicPrettyPrinter, isFunctionExpression, isColumn, mutate } from '@kbn/esql-ast';
import type { DataViewFieldMap } from '@kbn/data-views-plugin/common';
import { partition } from 'lodash/fp';
import type { ESQLProperNode } from '@kbn/esql-ast/src/types';
import { Parser } from '@kbn/esql-ast/src/parser/parser';
import { isAsExpression, isFieldExpression } from '@kbn/esql-ast/src/ast/helpers';
import { getPrivilegedMonitorUsersIndex } from '../../../../../common/entity_analytics/privilege_monitoring/constants';
export const getPrivilegedMonitorUsersJoin = (
namespace: string
) => `| RENAME @timestamp AS event_timestamp
| LOOKUP JOIN ${getPrivilegedMonitorUsersIndex(namespace)} ON user.name
| RENAME event_timestamp AS @timestamp
| WHERE user.is_privileged == true`;
/**
* Rewrites que query to remove FORK branches that contain columns not available.
*/
export function removeInvalidForkBranchesFromESQL(fields: DataViewFieldMap, esql: string) {
const { root } = Parser.parse(esql);
const forkCommands = Walker.findAll(root, (node) => node.name === 'fork') as Array<
ESQLCommand<'fork'>
>;
// The query has no FORK command, so we can return the original ESQL query
if (forkCommands.length === 0) {
return esql;
}
// There is no technical limitation preventing us from having multiple FORK commands in the query,
// but the current implementation only supports a single FORK command.
if (forkCommands.length > 1) {
throw new Error('removeInvalidForkBranchesFromESQL does not support Multiple FORK commands');
}
const forkCommand = forkCommands[0];
const forkArguments = forkCommand?.args as ESQLAstQueryExpression[];
if (!forkArguments || forkArguments.length < 2) {
throw new Error('Invalid ESQL query: FORK command must have at least two arguments');
}
// Columns create by the EVAL and RENAME command
const createdColumns = getAllCreatedColumns(root);
const isInvalidColumn = (node: ESQLProperNode) =>
isColumn(node) && !createdColumns.includes(node.name) && !fields[node.name]; // Check if the column was created or exists in the fields map
const [invalidBranches, validBranches] = partition(
(forkArgument) => Walker.find(forkArgument, isInvalidColumn),
forkArguments
);
// When all branches are valid we can return the original ESQL query
if (invalidBranches.length === 0) {
return esql;
}
// No valid FORK branches found
if (validBranches.length === 0) {
return undefined; // TODO can we throw an error here? or return an empty query?
}
// When FORK has only one valid branch we need to remove the fork command from query and add the valid branch back to the root
if (validBranches.length === 1) {
return moveForkBranchToToplevel(root, forkCommand, validBranches[0]);
}
// Remove the invalid branches
invalidBranches.forEach((branch) => {
mutate.generic.commands.args.remove(root, branch);
});
return BasicPrettyPrinter.multiline(root);
}
function moveForkBranchToToplevel(
root: ESQLAstQueryExpression,
forkCommand: ESQLCommand<'fork'>,
validBranch: ESQLAstQueryExpression
) {
mutate.generic.commands.remove(root, forkCommand);
// Find where the fork index is to insert the valid branch
const forkIndex = root.commands.findIndex((cmd) => cmd.name === 'fork');
validBranch.commands.reverse().forEach((command) => {
mutate.generic.commands.insert(root, command, forkIndex);
});
return BasicPrettyPrinter.multiline(root);
}
function getAllCreatedColumns(root: ESQLAstQueryExpression) {
const evalCommands = Walker.findAll(root, (node) => node.name === 'eval') as Array<
ESQLCommand<'eval'>
>;
// Columns create by the EVAL command
// Syntax: | EVAL new_column = column
const evalColumns = evalCommands
.map((command) => {
return command.args.map((arg) => {
if (isFunctionExpression(arg) && isColumn(arg.args[0])) {
return arg.args[0].name;
}
return null;
});
})
.flat();
const renameCommands = Walker.findAll(root, (node) => node.name === 'rename') as Array<
ESQLCommand<'rename'>
>;
// Columns create by the RENAME command
// Syntaxes:
// 1. | RENAME column AS new_column, column2 AS new_column2
// 2. | RENAME new_column = column, new_column2 = column2 (9.1+)
const renamedColumns = renameCommands
.map((command) => {
return command.args.map((arg) => {
if (isAsExpression(arg)) {
if (isColumn(arg.args[1])) {
return arg.args[1].name;
}
}
if (isFieldExpression(arg)) {
if (isColumn(arg.args[0])) {
return arg.args[0].name;
}
}
return null;
});
})
.flat();
// Here we get all created columns from EVAL and RENAME commands
// We don't care where they are located, we just need to know which columns are available in the query
// If a column is used on a place where it isn't available ESQL will handle the error
const createdColumns = [...evalColumns, ...renamedColumns].filter(
(column): column is string => column !== null
);
return createdColumns;
}

View file

@ -5,8 +5,8 @@
* 2.0.
*/
import { RiskScoreFields } from '../../../../../../common/search_strategy';
import { getPrivilegedMonitorUsersJoin } from '../../helpers';
import { RiskScoreFields } from '../../../../../common/search_strategy';
import { getPrivilegedMonitorUsersJoin } from './helpers';
export const getRiskLevelsPrivilegedUsersQueryBody = (namespace: string) => `
| WHERE ${RiskScoreFields.userName} IS NOT NULL
@ -15,5 +15,3 @@ ${getPrivilegedMonitorUsersJoin(namespace)}
| RENAME ${RiskScoreFields.userRisk} AS level`;
export const RISK_LEVELS_PRIVILEGED_USERS_QUERY_ID = 'risk_levels_privileged_users';
export const DONUT_CHART_HEIGHT = 160;

View file

@ -214,6 +214,7 @@ export const EntityAnalyticsPrivilegedUserMonitoringPage = () => {
<PrivilegedUserMonitoring
callout={state.onboardingCallout}
onManageUserClicked={onManageUserClicked}
sourcererDataView={sourcererDataView}
/>
</>
)}