[8.16] [Security Solution][Data Quality Dashboard][Serverless] add start/end time support for latest_results (#199248) (#200635)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[Security Solution][Data Quality Dashboard][Serverless] add start/end
time support for latest_results
(#199248)](https://github.com/elastic/kibana/pull/199248)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Karen
Grigoryan","email":"karen.grigoryan@elastic.co"},"sourceCommit":{"committedDate":"2024-11-07T17:42:31Z","message":"[Security
Solution][Data Quality Dashboard][Serverless] add start/end time support
for latest_results (#199248)\n\naddresses #191053\r\n\r\n- Introduce
`defaultStartTime` and `defaultEndTime` props across data\r\nquality
context and panels for fetching latest_results and align them\r\nwith
serverless default time range of last week\r\n- Update hooks to handle
new time parameters and include them in storage\r\nresults queries.\r\n-
Modify server-side helpers and routes to process and filter
indices\r\nbased on the provided time range.\r\n- Update related tests
to accommodate the new time
parameters.","sha":"1df04aef8d5859507c85a2ad37594075e9054b70","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Threat
Hunting","Team:Threat
Hunting:Explore","backport:prev-minor","ci:cloud-deploy","ci:project-deploy-security","v8.17.0"],"number":199248,"url":"https://github.com/elastic/kibana/pull/199248","mergeCommit":{"message":"[Security
Solution][Data Quality Dashboard][Serverless] add start/end time support
for latest_results (#199248)\n\naddresses #191053\r\n\r\n- Introduce
`defaultStartTime` and `defaultEndTime` props across data\r\nquality
context and panels for fetching latest_results and align them\r\nwith
serverless default time range of last week\r\n- Update hooks to handle
new time parameters and include them in storage\r\nresults queries.\r\n-
Modify server-side helpers and routes to process and filter
indices\r\nbased on the provided time range.\r\n- Update related tests
to accommodate the new time
parameters.","sha":"1df04aef8d5859507c85a2ad37594075e9054b70"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199248","number":199248,"mergeCommit":{"message":"[Security
Solution][Data Quality Dashboard][Serverless] add start/end time support
for latest_results (#199248)\n\naddresses #191053\r\n\r\n- Introduce
`defaultStartTime` and `defaultEndTime` props across data\r\nquality
context and panels for fetching latest_results and align them\r\nwith
serverless default time range of last week\r\n- Update hooks to handle
new time parameters and include them in storage\r\nresults queries.\r\n-
Modify server-side helpers and routes to process and filter
indices\r\nbased on the provided time range.\r\n- Update related tests
to accommodate the new time
parameters.","sha":"1df04aef8d5859507c85a2ad37594075e9054b70"}},{"branch":"8.x","label":"v8.17.0","labelRegex":"^v8.17.0$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/199385","number":199385,"state":"MERGED","mergeCommit":{"sha":"de09e3af76d4bd1ce029830bd866e20b46e3aa9e","message":"[8.x]
[Security Solution][Data Quality Dashboard][Serverless] add start/end
time support for latest_results (#199248) (#199385)\n\n#
Backport\n\nThis will backport the following commits from `main` to
`8.x`:\n- [[Security Solution][Data Quality Dashboard][Serverless] add
start/end\ntime support for
latest_results\n(#199248)](https://github.com/elastic/kibana/pull/199248)\n\n<!---
Backport version: 9.4.3 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Karen\nGrigoryan\",\"email\":\"karen.grigoryan@elastic.co\"},\"sourceCommit\":{\"committedDate\":\"2024-11-07T17:42:31Z\",\"message\":\"[Security\nSolution][Data
Quality Dashboard][Serverless] add start/end time support\nfor
latest_results (#199248)\\n\\naddresses #191053\\r\\n\\r\\n-
Introduce\n`defaultStartTime` and `defaultEndTime` props across
data\\r\\nquality\ncontext and panels for fetching latest_results and
align them\\r\\nwith\nserverless default time range of last week\\r\\n-
Update hooks to handle\nnew time parameters and include them in
storage\\r\\nresults queries.\\r\\n-\nModify server-side helpers and
routes to process and filter\nindices\\r\\nbased on the provided time
range.\\r\\n- Update related tests\nto accommodate the new
time\nparameters.\",\"sha\":\"1df04aef8d5859507c85a2ad37594075e9054b70\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.17.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:skip\",\"v9.0.0\",\"Team:Threat\nHunting\",\"Team:Threat\nHunting:Explore\",\"backport:prev-minor\",\"ci:cloud-deploy\",\"ci:project-deploy-security\"],\"title\":\"[Security\nSolution][Data
Quality Dashboard][Serverless] add start/end time
support\nfor\nlatest_results\",\"number\":199248,\"url\":\"https://github.com/elastic/kibana/pull/199248\",\"mergeCommit\":{\"message\":\"[Security\nSolution][Data
Quality Dashboard][Serverless] add start/end time support\nfor
latest_results (#199248)\\n\\naddresses #191053\\r\\n\\r\\n-
Introduce\n`defaultStartTime` and `defaultEndTime` props across
data\\r\\nquality\ncontext and panels for fetching latest_results and
align them\\r\\nwith\nserverless default time range of last week\\r\\n-
Update hooks to handle\nnew time parameters and include them in
storage\\r\\nresults queries.\\r\\n-\nModify server-side helpers and
routes to process and filter\nindices\\r\\nbased on the provided time
range.\\r\\n- Update related tests\nto accommodate the new
time\nparameters.\",\"sha\":\"1df04aef8d5859507c85a2ad37594075e9054b70\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"branchLabelMappingKey\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/199248\",\"number\":199248,\"mergeCommit\":{\"message\":\"[Security\nSolution][Data
Quality Dashboard][Serverless] add start/end time support\nfor
latest_results (#199248)\\n\\naddresses #191053\\r\\n\\r\\n-
Introduce\n`defaultStartTime` and `defaultEndTime` props across
data\\r\\nquality\ncontext and panels for fetching latest_results and
align them\\r\\nwith\nserverless default time range of last week\\r\\n-
Update hooks to handle\nnew time parameters and include them in
storage\\r\\nresults queries.\\r\\n-\nModify server-side helpers and
routes to process and filter\nindices\\r\\nbased on the provided time
range.\\r\\n- Update related tests\nto accommodate the new
time\nparameters.\",\"sha\":\"1df04aef8d5859507c85a2ad37594075e9054b70\"}}]}]\nBACKPORT-->\n\nCo-authored-by:
Karen Grigoryan <karen.grigoryan@elastic.co>"}}]}] BACKPORT-->
This commit is contained in:
Karen Grigoryan 2024-11-18 23:57:22 +01:00 committed by GitHub
parent c8b46e87c4
commit b900fa0cab
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 642 additions and 58 deletions

View file

@ -65,6 +65,8 @@ const ContextWrapper: FC<PropsWithChildren<unknown>> = ({ children }) => (
},
]}
setSelectedIlmPhaseOptions={jest.fn()}
defaultStartTime="now-7d"
defaultEndTime="now"
>
{children}
</DataQualityProvider>

View file

@ -41,6 +41,8 @@ export interface DataQualityProviderProps {
ilmPhases: string[];
selectedIlmPhaseOptions: EuiComboBoxOptionOption[];
setSelectedIlmPhaseOptions: (options: EuiComboBoxOptionOption[]) => void;
defaultStartTime: string;
defaultEndTime: string;
}
const DataQualityContext = React.createContext<DataQualityProviderProps | undefined>(undefined);
@ -67,6 +69,8 @@ export const DataQualityProvider: React.FC<PropsWithChildren<DataQualityProvider
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
}) => {
const value = useMemo(
() => ({
@ -90,6 +94,8 @@ export const DataQualityProvider: React.FC<PropsWithChildren<DataQualityProvider
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
}),
[
httpFetch,
@ -112,6 +118,8 @@ export const DataQualityProvider: React.FC<PropsWithChildren<DataQualityProvider
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
]
);

View file

@ -71,6 +71,8 @@ const ContextWrapper: React.FC<{ children: React.ReactNode; isILMAvailable: bool
},
]}
setSelectedIlmPhaseOptions={jest.fn()}
defaultStartTime={'now-7d'}
defaultEndTime={'now'}
>
{children}
</DataQualityProvider>
@ -159,6 +161,8 @@ describe('useIlmExplain', () => {
},
]}
setSelectedIlmPhaseOptions={jest.fn()}
defaultStartTime={'now-7d'}
defaultEndTime={'now'}
>
{children}
</DataQualityProvider>

View file

@ -69,6 +69,8 @@ const ContextWrapper: FC<PropsWithChildren<unknown>> = ({ children }) => (
},
]}
setSelectedIlmPhaseOptions={jest.fn()}
defaultStartTime={'now-7d'}
defaultEndTime={'now'}
>
{children}
</DataQualityProvider>
@ -119,6 +121,8 @@ const ContextWrapperILMNotAvailable: FC<PropsWithChildren<unknown>> = ({ childre
},
]}
setSelectedIlmPhaseOptions={jest.fn()}
defaultStartTime={'now-7d'}
defaultEndTime={'now'}
>
{children}
</DataQualityProvider>

View file

@ -11,6 +11,10 @@ import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
import { getHistoricalResultStub } from '../../../../stub/get_historical_result_stub';
import { useStoredPatternResults } from '.';
const startTime = 'now-7d';
const endTime = 'now';
const isILMAvailable = true;
describe('useStoredPatternResults', () => {
const httpFetch = jest.fn();
const mockToasts = notificationServiceMock.createStartContract().toasts;
@ -21,7 +25,16 @@ describe('useStoredPatternResults', () => {
describe('when patterns are empty', () => {
it('should return an empty array and not call getStorageResults', () => {
const { result } = renderHook(() => useStoredPatternResults([], mockToasts, httpFetch));
const { result } = renderHook(() =>
useStoredPatternResults({
patterns: [],
toasts: mockToasts,
httpFetch,
isILMAvailable,
startTime,
endTime,
})
);
expect(result.current).toEqual([]);
expect(httpFetch).not.toHaveBeenCalled();
@ -45,7 +58,14 @@ describe('useStoredPatternResults', () => {
});
const { result, waitFor } = renderHook(() =>
useStoredPatternResults(patterns, mockToasts, httpFetch)
useStoredPatternResults({
patterns,
toasts: mockToasts,
httpFetch,
isILMAvailable,
startTime,
endTime,
})
);
await waitFor(() => result.current.length > 0);
@ -104,5 +124,63 @@ describe('useStoredPatternResults', () => {
},
]);
});
describe('when isILMAvailable is false', () => {
it('should call getStorageResults with startDate and endDate', async () => {
const patterns = ['pattern1-*', 'pattern2-*'];
httpFetch.mockImplementation((path: string) => {
if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern1-*') {
return Promise.resolve([getHistoricalResultStub('pattern1-index1')]);
}
if (path === '/internal/ecs_data_quality_dashboard/results_latest/pattern2-*') {
return Promise.resolve([getHistoricalResultStub('pattern2-index1')]);
}
return Promise.reject(new Error('Invalid path'));
});
const { result, waitFor } = renderHook(() =>
useStoredPatternResults({
patterns,
toasts: mockToasts,
httpFetch,
isILMAvailable: false,
startTime,
endTime,
})
);
await waitFor(() => result.current.length > 0);
expect(httpFetch).toHaveBeenCalledTimes(2);
expect(httpFetch).toHaveBeenCalledWith(
'/internal/ecs_data_quality_dashboard/results_latest/pattern1-*',
{
method: 'GET',
signal: expect.any(AbortSignal),
version: '1',
query: {
startDate: startTime,
endDate: endTime,
},
}
);
expect(httpFetch).toHaveBeenCalledWith(
'/internal/ecs_data_quality_dashboard/results_latest/pattern2-*',
{
method: 'GET',
signal: expect.any(AbortSignal),
version: '1',
query: {
startDate: startTime,
endDate: endTime,
},
}
);
});
});
});
});

View file

@ -10,13 +10,34 @@ import { HttpHandler } from '@kbn/core-http-browser';
import { isEmpty } from 'lodash/fp';
import { DataQualityCheckResult } from '../../../../types';
import { formatResultFromStorage, getStorageResults } from '../../utils/storage';
import {
GetStorageResultsOpts,
formatResultFromStorage,
getStorageResults,
} from '../../utils/storage';
export const useStoredPatternResults = (
patterns: string[],
toasts: IToasts,
httpFetch: HttpHandler
) => {
export interface UseStoredPatternResultsOpts {
patterns: string[];
toasts: IToasts;
httpFetch: HttpHandler;
isILMAvailable: boolean;
startTime: string;
endTime: string;
}
export type UseStoredPatternResultsReturnValue = Array<{
pattern: string;
results: Record<string, DataQualityCheckResult>;
}>;
export const useStoredPatternResults = ({
patterns,
toasts,
httpFetch,
isILMAvailable,
startTime,
endTime,
}: UseStoredPatternResultsOpts): UseStoredPatternResultsReturnValue => {
const [storedPatternResults, setStoredPatternResults] = useState<
Array<{ pattern: string; results: Record<string, DataQualityCheckResult> }>
>([]);
@ -28,8 +49,20 @@ export const useStoredPatternResults = (
const abortController = new AbortController();
const fetchStoredPatternResults = async () => {
const requests = patterns.map((pattern) =>
getStorageResults({ pattern, httpFetch, abortController, toasts }).then((results = []) => ({
const requests = patterns.map(async (pattern) => {
const getStorageResultsOpts: GetStorageResultsOpts = {
pattern,
httpFetch,
abortController,
toasts,
};
if (!isILMAvailable) {
getStorageResultsOpts.startTime = startTime;
getStorageResultsOpts.endTime = endTime;
}
return getStorageResults(getStorageResultsOpts).then((results) => ({
pattern,
results: Object.fromEntries(
results.map((storageResult) => [
@ -37,8 +70,8 @@ export const useStoredPatternResults = (
formatResultFromStorage({ storageResult, pattern }),
])
),
}))
);
}));
});
const patternResults = await Promise.all(requests);
if (patternResults?.length) {
@ -47,7 +80,7 @@ export const useStoredPatternResults = (
};
fetchStoredPatternResults();
}, [httpFetch, patterns, toasts]);
}, [endTime, httpFetch, isILMAvailable, patterns, startTime, toasts]);
return storedPatternResults;
};

View file

@ -35,6 +35,8 @@ describe('useResultsRollup', () => {
const patterns = ['auditbeat-*', 'packetbeat-*'];
const isILMAvailable = true;
const startTime = 'now-7d';
const endTime = 'now';
const useStoredPatternResultsMock = useStoredPatternResults as jest.Mock;
@ -52,6 +54,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -94,10 +98,19 @@ describe('useResultsRollup', () => {
patterns: ['auditbeat-*'],
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
expect(useStoredPatternResultsMock).toHaveBeenCalledWith(['auditbeat-*'], toasts, httpFetch);
expect(useStoredPatternResultsMock).toHaveBeenCalledWith({
patterns: ['auditbeat-*'],
toasts,
httpFetch,
isILMAvailable,
startTime,
endTime,
});
expect(result.current.patternRollups).toEqual({
'auditbeat-*': {
@ -119,6 +132,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -144,6 +159,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -180,6 +197,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -369,6 +388,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable: false,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -532,6 +553,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -592,6 +615,8 @@ describe('useResultsRollup', () => {
patterns,
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);
@ -654,6 +679,8 @@ describe('useResultsRollup', () => {
patterns: ['packetbeat-*', 'auditbeat-*'],
isILMAvailable,
telemetryEvents: mockTelemetryEvents,
startTime,
endTime,
})
);

View file

@ -40,6 +40,8 @@ interface Props {
httpFetch: HttpHandler;
telemetryEvents: TelemetryEvents;
isILMAvailable: boolean;
startTime: string;
endTime: string;
}
export const useResultsRollup = ({
httpFetch,
@ -47,11 +49,20 @@ export const useResultsRollup = ({
patterns,
isILMAvailable,
telemetryEvents,
startTime,
endTime,
}: Props): UseResultsRollupReturnValue => {
const [patternIndexNames, setPatternIndexNames] = useState<Record<string, string[]>>({});
const [patternRollups, setPatternRollups] = useState<Record<string, PatternRollup>>({});
const storedPatternsResults = useStoredPatternResults(patterns, toasts, httpFetch);
const storedPatternsResults = useStoredPatternResults({
httpFetch,
patterns,
toasts,
isILMAvailable,
startTime,
endTime,
});
useEffect(() => {
if (!isEmpty(storedPatternsResults)) {

View file

@ -200,4 +200,26 @@ describe('getStorageResults', () => {
expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
expect(results).toEqual([]);
});
it('should provide stad and end date', async () => {
await getStorageResults({
httpFetch: fetch,
abortController: new AbortController(),
pattern: 'auditbeat-*',
toasts,
startTime: 'now-7d',
endTime: 'now',
});
expect(fetch).toHaveBeenCalledWith(
'/internal/ecs_data_quality_dashboard/results_latest/auditbeat-*',
expect.objectContaining({
method: 'GET',
query: {
startDate: 'now-7d',
endDate: 'now',
},
})
);
});
});

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { HttpHandler } from '@kbn/core-http-browser';
import { HttpFetchQuery, HttpHandler } from '@kbn/core-http-browser';
import { IToasts } from '@kbn/core-notifications-browser';
import {
@ -131,23 +131,40 @@ export async function postStorageResult({
}
}
export interface GetStorageResultsOpts {
pattern: string;
httpFetch: HttpHandler;
toasts: IToasts;
abortController: AbortController;
startTime?: string;
endTime?: string;
}
export async function getStorageResults({
pattern,
httpFetch,
toasts,
abortController,
}: {
pattern: string;
httpFetch: HttpHandler;
toasts: IToasts;
abortController: AbortController;
}): Promise<StorageResult[]> {
startTime,
endTime,
}: GetStorageResultsOpts): Promise<StorageResult[]> {
try {
const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern);
const query: HttpFetchQuery = {};
if (startTime) {
query.startDate = startTime;
}
if (endTime) {
query.endDate = endTime;
}
const results = await httpFetch<StorageResult[]>(route, {
method: 'GET',
signal: abortController.signal,
version: INTERNAL_API_VERSION,
...(Object.keys(query).length > 0 ? { query } : {}),
});
return results;
} catch (err) {

View file

@ -67,6 +67,8 @@ describe('DataQualityPanel', () => {
setLastChecked={jest.fn()}
baseTheme={DARK_THEME}
toasts={toasts}
defaultStartTime={'now-7d'}
defaultEndTime={'now'}
/>
</TestExternalProviders>
);

View file

@ -46,6 +46,8 @@ interface Props {
setLastChecked: (lastChecked: string) => void;
startDate?: string | null;
theme?: PartialTheme;
defaultStartTime: string;
defaultEndTime: string;
}
const defaultSelectedIlmPhaseOptions: EuiComboBoxOptionOption[] = ilmPhaseOptionsStatic.filter(
@ -71,6 +73,8 @@ const DataQualityPanelComponent: React.FC<Props> = ({
setLastChecked,
startDate,
theme,
defaultStartTime,
defaultEndTime,
}) => {
const [selectedIlmPhaseOptions, setSelectedIlmPhaseOptions] = useState<EuiComboBoxOptionOption[]>(
defaultSelectedIlmPhaseOptions
@ -109,6 +113,8 @@ const DataQualityPanelComponent: React.FC<Props> = ({
toasts,
isILMAvailable,
telemetryEvents,
startTime: defaultStartTime,
endTime: defaultEndTime,
});
const indicesCheckHookReturnValue = useIndicesCheck({
@ -137,6 +143,8 @@ const DataQualityPanelComponent: React.FC<Props> = ({
ilmPhases={ilmPhases}
selectedIlmPhaseOptions={selectedIlmPhaseOptions}
setSelectedIlmPhaseOptions={setSelectedIlmPhaseOptions}
defaultStartTime={defaultStartTime}
defaultEndTime={defaultEndTime}
>
<ResultsRollupContext.Provider value={resultsRollupHookReturnValue}>
<IndicesCheckContext.Provider value={indicesCheckHookReturnValue}>

View file

@ -135,6 +135,8 @@ const TestDataQualityProvidersComponent: React.FC<TestDataQualityProvidersProps>
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
} = getMergedDataQualityContextProps(dataQualityContextProps);
const mergedResultsRollupContextProps =
@ -162,6 +164,8 @@ const TestDataQualityProvidersComponent: React.FC<TestDataQualityProvidersProps>
ilmPhases={ilmPhases}
selectedIlmPhaseOptions={selectedIlmPhaseOptions}
setSelectedIlmPhaseOptions={setSelectedIlmPhaseOptions}
defaultStartTime={defaultStartTime}
defaultEndTime={defaultEndTime}
>
<ResultsRollupContext.Provider value={mergedResultsRollupContextProps}>
<IndicesCheckContext.Provider

View file

@ -30,6 +30,8 @@ export const getMergedDataQualityContextProps = (
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
} = {
isILMAvailable: true,
addSuccessToast: jest.fn(),
@ -69,6 +71,8 @@ export const getMergedDataQualityContextProps = (
},
],
setSelectedIlmPhaseOptions: jest.fn(),
defaultStartTime: 'now-7d/d',
defaultEndTime: 'now/d',
...dataQualityContextProps,
};
@ -90,5 +94,7 @@ export const getMergedDataQualityContextProps = (
ilmPhases,
selectedIlmPhaseOptions,
setSelectedIlmPhaseOptions,
defaultStartTime,
defaultEndTime,
};
};

View file

@ -8,15 +8,15 @@
import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
export const getRequestBody = ({
indexPattern,
indexNameOrPattern,
startDate = 'now-7d/d',
endDate = 'now/d',
}: {
indexPattern: string;
indexNameOrPattern: string;
startDate: string;
endDate: string;
}): SearchRequest => ({
index: indexPattern,
index: indexNameOrPattern,
aggs: {
index: {
terms: {

View file

@ -0,0 +1,96 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getRangeFilteredIndices } from './get_range_filtered_indices';
import { fetchAvailableIndices } from '../lib/fetch_available_indices';
import type { IScopedClusterClient, Logger } from '@kbn/core/server';
jest.mock('../lib/fetch_available_indices');
const fetchAvailableIndicesMock = fetchAvailableIndices as jest.Mock;
describe('getRangeFilteredIndices', () => {
let client: jest.Mocked<IScopedClusterClient>;
let logger: jest.Mocked<Logger>;
beforeEach(() => {
client = {
asCurrentUser: jest.fn(),
} as unknown as jest.Mocked<IScopedClusterClient>;
logger = {
warn: jest.fn(),
error: jest.fn(),
} as unknown as jest.Mocked<Logger>;
jest.clearAllMocks();
});
describe('when fetching available indices is successful', () => {
describe('and there are available indices', () => {
it('should return the flattened available indices', async () => {
fetchAvailableIndicesMock.mockResolvedValueOnce(['index1', 'index2']);
fetchAvailableIndicesMock.mockResolvedValueOnce(['index3']);
const result = await getRangeFilteredIndices({
client,
authorizedIndexNames: ['auth1', 'auth2'],
startDate: '2023-01-01',
endDate: '2023-01-31',
logger,
pattern: 'pattern*',
});
expect(fetchAvailableIndices).toHaveBeenCalledTimes(2);
expect(result).toEqual(['index1', 'index2', 'index3']);
expect(logger.warn).not.toHaveBeenCalled();
});
});
describe('and there are no available indices', () => {
it('should log a warning and return an empty array', async () => {
fetchAvailableIndicesMock.mockResolvedValue([]);
const result = await getRangeFilteredIndices({
client,
authorizedIndexNames: ['auth1', 'auth2'],
startDate: '2023-01-01',
endDate: '2023-01-31',
logger,
pattern: 'pattern*',
});
expect(fetchAvailableIndices).toHaveBeenCalledTimes(2);
expect(result).toEqual([]);
expect(logger.warn).toHaveBeenCalledWith(
'No available authorized indices found under pattern: pattern*, in the given date range: 2023-01-01 - 2023-01-31'
);
});
});
});
describe('when fetching available indices fails', () => {
it('should log an error and return an empty array', async () => {
fetchAvailableIndicesMock.mockRejectedValue(new Error('Fetch error'));
const result = await getRangeFilteredIndices({
client,
authorizedIndexNames: ['auth1'],
startDate: '2023-01-01',
endDate: '2023-01-31',
logger,
pattern: 'pattern*',
});
expect(fetchAvailableIndices).toHaveBeenCalledTimes(1);
expect(result).toEqual([]);
expect(logger.error).toHaveBeenCalledWith(
'Error fetching available indices in the given data range: 2023-01-01 - 2023-01-31'
);
});
});
});

View file

@ -0,0 +1,61 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IScopedClusterClient, Logger } from '@kbn/core/server';
import { fetchAvailableIndices } from '../lib/fetch_available_indices';
export const getRangeFilteredIndices = async ({
client,
authorizedIndexNames,
startDate,
endDate,
logger,
pattern,
}: {
client: IScopedClusterClient;
authorizedIndexNames: string[];
startDate: string;
endDate: string;
logger: Logger;
pattern: string;
}): Promise<string[]> => {
const decodedStartDate = decodeURIComponent(startDate);
const decodedEndDate = decodeURIComponent(endDate);
try {
const currentUserEsClient = client.asCurrentUser;
const availableIndicesPromises: Array<Promise<string[]>> = [];
for (const indexName of authorizedIndexNames) {
availableIndicesPromises.push(
fetchAvailableIndices(currentUserEsClient, {
indexNameOrPattern: indexName,
startDate: decodedStartDate,
endDate: decodedEndDate,
})
);
}
const availableIndices = await Promise.all(availableIndicesPromises);
const flattenedAvailableIndices = availableIndices.flat();
if (flattenedAvailableIndices.length === 0) {
logger.warn(
`No available authorized indices found under pattern: ${pattern}, in the given date range: ${decodedStartDate} - ${decodedEndDate}`
);
}
return flattenedAvailableIndices;
} catch (err) {
logger.error(
`Error fetching available indices in the given data range: ${decodedStartDate} - ${decodedEndDate}`
);
return [];
}
};

View file

@ -61,7 +61,7 @@ describe('fetchAvailableIndices', () => {
const esClientMock = getEsClientMock();
await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -101,7 +101,7 @@ describe('fetchAvailableIndices', () => {
const esClientMock = getEsClientMock();
await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -133,7 +133,7 @@ describe('fetchAvailableIndices', () => {
]);
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -164,7 +164,7 @@ describe('fetchAvailableIndices', () => {
]);
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -180,7 +180,7 @@ describe('fetchAvailableIndices', () => {
esClientMock.cat.indices.mockResolvedValue([]);
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'nonexistent-*',
indexNameOrPattern: 'nonexistent-*',
startDate: startDateString,
endDate: endDateString,
});
@ -209,7 +209,7 @@ describe('fetchAvailableIndices', () => {
});
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -243,7 +243,7 @@ describe('fetchAvailableIndices', () => {
});
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -268,7 +268,7 @@ describe('fetchAvailableIndices', () => {
]);
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -285,7 +285,7 @@ describe('fetchAvailableIndices', () => {
await expect(
fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
})
@ -307,7 +307,7 @@ describe('fetchAvailableIndices', () => {
});
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -336,7 +336,7 @@ describe('fetchAvailableIndices', () => {
});
const result = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
});
@ -371,7 +371,7 @@ describe('fetchAvailableIndices', () => {
]);
const results = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: 'now-7d/d',
endDate: 'now/d',
});
@ -390,7 +390,7 @@ describe('fetchAvailableIndices', () => {
]);
const results = await fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: 'now-7d/d',
endDate: 'now-1d/d',
});
@ -415,7 +415,7 @@ describe('fetchAvailableIndices', () => {
await expect(
fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: endDateString,
})
@ -429,7 +429,7 @@ describe('fetchAvailableIndices', () => {
await expect(
fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: 'invalid-date',
endDate: endDateString,
})
@ -443,7 +443,7 @@ describe('fetchAvailableIndices', () => {
await expect(
fetchAvailableIndices(esClientMock, {
indexPattern: 'logs-*',
indexNameOrPattern: 'logs-*',
startDate: startDateString,
endDate: 'invalid-date',
})

View file

@ -32,15 +32,15 @@ const getParsedDateMs = (dateStr: string, roundUp = false) => {
export const fetchAvailableIndices = async (
esClient: ElasticsearchClient,
params: { indexPattern: string; startDate: string; endDate: string }
params: { indexNameOrPattern: string; startDate: string; endDate: string }
): Promise<string[]> => {
const { indexPattern, startDate, endDate } = params;
const { indexNameOrPattern, startDate, endDate } = params;
const startDateMs = getParsedDateMs(startDate);
const endDateMs = getParsedDateMs(endDate, true);
const indicesCats = (await esClient.cat.indices({
index: indexPattern,
index: indexNameOrPattern,
format: 'json',
h: 'index,creation.date',
})) as FetchAvailableCatIndicesResponseRequired;

View file

@ -81,7 +81,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => {
const meteringStatsIndices = parseMeteringStats(meteringStats.indices);
const availableIndices = await fetchAvailableIndices(esClient, {
indexPattern: decodedIndexName,
indexNameOrPattern: decodedIndexName,
startDate: decodedStartDate,
endDate: decodedEndDate,
});

View file

@ -16,6 +16,24 @@ import { resultDocument } from './results.mock';
import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types';
import type { ResultDocument } from '../../schemas/result';
import type { CheckIndicesPrivilegesParam } from './privileges';
import { getRangeFilteredIndices } from '../../helpers/get_range_filtered_indices';
const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) =>
Promise.resolve(Object.fromEntries(indices.map((index) => [index, true])))
);
jest.mock('./privileges', () => ({
checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) =>
mockCheckIndicesPrivileges(params),
}));
jest.mock('../../helpers/get_range_filtered_indices', () => ({
getRangeFilteredIndices: jest.fn(),
}));
const mockGetRangeFilteredIndices = getRangeFilteredIndices as jest.Mock;
const startDate = 'now-7d';
const endDate = 'now';
const searchResponse = {
aggregations: {
@ -33,14 +51,6 @@ const searchResponse = {
Record<string, { buckets: LatestAggResponseBucket[] }>
>;
const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) =>
Promise.resolve(Object.fromEntries(indices.map((index) => [index, true])))
);
jest.mock('./privileges', () => ({
checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) =>
mockCheckIndicesPrivileges(params),
}));
describe('getIndexResultsLatestRoute route', () => {
describe('querying', () => {
let server: ReturnType<typeof serverMock.create>;
@ -68,7 +78,7 @@ describe('getIndexResultsLatestRoute route', () => {
getIndexResultsLatestRoute(server.router, logger);
});
it('gets result', async () => {
it('gets result without startDate and endDate', async () => {
const mockSearch = context.core.elasticsearch.client.asInternalUser.search;
mockSearch.mockResolvedValueOnce(searchResponse);
@ -80,6 +90,159 @@ describe('getIndexResultsLatestRoute route', () => {
expect(response.status).toEqual(200);
expect(response.body).toEqual([resultDocument]);
expect(mockGetRangeFilteredIndices).not.toHaveBeenCalled();
});
it('gets result with startDate and endDate', async () => {
const reqWithDate = requestMock.create({
method: 'get',
path: GET_INDEX_RESULTS_LATEST,
params: { pattern: 'logs-*' },
query: { startDate, endDate },
});
const filteredIndices = ['filtered-index-1', 'filtered-index-2'];
mockGetRangeFilteredIndices.mockResolvedValueOnce(filteredIndices);
const mockSearch = context.core.elasticsearch.client.asInternalUser.search;
mockSearch.mockResolvedValueOnce(searchResponse);
const response = await server.inject(reqWithDate, requestContextMock.convertContext(context));
expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({
client: context.core.elasticsearch.client,
authorizedIndexNames: [resultDocument.indexName],
startDate,
endDate,
logger,
pattern: 'logs-*',
});
expect(mockSearch).toHaveBeenCalledWith({
index: expect.any(String),
...getQuery(filteredIndices),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual([resultDocument]);
});
it('handles getRangeFilteredIndices error', async () => {
const errorMessage = 'Range Filter Error';
const reqWithDate = requestMock.create({
method: 'get',
path: GET_INDEX_RESULTS_LATEST,
params: { pattern: 'logs-*' },
query: { startDate, endDate },
});
mockGetRangeFilteredIndices.mockRejectedValueOnce(new Error(errorMessage));
const response = await server.inject(reqWithDate, requestContextMock.convertContext(context));
expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({
client: context.core.elasticsearch.client,
authorizedIndexNames: [resultDocument.indexName],
startDate,
endDate,
logger,
pattern: 'logs-*',
});
expect(response.status).toEqual(500);
expect(response.body).toEqual({ message: errorMessage, status_code: 500 });
expect(logger.error).toHaveBeenCalledWith(errorMessage);
});
it('gets result with startDate and endDate and multiple filtered indices', async () => {
const filteredIndices = ['filtered-index-1', 'filtered-index-2', 'filtered-index-3'];
const filteredIndicesSearchResponse = {
aggregations: {
latest: {
buckets: filteredIndices.map((indexName) => ({
key: indexName,
latest_doc: { hits: { hits: [{ _source: { indexName } }] } },
})),
},
},
} as unknown as SearchResponse<
ResultDocument,
Record<string, { buckets: LatestAggResponseBucket[] }>
>;
const reqWithDate = requestMock.create({
method: 'get',
path: GET_INDEX_RESULTS_LATEST,
params: { pattern: 'logs-*' },
query: { startDate, endDate },
});
mockGetRangeFilteredIndices.mockResolvedValueOnce(filteredIndices);
context.core.elasticsearch.client.asInternalUser.search.mockResolvedValueOnce(
filteredIndicesSearchResponse
);
const response = await server.inject(reqWithDate, requestContextMock.convertContext(context));
expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({
client: context.core.elasticsearch.client,
authorizedIndexNames: [resultDocument.indexName],
startDate,
endDate,
logger,
pattern: 'logs-*',
});
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({
index: expect.any(String),
...getQuery(filteredIndices),
});
const expectedResults = filteredIndices.map((indexName) => ({
indexName,
})) as ResultDocument[];
expect(response.status).toEqual(200);
expect(response.body).toEqual(expectedResults);
});
it('handles partial authorization when using startDate and endDate', async () => {
const authorizationResult = {
'filtered-index-1': true,
'filtered-index-2': false,
};
mockGetRangeFilteredIndices.mockResolvedValueOnce(['filtered-index-1']);
mockCheckIndicesPrivileges.mockResolvedValueOnce(authorizationResult);
const mockSearch = context.core.elasticsearch.client.asInternalUser.search;
mockSearch.mockResolvedValueOnce(searchResponse);
const reqWithDate = requestMock.create({
method: 'get',
path: GET_INDEX_RESULTS_LATEST,
params: { pattern: 'logs-*' },
query: { startDate, endDate },
});
const response = await server.inject(reqWithDate, requestContextMock.convertContext(context));
expect(mockGetRangeFilteredIndices).toHaveBeenCalledWith({
client: context.core.elasticsearch.client,
authorizedIndexNames: ['filtered-index-1'],
startDate,
endDate,
logger,
pattern: 'logs-*',
});
expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({
index: expect.any(String),
...getQuery(['filtered-index-1']),
});
expect(response.status).toEqual(200);
expect(response.body).toEqual([resultDocument]);
});
it('handles results data stream error', async () => {

View file

@ -4,18 +4,18 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { IRouter, Logger } from '@kbn/core/server';
import { INTERNAL_API_VERSION, GET_INDEX_RESULTS_LATEST } from '../../../common/constants';
import { buildResponse } from '../../lib/build_response';
import { buildRouteValidation } from '../../schemas/common';
import { GetIndexResultsLatestParams } from '../../schemas/result';
import { GetIndexResultsLatestParams, GetIndexResultsLatestQuery } from '../../schemas/result';
import type { ResultDocument } from '../../schemas/result';
import { API_DEFAULT_ERROR_MESSAGE } from '../../translations';
import type { DataQualityDashboardRequestHandlerContext } from '../../types';
import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations';
import { getAuthorizedIndexNames } from '../../helpers/get_authorized_index_names';
import { getRangeFilteredIndices } from '../../helpers/get_range_filtered_indices';
export const getQuery = (indexName: string[]) => ({
size: 0,
@ -49,6 +49,7 @@ export const getIndexResultsLatestRoute = (
validate: {
request: {
params: buildRouteValidation(GetIndexResultsLatestParams),
query: buildRouteValidation(GetIndexResultsLatestQuery),
},
},
},
@ -77,8 +78,27 @@ export const getIndexResultsLatestRoute = (
return response.ok({ body: [] });
}
const { startDate, endDate } = request.query;
let resultingIndices: string[] = [];
if (startDate && endDate) {
resultingIndices = resultingIndices.concat(
await getRangeFilteredIndices({
client,
authorizedIndexNames,
startDate,
endDate,
logger,
pattern,
})
);
} else {
resultingIndices = authorizedIndexNames;
}
// Get the latest result for each indexName
const query = { index, ...getQuery(authorizedIndexNames) };
const query = { index, ...getQuery(resultingIndices) };
const { aggregations } = await client.asInternalUser.search<
ResultDocument,
Record<string, { buckets: LatestAggResponseBucket[] }>

View file

@ -69,6 +69,11 @@ export const PostIndexResultBody = ResultDocument;
export const GetIndexResultsLatestParams = t.type({ pattern: t.string });
export type GetIndexResultsLatestParams = t.TypeOf<typeof GetIndexResultsLatestParams>;
export const GetIndexResultsLatestQuery = t.partial({
startDate: t.string,
endDate: t.string,
});
export const GetIndexResultsParams = t.type({
pattern: t.string,
});

View file

@ -8,6 +8,7 @@
import { render, screen, waitFor } from '@testing-library/react';
import React from 'react';
import { MemoryRouter } from 'react-router-dom';
import type { HttpFetchOptions } from '@kbn/core-http-browser';
import { useKibana as mockUseKibana } from '../../common/lib/kibana/__mocks__';
import { TestProviders } from '../../common/mock';
@ -22,7 +23,17 @@ jest.mock('../../common/lib/kibana', () => {
const mockKibanaServices = {
get: () => ({
http: { fetch: jest.fn() },
http: {
fetch: jest.fn().mockImplementation((path: string, options: HttpFetchOptions) => {
if (
path.startsWith('/internal/ecs_data_quality_dashboard/results_latest') &&
options.method === 'GET'
) {
return Promise.resolve([]);
}
return Promise.resolve();
}),
},
}),
};

View file

@ -171,6 +171,8 @@ const DataQualityComponent: React.FC = () => {
startDate={startDate}
theme={theme}
toasts={toasts}
defaultStartTime={DEFAULT_START_TIME}
defaultEndTime={DEFAULT_END_TIME}
/>
</SecuritySolutionPageWrapper>
) : (