github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ML] Data frame analytics: Adds functionality to map view (#83710)

Browse source 
* get all jobs from index node

* create map from modelId and enable url share

* highlight source node

* add map endpoint to api doc

* use variables in css.fix types.ensure map tab is shown

* fix translations
This commit is contained in:
Melissa Alvarez 2020-11-20 10:39:30 -05:00 • committed by GitHub
parent 22e494e386
commit 00e59512fa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 561 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/ml/common/constants/data_frame_analytics.ts
View file

@ -16,7 +16,7 @@ export const JOB_MAP_NODE_TYPES = {
ANALYTICS: 'analytics',
TRANSFORM: 'transform',
INDEX: 'index',
INFERENCE_MODEL: 'inferenceModel',
TRAINED_MODEL: 'trainedModel',
} as const;
export type JobMapNodeTypes = typeof JOB_MAP_NODE_TYPES[keyof typeof JOB_MAP_NODE_TYPES];

3
x-pack/plugins/ml/common/types/ml_url_generator.ts
View file

@ -156,6 +156,7 @@ export type TimeSeriesExplorerUrlState = MLPageState<
export interface DataFrameAnalyticsQueryState {
jobId?: JobId | JobId[];
modelId?: string;
groupIds?: string[];
globalState?: MlCommonGlobalState;
}
@ -170,6 +171,7 @@ export interface DataFrameAnalyticsExplorationQueryState {
jobId: JobId;
analysisType: DataFrameAnalysisConfigType;
defaultIsTraining?: boolean;
modelId?: string;
};
}
@ -180,6 +182,7 @@ export type DataFrameAnalyticsExplorationUrlState = MLPageState<
analysisType: DataFrameAnalysisConfigType;
globalState?: MlCommonGlobalState;
defaultIsTraining?: boolean;
modelId?: string;
}
>;

11
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_navigation_bar/analytics_navigation_bar.tsx
View file

@ -15,10 +15,11 @@ interface Tab {
path: string;
}
export const AnalyticsNavigationBar: FC<{ selectedTabId?: string; jobId?: string }> = ({
jobId,
selectedTabId,
}) => {
export const AnalyticsNavigationBar: FC<{
selectedTabId?: string;
jobId?: string;
modelId?: string;
}> = ({ jobId, modelId, selectedTabId }) => {
const navigateToPath = useNavigateToPath();
const tabs = useMemo(() => {
@ -38,7 +39,7 @@ export const AnalyticsNavigationBar: FC<{ selectedTabId?: string; jobId?: string
path: '/data_frame_analytics/models',
},
];
if (jobId !== undefined) {
if (jobId !== undefined || modelId !== undefined) {
navTabs.push({
id: 'map',
name: i18n.translate('xpack.ml.dataframe.mapTabLabel', {

2
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/models_management/models_list.tsx
View file

@ -342,7 +342,7 @@ export const ModelsList: FC = () => {
onClick: async (item) => {
const path = await mlUrlGenerator.createUrl({
page: ML_PAGES.DATA_FRAME_ANALYTICS_MAP,
pageState: { jobId: item.metadata?.analytics_config.id },
pageState: { modelId: item.model_id },
});
await navigateToPath(path, false);

11
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/page.tsx
View file

@ -59,6 +59,7 @@ export const Page: FC = () => {
const location = useLocation();
const selectedTabId = useMemo(() => location.pathname.split('/').pop(), [location]);
const mapJobId = globalState?.ml?.jobId;
const mapModelId = globalState?.ml?.modelId;
return (
<Fragment>
@ -106,8 +107,14 @@ export const Page: FC = () => {
<UpgradeWarning />
<EuiPageContent>
<AnalyticsNavigationBar selectedTabId={selectedTabId} jobId={mapJobId} />
{selectedTabId === 'map' && mapJobId && <JobMap analyticsId={mapJobId} />}
<AnalyticsNavigationBar
selectedTabId={selectedTabId}
jobId={mapJobId}
modelId={mapModelId}
/>
{selectedTabId === 'map' && (mapJobId || mapModelId) && (
<JobMap analyticsId={mapJobId} modelId={mapModelId} />
)}
{selectedTabId === 'data_frame_analytics' && (
<DataFrameAnalyticsList
blockRefresh={blockRefresh}

25
x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/_legend.scss
View file

@ -5,7 +5,7 @@
.mlJobMapLegend__indexPattern {
height: $euiSizeM;
width: $euiSizeM;
background-color: '#FFFFFF';
background-color: $euiColorGhost;
border: 1px solid $euiColorVis2;
transform: rotate(45deg);
display: 'inline-block';
@ -14,7 +14,7 @@
.mlJobMapLegend__transform {
height: $euiSizeM;
width: $euiSizeM;
background-color: '#FFFFFF';
background-color: $euiColorGhost;
border: 1px solid $euiColorVis1;
display: 'inline-block';
}
@ -22,17 +22,26 @@
.mlJobMapLegend__analytics {
height: $euiSizeM;
width: $euiSizeM;
background-color: '#FFFFFF';
background-color: $euiColorGhost;
border: 1px solid $euiColorVis0;
border-radius: 50%;
border-radius: $euiBorderRadius;
display: 'inline-block';
}
.mlJobMapLegend__inferenceModel {
.mlJobMapLegend__trainedModel {
height: $euiSizeM;
width: $euiSizeM;
background-color: '#FFFFFF';
border: 1px solid $euiColorMediumShade;
border-radius: 50%;
background-color: $euiColorGhost;
border: $euiBorderThin;
border-radius: $euiBorderRadius;
display: 'inline-block';
}
.mlJobMapLegend__sourceNode {
height: $euiSizeM;
width: $euiSizeM;
background-color: $euiColorLightShade;
border: $euiBorderThin;
border-radius: $euiBorderRadius;
display: 'inline-block';
}

13
x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/controls.tsx
View file

@ -25,10 +25,11 @@ import { EuiDescriptionListProps } from '@elastic/eui/src/components/description
import { CytoscapeContext } from './cytoscape';
import { formatHumanReadableDateTimeSeconds } from '../../../../../../common/util/date_utils';
import { JOB_MAP_NODE_TYPES } from '../../../../../../common/constants/data_frame_analytics';
// import { DeleteButton } from './delete_button';
// import { DeleteButton } from './delete_button'; // TODO: add delete functionality in followup
interface Props {
analyticsId: string;
analyticsId?: string;
modelId?: string;
details: any;
getNodeData: any;
}
@ -56,7 +57,7 @@ function getListItems(details: object): EuiDescriptionListProps['listItems'] {
});
}
export const Controls: FC<Props> = ({ analyticsId, details, getNodeData }) => {
export const Controls: FC<Props> = ({ analyticsId, modelId, details, getNodeData }) => {
const [showFlyout, setShowFlyout] = useState<boolean>(false);
const [selectedNode, setSelectedNode] = useState<cytoscape.NodeSingular | undefined>();
@ -98,10 +99,12 @@ export const Controls: FC<Props> = ({ analyticsId, details, getNodeData }) => {
}
const nodeDataButton =
analyticsId !== nodeLabel && nodeType === JOB_MAP_NODE_TYPES.ANALYTICS ? (
analyticsId !== nodeLabel &&
modelId !== nodeLabel &&
(nodeType === JOB_MAP_NODE_TYPES.ANALYTICS || nodeType === JOB_MAP_NODE_TYPES.INDEX) ? (
<EuiButtonEmpty
onClick={() => {
getNodeData(nodeLabel);
getNodeData({ id: nodeLabel, type: nodeType });
setShowFlyout(false);
}}
iconType="branch"

3
x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/cytoscape_options.tsx
View file

@ -80,7 +80,8 @@ export const cytoscapeOptions: cytoscape.CytoscapeOptions = {
{
selector: 'node',
style: {
'background-color': theme.euiColorGhost,
'background-color': (el: cytoscape.NodeSingular) =>
el.data('isRoot') ? theme.euiColorLightShade : theme.euiColorGhost,
'background-height': '60%',
'background-width': '60%',
'border-color': (el: cytoscape.NodeSingular) => borderColorForNode(el),

33
x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/components/legend.tsx
View file

@ -6,6 +6,7 @@
import React, { FC } from 'react';
import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { JOB_MAP_NODE_TYPES } from '../../../../../../common/constants/data_frame_analytics';
export const JobMapLegend: FC = () => (
@ -17,7 +18,10 @@ export const JobMapLegend: FC = () => (
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued">
{JOB_MAP_NODE_TYPES.INDEX}
<FormattedMessage
id="xpack.ml.dataframe.analyticsMap.legend.indexLabel"
defaultMessage="index"
/>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
@ -41,7 +45,10 @@ export const JobMapLegend: FC = () => (
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued">
{JOB_MAP_NODE_TYPES.ANALYTICS}
<FormattedMessage
id="xpack.ml.dataframe.analyticsMap.legend.analyticsJobLabel"
defaultMessage="analytics job"
/>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
@ -49,11 +56,29 @@ export const JobMapLegend: FC = () => (
<EuiFlexItem grow={false}>
<EuiFlexGroup gutterSize="xs" alignItems="center">
<EuiFlexItem grow={false}>
<span className="mlJobMapLegend__inferenceModel" />
<span className="mlJobMapLegend__trainedModel" />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued">
{'inference model'}
<FormattedMessage
id="xpack.ml.dataframe.analyticsMap.legend.trainedModelLabel"
defaultMessage="trained model"
/>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiFlexGroup gutterSize="xs" alignItems="center">
<EuiFlexItem grow={false}>
<span className="mlJobMapLegend__sourceNode" />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText size="xs" color="subdued">
<FormattedMessage
id="xpack.ml.dataframe.analyticsMap.legend.rootNodeLabel"
defaultMessage="source node"
/>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>

73
x-pack/plugins/ml/public/application/data_frame_analytics/pages/job_map/job_map.tsx
View file

@ -15,6 +15,7 @@ import { Cytoscape, Controls, JobMapLegend } from './components';
import { ml } from '../../../services/ml_api_service';
import { useMlKibana } from '../../../contexts/kibana';
import { useRefDimensions } from './components/use_ref_dimensions';
import { JOB_MAP_NODE_TYPES } from '../../../../../common/constants/data_frame_analytics';
const cytoscapeDivStyle = {
background: `linear-gradient(
@ -36,22 +37,36 @@ ${theme.euiColorLightShade}`,
marginTop: 0,
};
export const JobMapTitle: React.FC<{ analyticsId: string }> = ({ analyticsId }) => (
export const JobMapTitle: React.FC<{ analyticsId?: string; modelId?: string }> = ({
analyticsId,
modelId,
}) => (
<EuiTitle size="xs">
<span>
{i18n.translate('xpack.ml.dataframe.analyticsMap.analyticsIdTitle', {
defaultMessage: 'Map for analytics ID {analyticsId}',
values: { analyticsId },
})}
{analyticsId
? i18n.translate('xpack.ml.dataframe.analyticsMap.analyticsIdTitle', {
defaultMessage: 'Map for analytics ID {analyticsId}',
values: { analyticsId },
})
: i18n.translate('xpack.ml.dataframe.analyticsMap.modelIdTitle', {
defaultMessage: 'Map for trained model ID {modelId}',
values: { modelId },
})}
</span>
</EuiTitle>
);
interface Props {
analyticsId: string;
interface GetDataObjectParameter {
id: string;
type: string;
}
export const JobMap: FC<Props> = ({ analyticsId }) => {
interface Props {
analyticsId?: string;
modelId?: string;
}
export const JobMap: FC<Props> = ({ analyticsId, modelId }) => {
const [elements, setElements] = useState<cytoscape.ElementDefinition[]>([]);
const [nodeDetails, setNodeDetails] = useState({});
const [error, setError] = useState(undefined);
@ -60,14 +75,33 @@ export const JobMap: FC<Props> = ({ analyticsId }) => {
services: { notifications },
} = useMlKibana();
const getData = async (id?: string) => {
const getDataWrapper = async (params?: GetDataObjectParameter) => {
const { id, type } = params ?? {};
const treatAsRoot = id !== undefined;
const idToUse = treatAsRoot ? id : analyticsId;
// Pass in treatAsRoot flag - endpoint will take job destIndex to grab jobs created from it
let idToUse: string;
if (id !== undefined) {
idToUse = id;
} else if (modelId !== undefined) {
idToUse = modelId;
} else {
idToUse = analyticsId as string;
}
await getData(
idToUse,
treatAsRoot,
modelId !== undefined && treatAsRoot === false ? JOB_MAP_NODE_TYPES.TRAINED_MODEL : type
);
};
const getData = async (idToUse: string, treatAsRoot: boolean, type?: string) => {
// Pass in treatAsRoot flag - endpoint will take job or index to grab jobs created from it
// TODO: update analyticsMap return type here
const analyticsMap: any = await ml.dataFrameAnalytics.getDataFrameAnalyticsMap(
idToUse,
treatAsRoot
treatAsRoot,
type
);
const { elements: nodeElements, details, error: fetchError } = analyticsMap;
@ -86,7 +120,7 @@ export const JobMap: FC<Props> = ({ analyticsId }) => {
}
if (nodeElements && nodeElements.length > 0) {
if (id === undefined) {
if (treatAsRoot === false) {
setElements(nodeElements);
setNodeDetails(details);
} else {
@ -98,8 +132,8 @@ export const JobMap: FC<Props> = ({ analyticsId }) => {
};
useEffect(() => {
getData();
}, [analyticsId]);
getDataWrapper();
}, [analyticsId, modelId]);
if (error !== undefined) {
notifications.toasts.addDanger(
@ -119,14 +153,19 @@ export const JobMap: FC<Props> = ({ analyticsId }) => {
<div style={{ height: height - parseInt(theme.gutterTypes.gutterLarge, 10) }} ref={ref}>
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<JobMapTitle analyticsId={analyticsId} />
<JobMapTitle analyticsId={analyticsId} modelId={modelId} />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<JobMapLegend />
</EuiFlexItem>
</EuiFlexGroup>
<Cytoscape height={height} elements={elements} width={width} style={cytoscapeDivStyle}>
<Controls details={nodeDetails} getNodeData={getData} analyticsId={analyticsId} />
<Controls
details={nodeDetails}
getNodeData={getDataWrapper}
analyticsId={analyticsId}
modelId={modelId}
/>
</Cytoscape>
</div>
</>

8
x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts
View file

@ -83,12 +83,12 @@ export const dataFrameAnalytics = {
body,
});
},
getDataFrameAnalyticsMap(analyticsId?: string, treatAsRoot?: boolean) {
const analyticsIdString = analyticsId !== undefined ? `/${analyticsId}` : '';
getDataFrameAnalyticsMap(id: string, treatAsRoot: boolean, type?: string) {
const idString = id !== undefined ? `/${id}` : '';
return http({
path: `${basePath()}/data_frame/analytics/map${analyticsIdString}`,
path: `${basePath()}/data_frame/analytics/map${idString}`,
method: 'GET',
query: { treatAsRoot },
query: { treatAsRoot, type },
});
},
evaluateDataFrameAnalytics(evaluateConfig: any) {

3
x-pack/plugins/ml/public/ml_url_generator/data_frame_analytics_urls_generator.ts
View file

@ -104,11 +104,12 @@ export function createDataFrameAnalyticsMapUrl(
let url = `${appBasePath}/${ML_PAGES.DATA_FRAME_ANALYTICS_MAP}`;
if (mlUrlGeneratorState) {
const { jobId, analysisType, defaultIsTraining, globalState } = mlUrlGeneratorState;
const { jobId, modelId, analysisType, defaultIsTraining, globalState } = mlUrlGeneratorState;
const queryState: DataFrameAnalyticsExplorationQueryState = {
ml: {
jobId,
modelId,
analysisType,
defaultIsTraining,
},

562
x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts
View file

@ -10,12 +10,17 @@ import {
JOB_MAP_NODE_TYPES,
JobMapNodeTypes,
} from '../../../common/constants/data_frame_analytics';
import { TrainedModelConfigResponse } from '../../../common/types/trained_models';
import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer';
import { getAnalysisType } from '../../../common/util/analytics_utils';
import {
AnalyticsMapEdgeElement,
AnalyticsMapReturnType,
AnalyticsMapNodeElement,
ExtendAnalyticsMapArgs,
GetAnalyticsMapArgs,
InitialElementsReturnType,
isCompleteInitialReturnType,
isAnalyticsMapEdgeElement,
isAnalyticsMapNodeElement,
isIndexPatternLinkReturnType,
@ -29,7 +34,7 @@ import type { MlClient } from '../../lib/ml_client';
export class AnalyticsManager {
private _client: IScopedClusterClient['asInternalUser'];
private _mlClient: MlClient;
public _inferenceModels: any; // TODO: update types
public _inferenceModels: TrainedModelConfigResponse[];
constructor(mlClient: MlClient, client: IScopedClusterClient['asInternalUser']) {
this._client = client;
@ -37,11 +42,11 @@ export class AnalyticsManager {
this._inferenceModels = [];
}
public set inferenceModels(models: any) {
public set inferenceModels(models) {
this._inferenceModels = models;
}
public get inferenceModels(): any {
public get inferenceModels() {
return this._inferenceModels;
}
@ -56,16 +61,20 @@ export class AnalyticsManager {
}
}
private isDuplicateElement(analyticsId: string, elements: any[]): boolean {
private isDuplicateElement(analyticsId: string, elements: MapElements[]): boolean {
let isDuplicate = false;
elements.forEach((elem: any) => {
if (elem.data.label === analyticsId && elem.data.type === JOB_MAP_NODE_TYPES.ANALYTICS) {
elements.forEach((elem) => {
if (
isAnalyticsMapNodeElement(elem) &&
elem.data.label === analyticsId &&
elem.data.type === JOB_MAP_NODE_TYPES.ANALYTICS
) {
isDuplicate = true;
}
});
return isDuplicate;
}
// @ts-ignore // TODO: is this needed?
private async getAnalyticsModelData(modelId: string) {
const resp = await this._mlClient.getTrainedModels({
model_id: modelId,
@ -80,11 +89,17 @@ export class AnalyticsManager {
return models;
}
private async getAnalyticsJobData(analyticsId: string) {
const resp = await this._mlClient.getDataFrameAnalytics({
id: analyticsId,
});
const jobData = resp?.body?.data_frame_analytics[0];
private async getAnalyticsData(analyticsId?: string) {
const options = analyticsId
? {
id: analyticsId,
}
: undefined;
const resp = await this._mlClient.getDataFrameAnalytics(options);
const jobData = analyticsId
? resp?.body?.data_frame_analytics[0]
: resp?.body?.data_frame_analytics;
return jobData;
}
@ -130,7 +145,7 @@ export class AnalyticsManager {
return { isWildcardIndexPattern, isIndexPattern: true, indexData, meta };
} else if (type.includes(JOB_MAP_NODE_TYPES.ANALYTICS)) {
// fetch job associated with this index
const jobData = await this.getAnalyticsJobData(id);
const jobData = await this.getAnalyticsData(id);
return { jobData, isJob: true };
} else if (type === JOB_MAP_NODE_TYPES.TRANSFORM) {
// fetch transform so we can get original index pattern
@ -155,12 +170,12 @@ export class AnalyticsManager {
let edgeElement;
if (analyticsModel !== undefined) {
const modelId = `${analyticsModel.model_id}-${JOB_MAP_NODE_TYPES.INFERENCE_MODEL}`;
const modelId = `${analyticsModel.model_id}-${JOB_MAP_NODE_TYPES.TRAINED_MODEL}`;
modelElement = {
data: {
id: modelId,
label: analyticsModel.model_id,
type: JOB_MAP_NODE_TYPES.INFERENCE_MODEL,
type: JOB_MAP_NODE_TYPES.TRAINED_MODEL,
},
};
// Create edge for job and corresponding model
@ -201,29 +216,41 @@ export class AnalyticsManager {
}
/**
* Works backward from jobId to return related jobs from source indices
* @param jobId
* Prepares the initial elements for incoming modelId
* @param modelId
*/
async getAnalyticsMap(analyticsId: string): Promise<AnalyticsMapReturnType> {
const result: any = { elements: [], details: {}, error: null };
const modelElements: MapElements[] = [];
const indexPatternElements: MapElements[] = [];
async getInitialElementsModelRoot(modelId: string): Promise<InitialElementsReturnType> {
const resultElements = [];
const modelElements = [];
const details: any = {};
// fetch model data and create model elements
let data = await this.getAnalyticsModelData(modelId);
const modelNodeId = `${data.model_id}-${JOB_MAP_NODE_TYPES.TRAINED_MODEL}`;
const sourceJobId = data?.metadata?.analytics_config?.id;
let nextLinkId: string | undefined;
let nextType: JobMapNodeTypes | undefined;
let previousNodeId: string | undefined;
try {
await this.setInferenceModels();
// Create first node for incoming analyticsId
let data = await this.getAnalyticsJobData(analyticsId);
let nextLinkId = data?.source?.index[0];
let nextType: JobMapNodeTypes = JOB_MAP_NODE_TYPES.INDEX;
let complete = false;
let link: NextLinkReturnType;
let count = 0;
let rootTransform;
let rootIndexPattern;
modelElements.push({
data: {
id: modelNodeId,
label: data.model_id,
type: JOB_MAP_NODE_TYPES.TRAINED_MODEL,
isRoot: true,
},
});
let previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
details[modelNodeId] = data;
// fetch source job data and create elements
if (sourceJobId !== undefined) {
data = await this.getAnalyticsData(sourceJobId);
result.elements.push({
nextLinkId = data?.source?.index[0];
nextType = JOB_MAP_NODE_TYPES.INDEX;
previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
resultElements.push({
data: {
id: previousNodeId,
label: data.id,
@ -231,167 +258,178 @@ export class AnalyticsManager {
analysisType: getAnalysisType(data?.analysis),
},
});
result.details[previousNodeId] = data;
// Create edge between job and model
modelElements.push({
data: {
id: `${previousNodeId}~${modelNodeId}`,
source: previousNodeId,
target: modelNodeId,
},
});
let { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(analyticsId);
if (isAnalyticsMapNodeElement(modelElement)) {
modelElements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
details[previousNodeId] = data;
}
return { data, details, resultElements, modelElements, nextLinkId, nextType, previousNodeId };
}
/**
* Prepares the initial elements for incoming jobId
* @param jobId
*/
async getInitialElementsJobRoot(jobId: string): Promise<InitialElementsReturnType> {
const resultElements = [];
const modelElements = [];
const details: any = {};
const data = await this.getAnalyticsData(jobId);
const nextLinkId = data?.source?.index[0];
const nextType: JobMapNodeTypes = JOB_MAP_NODE_TYPES.INDEX;
const previousNodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
resultElements.push({
data: {
id: previousNodeId,
label: data.id,
type: JOB_MAP_NODE_TYPES.ANALYTICS,
analysisType: getAnalysisType(data?.analysis),
isRoot: true,
},
});
details[previousNodeId] = data;
const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(jobId);
if (isAnalyticsMapNodeElement(modelElement)) {
modelElements.push(modelElement);
details[modelElement.data.id] = modelDetails;
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
modelElements.push(edgeElement);
}
return { data, details, resultElements, modelElements, nextLinkId, nextType, previousNodeId };
}
/**
* Works backward from jobId or modelId to return related jobs, indices, models, and transforms
* @param jobId (optional)
* @param modelId (optional)
*/
async getAnalyticsMap({
analyticsId,
modelId,
}: GetAnalyticsMapArgs): Promise<AnalyticsMapReturnType> {
const result: AnalyticsMapReturnType = { elements: [], details: {}, error: null };
const modelElements: MapElements[] = [];
const indexPatternElements: MapElements[] = [];
try {
await this.setInferenceModels();
// Create first node for incoming analyticsId or modelId
let initialData: InitialElementsReturnType = {} as InitialElementsReturnType;
if (analyticsId !== undefined) {
initialData = await this.getInitialElementsJobRoot(analyticsId);
} else if (modelId !== undefined) {
initialData = await this.getInitialElementsModelRoot(modelId);
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
modelElements.push(edgeElement);
}
// Add a safeguard against infinite loops.
while (complete === false) {
count++;
if (count >= 100) {
break;
}
try {
link = await this.getNextLink({
id: nextLinkId,
type: nextType,
});
} catch (error) {
result.error = error.message || 'Something went wrong';
break;
}
// If it's index pattern, check meta data to see what to fetch next
if (isIndexPatternLinkReturnType(link) && link.isIndexPattern === true) {
if (link.isWildcardIndexPattern === true) {
// Create index nodes for each of the indices included in the index pattern then break
const { details, elements } = this.getIndexPatternElements(
link.indexData,
previousNodeId
);
const {
resultElements,
details: initialDetails,
modelElements: initialModelElements,
} = initialData;
indexPatternElements.push(...elements);
result.details = { ...result.details, ...details };
complete = true;
} else {
const nodeId = `${nextLinkId}-${JOB_MAP_NODE_TYPES.INDEX}`;
result.elements.unshift({
data: { id: nodeId, label: nextLinkId, type: JOB_MAP_NODE_TYPES.INDEX },
});
result.details[nodeId] = link.indexData;
}
result.elements.push(...resultElements);
result.details = initialDetails;
modelElements.push(...initialModelElements);
// Check meta data
if (
link.isWildcardIndexPattern === false &&
(link.meta === undefined || link.meta?.created_by === INDEX_META_DATA_CREATED_BY)
) {
rootIndexPattern = nextLinkId;
complete = true;
if (isCompleteInitialReturnType(initialData)) {
let { data, nextLinkId, nextType, previousNodeId } = initialData;
let complete = false;
let link: NextLinkReturnType;
let count = 0;
let rootTransform;
let rootIndexPattern;
let modelElement;
let modelDetails;
let edgeElement;
// Add a safeguard against infinite loops.
while (complete === false) {
count++;
if (count >= 100) {
break;
}
if (link.meta?.created_by === 'data-frame-analytics') {
nextLinkId = link.meta.analytics;
nextType = JOB_MAP_NODE_TYPES.ANALYTICS;
try {
link = await this.getNextLink({
id: nextLinkId,
type: nextType,
});
} catch (error) {
result.error = error.message || 'Something went wrong';
break;
}
// If it's index pattern, check meta data to see what to fetch next
if (isIndexPatternLinkReturnType(link) && link.isIndexPattern === true) {
if (link.isWildcardIndexPattern === true) {
// Create index nodes for each of the indices included in the index pattern then break
const { details, elements } = this.getIndexPatternElements(
link.indexData,
previousNodeId
);
if (link.meta?.created_by === JOB_MAP_NODE_TYPES.TRANSFORM) {
nextLinkId = link.meta._transform?.transform;
nextType = JOB_MAP_NODE_TYPES.TRANSFORM;
}
} else if (isJobDataLinkReturnType(link) && link.isJob === true) {
data = link.jobData;
const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
previousNodeId = nodeId;
indexPatternElements.push(...elements);
result.details = { ...result.details, ...details };
complete = true;
} else {
const nodeId = `${nextLinkId}-${JOB_MAP_NODE_TYPES.INDEX}`;
result.elements.unshift({
data: { id: nodeId, label: nextLinkId, type: JOB_MAP_NODE_TYPES.INDEX },
});
result.details[nodeId] = link.indexData;
}
result.elements.unshift({
data: {
id: nodeId,
label: data.id,
type: JOB_MAP_NODE_TYPES.ANALYTICS,
analysisType: getAnalysisType(data?.analysis),
},
});
result.details[nodeId] = data;
nextLinkId = data?.source?.index[0];
nextType = JOB_MAP_NODE_TYPES.INDEX;
// Check meta data
if (
link.isWildcardIndexPattern === false &&
(link.meta === undefined || link.meta?.created_by === INDEX_META_DATA_CREATED_BY)
) {
rootIndexPattern = nextLinkId;
complete = true;
break;
}
// Get inference model for analytics job and create model node
({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(data.id));
if (isAnalyticsMapNodeElement(modelElement)) {
modelElements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
modelElements.push(edgeElement);
}
} else if (isTransformLinkReturnType(link) && link.isTransform === true) {
data = link.transformData;
if (link.meta?.created_by === 'data-frame-analytics') {
nextLinkId = link.meta.analytics;
nextType = JOB_MAP_NODE_TYPES.ANALYTICS;
}
const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.TRANSFORM}`;
previousNodeId = nodeId;
rootTransform = data.dest.index;
if (link.meta?.created_by === JOB_MAP_NODE_TYPES.TRANSFORM) {
nextLinkId = link.meta._transform?.transform;
nextType = JOB_MAP_NODE_TYPES.TRANSFORM;
}
} else if (isJobDataLinkReturnType(link) && link.isJob === true) {
data = link.jobData;
const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
previousNodeId = nodeId;
result.elements.unshift({
data: { id: nodeId, label: data.id, type: JOB_MAP_NODE_TYPES.TRANSFORM },
});
result.details[nodeId] = data;
nextLinkId = data?.source?.index[0];
nextType = JOB_MAP_NODE_TYPES.INDEX;
}
} // end while
// create edge elements
const elemLength = result.elements.length - 1;
for (let i = 0; i < elemLength; i++) {
const currentElem = result.elements[i];
const nextElem = result.elements[i + 1];
if (
currentElem !== undefined &&
nextElem !== undefined &&
currentElem?.data?.id.includes('*') === false &&
nextElem?.data?.id.includes('*') === false
) {
result.elements.push({
data: {
id: `${currentElem.data.id}~${nextElem.data.id}`,
source: currentElem.data.id,
target: nextElem.data.id,
},
});
}
}
// fetch all jobs associated with root transform if defined, otherwise check root index
if (rootTransform !== undefined || rootIndexPattern !== undefined) {
const analyticsJobs = await this._mlClient.getDataFrameAnalytics();
const jobs = analyticsJobs?.body?.data_frame_analytics || [];
const comparator = rootTransform !== undefined ? rootTransform : rootIndexPattern;
for (let i = 0; i < jobs.length; i++) {
if (
jobs[i]?.source?.index[0] === comparator &&
this.isDuplicateElement(jobs[i].id, result.elements) === false
) {
const nodeId = `${jobs[i].id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
result.elements.push({
result.elements.unshift({
data: {
id: nodeId,
label: jobs[i].id,
label: data.id,
type: JOB_MAP_NODE_TYPES.ANALYTICS,
analysisType: getAnalysisType(jobs[i]?.analysis),
},
});
result.details[nodeId] = jobs[i];
const source = `${comparator}-${JOB_MAP_NODE_TYPES.INDEX}`;
result.elements.push({
data: {
id: `${source}~${nodeId}`,
source,
target: nodeId,
analysisType: getAnalysisType(data?.analysis),
},
});
result.details[nodeId] = data;
nextLinkId = data?.source?.index[0];
nextType = JOB_MAP_NODE_TYPES.INDEX;
// Get inference model for analytics job and create model node
({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(
jobs[i].id
));
({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(data.id));
if (isAnalyticsMapNodeElement(modelElement)) {
modelElements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
@ -399,12 +437,88 @@ export class AnalyticsManager {
if (isAnalyticsMapEdgeElement(edgeElement)) {
modelElements.push(edgeElement);
}
} else if (isTransformLinkReturnType(link) && link.isTransform === true) {
data = link.transformData;
const nodeId = `${data.id}-${JOB_MAP_NODE_TYPES.TRANSFORM}`;
previousNodeId = nodeId;
rootTransform = data.dest.index;
result.elements.unshift({
data: { id: nodeId, label: data.id, type: JOB_MAP_NODE_TYPES.TRANSFORM },
});
result.details[nodeId] = data;
nextLinkId = data?.source?.index[0];
nextType = JOB_MAP_NODE_TYPES.INDEX;
}
} // end while
// create edge elements
const elemLength = result.elements.length - 1;
for (let i = 0; i < elemLength; i++) {
const currentElem = result.elements[i];
const nextElem = result.elements[i + 1];
if (
currentElem !== undefined &&
nextElem !== undefined &&
currentElem?.data?.id.includes('*') === false &&
nextElem?.data?.id.includes('*') === false
) {
result.elements.push({
data: {
id: `${currentElem.data.id}~${nextElem.data.id}`,
source: currentElem.data.id,
target: nextElem.data.id,
},
});
}
}
// fetch all jobs associated with root transform if defined, otherwise check root index
if (rootTransform !== undefined || rootIndexPattern !== undefined) {
const jobs = await this.getAnalyticsData();
const comparator = rootTransform !== undefined ? rootTransform : rootIndexPattern;
for (let i = 0; i < jobs.length; i++) {
if (
jobs[i]?.source?.index[0] === comparator &&
this.isDuplicateElement(jobs[i].id, result.elements) === false
) {
const nodeId = `${jobs[i].id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
result.elements.push({
data: {
id: nodeId,
label: jobs[i].id,
type: JOB_MAP_NODE_TYPES.ANALYTICS,
analysisType: getAnalysisType(jobs[i]?.analysis),
},
});
result.details[nodeId] = jobs[i];
const source = `${comparator}-${JOB_MAP_NODE_TYPES.INDEX}`;
result.elements.push({
data: {
id: `${source}~${nodeId}`,
source,
target: nodeId,
},
});
// Get inference model for analytics job and create model node
({ modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(
jobs[i].id
));
if (isAnalyticsMapNodeElement(modelElement)) {
modelElements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
modelElements.push(edgeElement);
}
}
}
}
}
// Include model and index pattern nodes in result elements now that all other nodes have been created
result.elements.push(...modelElements, ...indexPatternElements);
return result;
} catch (error) {
result.error = error.message || 'An error occurred fetching map';
@ -412,56 +526,64 @@ export class AnalyticsManager {
}
}
async extendAnalyticsMapForAnalyticsJob(analyticsId: string): Promise<AnalyticsMapReturnType> {
const result: any = { elements: [], details: {}, error: null };
async extendAnalyticsMapForAnalyticsJob({
analyticsId,
index,
}: ExtendAnalyticsMapArgs): Promise<AnalyticsMapReturnType> {
const result: AnalyticsMapReturnType = { elements: [], details: {}, error: null };
try {
await this.setInferenceModels();
const jobs = await this.getAnalyticsData();
let rootIndex;
let rootIndexNodeId;
const jobData = await this.getAnalyticsJobData(analyticsId);
const currentJobNodeId = `${jobData.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
const destIndex = Array.isArray(jobData?.dest?.index)
? jobData?.dest?.index[0]
: jobData?.dest?.index;
const destIndexNodeId = `${destIndex}-${JOB_MAP_NODE_TYPES.INDEX}`;
const analyticsJobs = await this._mlClient.getDataFrameAnalytics();
const jobs = analyticsJobs?.body?.data_frame_analytics || [];
if (analyticsId !== undefined) {
const jobData = await this.getAnalyticsData(analyticsId);
const currentJobNodeId = `${jobData.id}-${JOB_MAP_NODE_TYPES.ANALYTICS}`;
rootIndex = Array.isArray(jobData?.dest?.index)
? jobData?.dest?.index[0]
: jobData?.dest?.index;
rootIndexNodeId = `${rootIndex}-${JOB_MAP_NODE_TYPES.INDEX}`;
// Fetch inference model for incoming job id and add node and edge
const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(
analyticsId
);
if (isAnalyticsMapNodeElement(modelElement)) {
result.elements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
// Fetch inference model for incoming job id and add node and edge
const { modelElement, modelDetails, edgeElement } = this.getAnalyticsModelElements(
analyticsId
);
if (isAnalyticsMapNodeElement(modelElement)) {
result.elements.push(modelElement);
result.details[modelElement.data.id] = modelDetails;
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
result.elements.push(edgeElement);
}
// If rootIndex node has not been created, create it
const rootIndexDetails = await this.getIndexData(rootIndex);
result.elements.push({
data: {
id: rootIndexNodeId,
label: rootIndex,
type: JOB_MAP_NODE_TYPES.INDEX,
},
});
result.details[rootIndexNodeId] = rootIndexDetails;
// Connect incoming job to rootIndex
result.elements.push({
data: {
id: `${currentJobNodeId}~${rootIndexNodeId}`,
source: currentJobNodeId,
target: rootIndexNodeId,
},
});
} else {
rootIndex = index;
rootIndexNodeId = `${rootIndex}-${JOB_MAP_NODE_TYPES.INDEX}`;
}
if (isAnalyticsMapEdgeElement(edgeElement)) {
result.elements.push(edgeElement);
}
// If destIndex node has not been created, create it
const destIndexDetails = await this.getIndexData(destIndex);
result.elements.push({
data: {
id: destIndexNodeId,
label: destIndex,
type: JOB_MAP_NODE_TYPES.INDEX,
},
});
result.details[destIndexNodeId] = destIndexDetails;
// Connect incoming job to destIndex
result.elements.push({
data: {
id: `${currentJobNodeId}~${destIndexNodeId}`,
source: currentJobNodeId,
target: destIndexNodeId,
},
});
for (let i = 0; i < jobs.length; i++) {
if (
jobs[i]?.source?.index[0] === destIndex &&
jobs[i]?.source?.index[0] === rootIndex &&
this.isDuplicateElement(jobs[i].id, result.elements) === false
) {
// Create node for associated job
@ -478,8 +600,8 @@ export class AnalyticsManager {
result.elements.push({
data: {
id: `${destIndexNodeId}~${nodeId}`,
source: destIndexNodeId,
id: `${rootIndexNodeId}~${nodeId}`,
source: rootIndexNodeId,
target: nodeId,
},
});

53
x-pack/plugins/ml/server/models/data_frame_analytics/types.ts
View file

@ -4,6 +4,29 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { JobMapNodeTypes } from '../../../common/constants/data_frame_analytics';
interface AnalyticsMapArg {
analyticsId: string;
}
interface GetAnalyticsJobIdArg extends AnalyticsMapArg {
modelId?: never;
}
interface GetAnalyticsModelIdArg {
analyticsId?: never;
modelId: string;
}
interface ExtendAnalyticsJobIdArg extends AnalyticsMapArg {
index?: never;
}
interface ExtendAnalyticsIndexArg {
analyticsId?: never;
index: string;
}
export type GetAnalyticsMapArgs = GetAnalyticsJobIdArg | GetAnalyticsModelIdArg;
export type ExtendAnalyticsMapArgs = ExtendAnalyticsJobIdArg | ExtendAnalyticsIndexArg;
export interface IndexPatternLinkReturnType {
isWildcardIndexPattern: boolean;
isIndexPattern: boolean;
@ -26,9 +49,27 @@ export type NextLinkReturnType =
export type MapElements = AnalyticsMapNodeElement | AnalyticsMapEdgeElement;
export interface AnalyticsMapReturnType {
elements: MapElements[];
details: object; // transform, job, or index details
details: Record<string, any>; // transform, job, or index details
error: null | any;
}
interface BasicInitialElementsReturnType {
data: any;
details: object;
resultElements: MapElements[];
modelElements: MapElements[];
}
export interface InitialElementsReturnType extends BasicInitialElementsReturnType {
nextLinkId?: string;
nextType?: JobMapNodeTypes;
previousNodeId?: string;
}
interface CompleteInitialElementsReturnType extends BasicInitialElementsReturnType {
nextLinkId: string;
nextType: JobMapNodeTypes;
previousNodeId: string;
}
export interface AnalyticsMapNodeElement {
data: {
id: string;
@ -44,6 +85,16 @@ export interface AnalyticsMapEdgeElement {
target: string;
};
}
export const isCompleteInitialReturnType = (arg: any): arg is CompleteInitialElementsReturnType => {
if (typeof arg !== 'object' || arg === null) return false;
const keys = Object.keys(arg);
return (
keys.length > 0 &&
keys.includes('nextLinkId') &&
keys.includes('nextType') &&
keys.includes('previousNodeId')
);
};
export const isAnalyticsMapNodeElement = (arg: any): arg is AnalyticsMapNodeElement => {
if (typeof arg !== 'object' || arg === null) return false;
const keys = Object.keys(arg);

1
x-pack/plugins/ml/server/routes/apidoc.json
View file

@ -17,6 +17,7 @@
"UpdateDataFrameAnalytics",
"DeleteDataFrameAnalytics",
"JobsExist",
"GetDataFrameAnalyticsIdMap",
"DataVisualizer",
"GetOverallStats",

34
x-pack/plugins/ml/server/routes/data_frame_analytics.ts
View file

@ -8,6 +8,7 @@ import { RequestHandlerContext, IScopedClusterClient } from 'kibana/server';
import { wrapError } from '../client/error_wrapper';
import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages';
import { RouteInitialization } from '../types';
import { JOB_MAP_NODE_TYPES } from '../../common/constants/data_frame_analytics';
import {
dataAnalyticsJobConfigSchema,
dataAnalyticsJobUpdateSchema,
@ -19,6 +20,7 @@ import {
deleteDataFrameAnalyticsJobSchema,
jobsExistSchema,
} from './schemas/data_analytics_schema';
import { GetAnalyticsMapArgs, ExtendAnalyticsMapArgs } from '../models/data_frame_analytics/types';
import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns';
import { AnalyticsManager } from '../models/data_frame_analytics/analytics_manager';
import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics';
@ -36,14 +38,22 @@ function deleteDestIndexPatternById(context: RequestHandlerContext, indexPattern
return iph.deleteIndexPatternById(indexPatternId);
}
function getAnalyticsMap(mlClient: MlClient, client: IScopedClusterClient, analyticsId: string) {
function getAnalyticsMap(
mlClient: MlClient,
client: IScopedClusterClient,
idOptions: GetAnalyticsMapArgs
) {
const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
return analytics.getAnalyticsMap(analyticsId);
return analytics.getAnalyticsMap(idOptions);
}
function getExtendedMap(mlClient: MlClient, client: IScopedClusterClient, analyticsId: string) {
function getExtendedMap(
mlClient: MlClient,
client: IScopedClusterClient,
idOptions: ExtendAnalyticsMapArgs
) {
const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
return analytics.extendAnalyticsMapForAnalyticsJob(analyticsId);
return analytics.extendAnalyticsMapForAnalyticsJob(idOptions);
}
/**
@ -633,10 +643,20 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout
try {
const { analyticsId } = request.params;
const treatAsRoot = request.query?.treatAsRoot;
const caller =
treatAsRoot === 'true' || treatAsRoot === true ? getExtendedMap : getAnalyticsMap;
const type = request.query?.type;
const results = await caller(mlClient, client, analyticsId);
let results;
if (treatAsRoot === 'true' || treatAsRoot === true) {
results = await getExtendedMap(mlClient, client, {
analyticsId: type !== JOB_MAP_NODE_TYPES.INDEX ? analyticsId : undefined,
index: type === JOB_MAP_NODE_TYPES.INDEX ? analyticsId : undefined,
});
} else {
results = await getAnalyticsMap(mlClient, client, {
analyticsId: type !== JOB_MAP_NODE_TYPES.TRAINED_MODEL ? analyticsId : undefined,
modelId: type === JOB_MAP_NODE_TYPES.TRAINED_MODEL ? analyticsId : undefined,
});
}
return response.ok({
body: results,

2
x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
View file

@ -89,5 +89,5 @@ export const jobsExistSchema = schema.object({
});
export const analyticsMapQuerySchema = schema.maybe(
schema.object({ treatAsRoot: schema.maybe(schema.any()) })
schema.object({ treatAsRoot: schema.maybe(schema.any()), type: schema.maybe(schema.string()) })
);