[ML] Adds data frame messages to data frames list (#39609) (#39738)

* kibana server endpoint for fetching DataFrame messages

* create Messages tab in expanded row

* update snapshot

* translate error message

* ensure messages fetched on refresh interval

* add tab translation

* update lastUpdate every time getJobs is run

* update expanded row snapshot

x-pack/legacy/plugins/ml/common/constants/index_patterns.ts

@@ -10,3 +10,4 @@ export const ML_ANNOTATIONS_INDEX_PATTERN = '.ml-annotations-6';
 
 export const ML_RESULTS_INDEX_PATTERN = '.ml-anomalies-*';
 export const ML_NOTIFICATION_INDEX_PATTERN = '.ml-notifications';
+export const ML_DF_NOTIFICATION_INDEX_PATTERN = '.data-frame-notifications-1';

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/__snapshots__/expanded_row.test.tsx.snap

@@ -205,6 +205,14 @@ exports[`Data Frame: Job List <ExpandedRow /> Minimal initialization 1`] = `
         "id": "job-json",
         "name": "JSON",
       },
+      Object {
+        "content": <TransformMessagesPane
+          lastUpdate={12345678}
+          transformId="fq_date_histogram_1m_1441"
+        />,
+        "id": "job-messages",
+        "name": "Messages",
+      },
     ]
   }
 />

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.test.tsx

@@ -16,7 +16,7 @@ describe('Data Frame: Job List <ExpandedRow />', () => {
   test('Minimal initialization', () => {
     const item: DataFrameJobListRow = dataFrameJobListRow;
 
-    const wrapper = shallow(<ExpandedRow item={item} />);
+    const wrapper = shallow(<ExpandedRow item={item} lastUpdate={12345678} />);
 
     expect(wrapper).toMatchSnapshot();
   });

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/expanded_row.tsx

@@ -13,6 +13,7 @@ import { i18n } from '@kbn/i18n';
 import { DataFrameJobListRow } from './common';
 import { JobDetailsPane, SectionConfig } from './job_details_pane';
 import { JobJsonPane } from './job_json_pane';
+import { TransformMessagesPane } from './transform_messages_pane';
 
 function getItemDescription(value: any) {
   if (typeof value === 'object') {
@@ -24,9 +25,10 @@ function getItemDescription(value: any) {
 
 interface Props {
   item: DataFrameJobListRow;
+  lastUpdate: number;
 }
 
-export const ExpandedRow: SFC<Props> = ({ item }) => {
+export const ExpandedRow: SFC<Props> = ({ item, lastUpdate }) => {
   const state: SectionConfig = {
     title: 'State',
     items: Object.entries(item.state).map(s => {
@@ -56,6 +58,13 @@ export const ExpandedRow: SFC<Props> = ({ item }) => {
       name: 'JSON',
       content: <JobJsonPane json={item.config} />,
     },
+    {
+      id: 'job-messages',
+      name: i18n.translate('xpack.ml.dataframe.jobsList.jobDetails.tabs.jobMessagesLabel', {
+        defaultMessage: 'Messages',
+      }),
+      content: <TransformMessagesPane transformId={item.id} lastUpdate={lastUpdate} />,
+    },
   ];
   return (
     <EuiTabbedContent

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/job_list.tsx

@@ -24,13 +24,14 @@ import { useRefreshInterval } from './use_refresh_interval';
 
 function getItemIdToExpandedRowMap(
   itemIds: JobId[],
-  dataFrameJobs: DataFrameJobListRow[]
+  dataFrameJobs: DataFrameJobListRow[],
+  lastUpdate: number
 ): ItemIdToExpandedRowMap {
   return itemIds.reduce(
     (m: ItemIdToExpandedRowMap, jobId: JobId) => {
       const item = dataFrameJobs.find(job => job.config.id === jobId);
       if (item !== undefined) {
-        m[jobId] = <ExpandedRow item={item} />;
+        m[jobId] = <ExpandedRow item={item} lastUpdate={lastUpdate} />;
       }
       return m;
     },
@@ -50,9 +51,10 @@ export const DataFrameJobList: SFC = () => {
   const [dataFrameJobs, setDataFrameJobs] = useState<DataFrameJobListRow[]>([]);
   const [blockRefresh, setBlockRefresh] = useState(false);
   const [expandedRowItemIds, setExpandedRowItemIds] = useState<JobId[]>([]);
+  const [lastUpdate, setlastUpdate] = useState(Date.now());
 
   const getJobs = getJobsFactory(setDataFrameJobs, blockRefresh);
-  useRefreshInterval(getJobs, setBlockRefresh);
+  useRefreshInterval(getJobs, setBlockRefresh, setlastUpdate);
 
   if (dataFrameJobs.length === 0) {
     return (
@@ -77,7 +79,11 @@ export const DataFrameJobList: SFC = () => {
     },
   };
 
-  const itemIdToExpandedRowMap = getItemIdToExpandedRowMap(expandedRowItemIds, dataFrameJobs);
+  const itemIdToExpandedRowMap = getItemIdToExpandedRowMap(
+    expandedRowItemIds,
+    dataFrameJobs,
+    lastUpdate
+  );
 
   return (
     <ExpandableTable

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/transform_messages_pane.tsx

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { Fragment, useState, useEffect } from 'react';
+
+import { EuiSpacer, EuiBasicTable } from '@elastic/eui';
+// @ts-ignore
+import { formatDate } from '@elastic/eui/lib/services/format';
+import { i18n } from '@kbn/i18n';
+import { ml } from '../../../../../services/ml_api_service';
+
+const TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+interface Props {
+  transformId: string;
+  lastUpdate: number;
+}
+
+export const TransformMessagesPane: React.SFC<Props> = ({ transformId, lastUpdate }) => {
+  const [messages, setMessages] = useState([]);
+  const [isLoading, setIsLoading] = useState(true);
+  const [errorMessage, setErrorMessage] = useState('');
+
+  async function getMessages() {
+    try {
+      const messagesResp = await ml.dataFrame.getTransformAuditMessages(transformId);
+      setMessages(messagesResp);
+      setIsLoading(false);
+    } catch (error) {
+      setIsLoading(false);
+      setErrorMessage(
+        i18n.translate('xpack.ml.dfJobsList.jobDetails.messagesPane.errorMessage', {
+          defaultMessage: 'Messages could not be loaded',
+        })
+      );
+    }
+  }
+
+  useEffect(
+    () => {
+      getMessages();
+    },
+    [lastUpdate]
+  );
+
+  const columns = [
+    {
+      name: i18n.translate('xpack.ml.dfJobsList.jobDetails.messagesPane.timeLabel', {
+        defaultMessage: 'Time',
+      }),
+      render: (message: any) => formatDate(message.timestamp, TIME_FORMAT),
+    },
+    {
+      field: 'node_name',
+      name: i18n.translate('xpack.ml.dfJobsList.jobDetails.messagesPane.nodeLabel', {
+        defaultMessage: 'Node',
+      }),
+    },
+    {
+      field: 'message',
+      name: i18n.translate('xpack.ml.dfJobsList.jobDetails.messagesPane.messageLabel', {
+        defaultMessage: 'Message',
+      }),
+      width: '50%',
+    },
+  ];
+
+  return (
+    <Fragment>
+      <EuiSpacer size="s" />
+      <EuiBasicTable
+        items={messages}
+        columns={columns}
+        compressed={true}
+        loading={isLoading}
+        error={errorMessage}
+      />
+    </Fragment>
+  );
+};

x-pack/legacy/plugins/ml/public/data_frame/pages/job_management/components/job_list/use_refresh_interval.ts

@@ -17,7 +17,8 @@ import { GetJobs } from './job_service/get_jobs';
 
 export const useRefreshInterval = (
   getJobs: GetJobs,
-  setBlockRefresh: React.Dispatch<React.SetStateAction<boolean>>
+  setBlockRefresh: React.Dispatch<React.SetStateAction<boolean>>,
+  setlastUpdate: React.Dispatch<React.SetStateAction<number>>
 ) => {
   useEffect(() => {
     let jobsRefreshInterval: null | number = null;
@@ -56,6 +57,7 @@ export const useRefreshInterval = (
       } else {
         setRefreshInterval(value);
       }
+      setlastUpdate(Date.now());
       getJobs(true);
     }
 
@@ -64,6 +66,7 @@ export const useRefreshInterval = (
       if (interval >= MINIMUM_REFRESH_INTERVAL_MS) {
         setBlockRefresh(false);
         const intervalId = window.setInterval(() => {
+          setlastUpdate(Date.now());
           getJobs();
         }, interval);
         jobsRefreshInterval = intervalId;

x-pack/legacy/plugins/ml/public/services/ml_api_service/data_frame.js

@@ -64,4 +64,10 @@ export const dataFrame = {
       method: 'POST',
     });
   },
+  getTransformAuditMessages(transformId) {
+    return http({
+      url: `${basePath}/_data_frame/transforms/${transformId}/messages`,
+      method: 'GET',
+    });
+  },
 };

x-pack/legacy/plugins/ml/public/services/ml_api_service/index.d.ts

@@ -29,6 +29,7 @@ declare interface Ml {
     getDataFrameTransformsPreview(payload: any): Promise<any>;
     startDataFrameTransformsJob(jobId: string): Promise<any>;
     stopDataFrameTransformsJob(jobId: string): Promise<any>;
+    getTransformAuditMessages(transformId: string): Promise<any>;
   };
 
   hasPrivileges(obj: object): Promise<any>;

x-pack/legacy/plugins/ml/server/models/data_frame/index.js

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+export { transformAuditMessagesProvider } from './transform_audit_messages';

x-pack/legacy/plugins/ml/server/models/data_frame/transform_audit_messages.js

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+import { ML_DF_NOTIFICATION_INDEX_PATTERN } from '../../../common/constants/index_patterns';
+
+const SIZE = 1000;
+
+export function transformAuditMessagesProvider(callWithRequest) {
+
+  // search for audit messages,
+  // transformId is optional. without it, all jobs will be listed.
+  async function getTransformAuditMessages(transformId) {
+    const query = {
+      bool: {
+        filter: [
+          {
+            bool: {
+              must_not: {
+                term: {
+                  level: 'activity'
+                }
+              }
+            }
+          },
+        ]
+      }
+    };
+
+    // if no transformId specified, load all of the messages
+    if (transformId !== undefined) {
+      query.bool.filter.push({
+        bool: {
+          should: [
+            {
+              term: {
+                transform_id: '' // catch system messages
+              }
+            },
+            {
+              term: {
+                transform_id: transformId // messages for specified transformId
+              }
+            }
+          ]
+        }
+      });
+    }
+
+    try {
+      const resp = await callWithRequest('search', {
+        index: ML_DF_NOTIFICATION_INDEX_PATTERN,
+        ignore_unavailable: true,
+        rest_total_hits_as_int: true,
+        size: SIZE,
+        body:
+        {
+          sort: [
+            { timestamp: { order: 'asc' } },
+            { transform_id: { order: 'asc' } }
+          ],
+          query
+        }
+      });
+
+      let messages = [];
+      if (resp.hits.total !== 0) {
+        messages = resp.hits.hits.map(hit => hit._source);
+      }
+      return messages;
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  return {
+    getTransformAuditMessages
+  };
+}

x-pack/legacy/plugins/ml/server/routes/data_frame.js

@@ -6,6 +6,7 @@
 
 import { callWithRequestFactory } from '../client/call_with_request_factory';
 import { wrapError } from '../client/errors';
+import { transformAuditMessagesProvider } from '../models/data_frame/transform_audit_messages';
 
 export function dataFrameRoutes({ commonRouteConfig, elasticsearchPlugin, route }) {
 
@@ -124,4 +125,19 @@ export function dataFrameRoutes({ commonRouteConfig, elasticsearchPlugin, route
     }
   });
 
+  route({
+    method: 'GET',
+    path: '/api/ml/_data_frame/transforms/{transformId}/messages',
+    handler(request) {
+      const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request);
+      const { getTransformAuditMessages } = transformAuditMessagesProvider(callWithRequest);
+      const { transformId } = request.params;
+      return getTransformAuditMessages(transformId)
+        .catch(resp => wrapError(resp));
+    },
+    config: {
+      ...commonRouteConfig
+    }
+  });
+
 }