[ML] Transforms: Use estypes.TransformGetTransformStatsTransformStats for transform stats. (#180347)

## Summary

Gets rid of a lot of the manually defined attributes of `TransformStats`
by extending from `estypes.TransformGetTransformStatsTransformStats`.

How we treat the transform health status needed to be adapted because
the types provided by `estypes` are both lower and upper case (e.g.
`green/GREEN`) but lack the `unknown` status we use in the UI.
`mapEsHealthStatus2TransformHealthStatus()` is used to transform a
health status returned via API to the one we use in the UI.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: Dima Arnautov <arnautov.dima@gmail.com>

x-pack/plugins/transform/common/constants.test.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mapEsHealthStatus2TransformHealthStatus } from './constants';
+
+describe('mapEsHealthStatus2TransformHealthStatus', () => {
+  it('maps estypes HealthStatus to TransformHealthStatus', () => {
+    expect(mapEsHealthStatus2TransformHealthStatus(undefined)).toBe('unknown');
+    expect(mapEsHealthStatus2TransformHealthStatus('green')).toBe('green');
+    expect(mapEsHealthStatus2TransformHealthStatus('GREEN')).toBe('green');
+    expect(mapEsHealthStatus2TransformHealthStatus('yellow')).toBe('yellow');
+    expect(mapEsHealthStatus2TransformHealthStatus('YELLOW')).toBe('yellow');
+    expect(mapEsHealthStatus2TransformHealthStatus('red')).toBe('red');
+    expect(mapEsHealthStatus2TransformHealthStatus('RED')).toBe('red');
+  });
+});

x-pack/plugins/transform/common/constants.ts

@@ -5,8 +5,9 @@
  * 2.0.
  */
 
-import { i18n } from '@kbn/i18n';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import { i18n } from '@kbn/i18n';
 import type { LicenseType } from '@kbn/licensing-plugin/common/types';
 import { ALERT_NAMESPACE } from '@kbn/rule-data-utils';
 import type { TransformHealthTests } from './types/alerting';
@@ -103,14 +104,21 @@ export const TRANSFORM_STATE = {
 
 export type TransformState = typeof TRANSFORM_STATE[keyof typeof TRANSFORM_STATE];
 
-export const TRANSFORM_HEALTH = {
+export const TRANSFORM_HEALTH_STATUS = {
   green: 'green',
-  unknown: 'unknown',
   yellow: 'yellow',
   red: 'red',
+  unknown: 'unknown',
 } as const;
-
-export type TransformHealth = typeof TRANSFORM_HEALTH[keyof typeof TRANSFORM_HEALTH];
+export type TransformHealthStatus = keyof typeof TRANSFORM_HEALTH_STATUS;
+export const isTransformHealthStatus = (arg: unknown): arg is TransformHealthStatus =>
+  typeof arg === 'string' && Object.keys(TRANSFORM_HEALTH_STATUS).includes(arg);
+export const mapEsHealthStatus2TransformHealthStatus = (
+  healthStatus?: estypes.HealthStatus
+): TransformHealthStatus =>
+  typeof healthStatus === 'string' && isTransformHealthStatus(healthStatus.toLowerCase())
+    ? (healthStatus.toLowerCase() as TransformHealthStatus)
+    : TRANSFORM_HEALTH_STATUS.unknown;
 
 export const TRANSFORM_HEALTH_COLOR = {
   green: 'success',

x-pack/plugins/transform/common/types/transform_stats.ts

@@ -5,10 +5,11 @@
  * 2.0.
  */
 
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 
-import { type TransformHealth, type TransformState, TRANSFORM_STATE } from '../constants';
-import type { TransformId } from './transform';
+import { type TransformState, TRANSFORM_STATE } from '../constants';
 
 export interface TransformHealthIssue {
   type: string;
@@ -18,56 +19,12 @@ export interface TransformHealthIssue {
   first_occurrence?: number;
 }
 
-export interface TransformStats {
-  id: TransformId;
-  checkpointing: {
-    last: {
-      checkpoint: number;
-      timestamp_millis?: number;
-    };
-    next?: {
-      checkpoint: number;
-      checkpoint_progress?: {
-        total_docs: number;
-        docs_remaining: number;
-        percent_complete: number;
-      };
-    };
-    changes_last_detected_at: number;
-    last_search_time?: number;
-    operations_behind?: number;
-  };
-  health: {
-    status: TransformHealth;
-    issues?: TransformHealthIssue[];
-  };
-  node?: {
-    id: string;
-    name: string;
-    ephemeral_id: string;
-    transport_address: string;
-    attributes: Record<string, any>;
-  };
-  stats: {
-    delete_time_in_ms: number;
-    documents_deleted: number;
-    documents_indexed: number;
-    documents_processed: number;
-    index_failures: number;
-    index_time_in_ms: number;
-    index_total: number;
-    pages_processed: number;
-    search_failures: number;
-    search_time_in_ms: number;
-    search_total: number;
-    trigger_count: number;
-    processing_time_in_ms: number;
-    processing_total: number;
-    exponential_avg_checkpoint_duration_ms: number;
-    exponential_avg_documents_indexed: number;
-    exponential_avg_documents_processed: number;
-  };
-  reason?: string;
+export interface TransformHealth extends estypes.TransformGetTransformStatsTransformStatsHealth {
+  issues?: TransformHealthIssue[];
+}
+
+export interface TransformStats extends estypes.TransformGetTransformStatsTransformStats {
+  health?: TransformHealth;
   state: TransformState;
 }
 

x-pack/plugins/transform/public/app/common/reauthorization_utils.ts

@@ -7,14 +7,19 @@
 
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import type { TransformHealthIssue } from '../../../common/types/transform_stats';
-import { TRANSFORM_HEALTH } from '../../../common/constants';
+import {
+  mapEsHealthStatus2TransformHealthStatus,
+  TRANSFORM_HEALTH_STATUS,
+} from '../../../common/constants';
 import type { TransformListRow } from './transform_list';
 
 export const needsReauthorization = (transform: Partial<TransformListRow>) => {
   return (
     isPopulatedObject(transform.config?.authorization, ['api_key']) &&
     isPopulatedObject(transform.stats) &&
-    transform.stats.health.status === TRANSFORM_HEALTH.red &&
+    isPopulatedObject(transform.stats.health) &&
+    mapEsHealthStatus2TransformHealthStatus(transform.stats.health.status) ===
+      TRANSFORM_HEALTH_STATUS.red &&
     transform.stats.health.issues?.find(
       (issue) => (issue as TransformHealthIssue).issue === 'Privileges check failed'
     ) !== undefined

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx

@@ -25,6 +25,9 @@ import { stringHash } from '@kbn/ml-string-hash';
 import { isDefined } from '@kbn/ml-is-defined';
 
 import { FormattedMessage } from '@kbn/i18n-react';
+
+import { mapEsHealthStatus2TransformHealthStatus } from '../../../../../../common/constants';
+
 import { useEnabledFeatures } from '../../../../serverless_context';
 import { isTransformListRowWithStats } from '../../../../common/transform_list';
 import type { TransformHealthAlertRule } from '../../../../../../common/types/alerting';
@@ -154,7 +157,11 @@ export const ExpandedRow: FC<Props> = ({ item, onAlertEdit, transformsStatsLoadi
     if (item.stats.health !== undefined) {
       stateItems.push({
         title: 'health',
-        description: <TransformHealthColoredDot healthStatus={item.stats.health.status} />,
+        description: (
+          <TransformHealthColoredDot
+            healthStatus={mapEsHealthStatus2TransformHealthStatus(item.stats.health.status)}
+          />
+        ),
       });
     }
 

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx

@@ -17,6 +17,8 @@ import type {
   TransformStats,
 } from '../../../../../../common/types/transform_stats';
 
+import { mapEsHealthStatus2TransformHealthStatus } from '../../../../../../common/constants';
+
 import { TransformHealthColoredDot } from './transform_health_colored_dot';
 
 interface ExpandedRowHealthPaneProps {
@@ -24,7 +26,8 @@ interface ExpandedRowHealthPaneProps {
 }
 
 export const ExpandedRowHealthPane: FC<ExpandedRowHealthPaneProps> = ({ health }) => {
-  const { status, issues } = health;
+  const healthStatus = mapEsHealthStatus2TransformHealthStatus(health?.status);
+  const issues = health?.issues;
 
   const sorting = {
     sort: {
@@ -80,7 +83,7 @@ export const ExpandedRowHealthPane: FC<ExpandedRowHealthPaneProps> = ({ health }
       data-test-subj="transformHealthTabContent"
     >
       <EuiSpacer size="s" />
-      <TransformHealthColoredDot healthStatus={status} compact={false} />
+      <TransformHealthColoredDot healthStatus={healthStatus} compact={false} />
       {Array.isArray(issues) && issues.length > 0 && (
         <>
           <EuiSpacer size="s" />

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_health_colored_dot.tsx

@@ -10,14 +10,14 @@ import React, { type FC } from 'react';
 import { EuiHealth, EuiToolTip } from '@elastic/eui';
 
 import {
-  type TransformHealth,
+  type TransformHealthStatus,
   TRANSFORM_HEALTH_COLOR,
   TRANSFORM_HEALTH_DESCRIPTION,
   TRANSFORM_HEALTH_LABEL,
 } from '../../../../../../common/constants';
 
 interface TransformHealthProps {
-  healthStatus: TransformHealth;
+  healthStatus: TransformHealthStatus;
   compact?: boolean;
   showToolTip?: boolean;
 }

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx

@@ -14,7 +14,7 @@ import {
   TRANSFORM_FUNCTION,
   TRANSFORM_MODE,
   TRANSFORM_STATE,
-  TRANSFORM_HEALTH,
+  TRANSFORM_HEALTH_STATUS,
 } from '../../../../../../common/constants';
 import { isLatestTransform, isPivotTransform } from '../../../../../../common/types/transform';
 import type { TransformListRow } from '../../../../common';
@@ -53,7 +53,7 @@ export const transformFilters: SearchFilterConfig[] = [
     field: 'health',
     name: i18n.translate('xpack.transform.healthFilter', { defaultMessage: 'Health' }),
     multiSelect: false,
-    options: Object.values(TRANSFORM_HEALTH).map((val) => ({
+    options: Object.values(TRANSFORM_HEALTH_STATUS).map((val) => ({
       value: val,
       name: val,
       view: <TransformHealthColoredDot compact={true} showToolTip={false} healthStatus={val} />,

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx

@@ -31,7 +31,10 @@ import { useTransformCapabilities } from '../../../../hooks';
 import { needsReauthorization } from '../../../../common/reauthorization_utils';
 import type { TransformId } from '../../../../../../common/types/transform';
 import { isLatestTransform, isPivotTransform } from '../../../../../../common/types/transform';
-import { TRANSFORM_STATE } from '../../../../../../common/constants';
+import {
+  mapEsHealthStatus2TransformHealthStatus,
+  TRANSFORM_STATE,
+} from '../../../../../../common/constants';
 
 import type { TransformListRow } from '../../../../common';
 import { getTransformProgress, TRANSFORM_LIST_COLUMN } from '../../../../common';
@@ -349,11 +352,13 @@ export const useColumns = (
     {
       name: i18n.translate('xpack.transform.health', { defaultMessage: 'Health' }),
       'data-test-subj': 'transformListColumnHealth',
-      sortable: (item: TransformListRow) => item.stats?.health.status,
+      sortable: (item: TransformListRow) => item.stats?.health?.status,
       truncateText: true,
       render(item: TransformListRow) {
-        return item.stats ? (
-          <TransformHealthColoredDot healthStatus={item.stats.health.status} />
+        return item.stats?.health ? (
+          <TransformHealthColoredDot
+            healthStatus={mapEsHealthStatus2TransformHealthStatus(item.stats.health.status)}
+          />
         ) : (
           <NoStatsFallbackComponent />
         );

x-pack/plugins/transform/server/lib/alerting/transform_health_rule_type/register_transform_health_rule_type.ts

@@ -27,7 +27,7 @@ import { ALERT_REASON } from '@kbn/rule-data-utils';
 import { ES_FIELD_TYPES } from '@kbn/field-types';
 import {
   PLUGIN,
-  type TransformHealth,
+  type TransformHealthStatus,
   TRANSFORM_RULE_TYPE,
   TRANSFORM_HEALTH_RESULTS,
 } from '../../../../common/constants';
@@ -38,7 +38,7 @@ import { transformHealthServiceProvider } from './transform_health_service';
 export interface BaseTransformAlertResponse {
   transform_id: string;
   description?: string;
-  health_status: TransformHealth;
+  health_status: TransformHealthStatus;
   issues?: Array<{ issue: string; details?: string; count: number; first_occurrence?: string }>;
 }
 

x-pack/plugins/transform/server/lib/alerting/transform_health_rule_type/transform_health_service.ts

@@ -13,8 +13,10 @@ import type { RulesClient } from '@kbn/alerting-plugin/server';
 import type { FieldFormatsRegistry } from '@kbn/field-formats-plugin/common';
 import { FIELD_FORMAT_IDS } from '@kbn/field-formats-plugin/common';
 import type { TransformStats } from '../../../../common/types/transform_stats';
+import { TRANSFORM_HEALTH_STATUS } from '../../../../common/constants';
 import type { TransformHealthRuleParams } from './schema';
 import {
+  mapEsHealthStatus2TransformHealthStatus,
   ALL_TRANSFORMS_SELECTION,
   TRANSFORM_HEALTH_CHECK_NAMES,
   TRANSFORM_NOTIFICATIONS_INDEX,
@@ -116,8 +118,8 @@ export function transformHealthServiceProvider({
       description: transformsDict.get(transformStats.id)?.description,
       transform_state: transformStats.state,
       node_name: transformStats.node?.name,
-      health_status: transformStats.health.status,
-      ...(transformStats.health.issues
+      health_status: mapEsHealthStatus2TransformHealthStatus(transformStats.health?.status),
+      ...(transformStats.health?.issues
         ? {
             issues: transformStats.health.issues.map((issue) => {
               return {
@@ -238,7 +240,11 @@ export function transformHealthServiceProvider({
       const transformsStats = await getTransformStats(transformIds);
 
       return transformsStats
-        .filter((t) => t.health.status !== 'green')
+        .filter(
+          (t) =>
+            mapEsHealthStatus2TransformHealthStatus(t.health?.status) !==
+            TRANSFORM_HEALTH_STATUS.green
+        )
         .map(baseTransformAlertResponseFormatter);
     },
     /**

x-pack/test/api_integration/apis/transform/reauthorize_transforms.ts

@@ -72,13 +72,13 @@ export default ({ getService }: FtrProviderContext) => {
         TRANSFORM_STATE.STOPPED,
         `Expected transform state of ${transformId} to be '${TRANSFORM_STATE.STOPPED}' (got ${stats.state})`
       );
-      expect(stats.health.status).to.eql(
+      expect(stats.health?.status).to.eql(
         'red',
-        `Expected transform health status of ${transformId} to be 'red' (got ${stats.health.status})`
+        `Expected transform health status of ${transformId} to be 'red' (got ${stats.health?.status})`
       );
-      expect(stats.health.issues![0].type).to.eql(
+      expect(stats.health?.issues![0].type).to.eql(
         'privileges_check_failed',
-        `Expected transform health issue of ${transformId} to be 'privileges_check_failed' (got ${stats.health.status})`
+        `Expected transform health issue of ${transformId} to be 'privileges_check_failed' (got ${stats.health?.status})`
       );
     }
 
@@ -94,9 +94,9 @@ export default ({ getService }: FtrProviderContext) => {
         )})`
       );
       const stats = await transform.api.getTransformStats(transformId);
-      expect(stats.health.status).to.eql(
+      expect(stats.health?.status).to.eql(
         'green',
-        `Expected transform health status of ${transformId} to be 'green' (got ${stats.health.status})`
+        `Expected transform health status of ${transformId} to be 'green' (got ${stats.health?.status})`
       );
     }
 

x-pack/test/functional/apps/transform/actions/starting.ts

@@ -7,7 +7,7 @@
 
 import {
   TRANSFORM_STATE,
-  TRANSFORM_HEALTH,
+  TRANSFORM_HEALTH_STATUS,
   TRANSFORM_HEALTH_LABEL,
   TRANSFORM_HEALTH_DESCRIPTION,
 } from '@kbn/transform-plugin/common/constants';
@@ -59,7 +59,7 @@ export default function ({ getService }: FtrProviderContext) {
         expected: {
           healthDescription: TRANSFORM_HEALTH_DESCRIPTION.green,
           healthLabel: TRANSFORM_HEALTH_LABEL.green,
-          healthStatus: TRANSFORM_HEALTH.green,
+          healthStatus: TRANSFORM_HEALTH_STATUS.green,
         },
       },
       {
@@ -70,7 +70,7 @@ export default function ({ getService }: FtrProviderContext) {
         expected: {
           healthDescription: TRANSFORM_HEALTH_DESCRIPTION.green,
           healthLabel: TRANSFORM_HEALTH_LABEL.green,
-          healthStatus: TRANSFORM_HEALTH.green,
+          healthStatus: TRANSFORM_HEALTH_STATUS.green,
         },
       },
       {
@@ -81,7 +81,7 @@ export default function ({ getService }: FtrProviderContext) {
         expected: {
           healthDescription: TRANSFORM_HEALTH_DESCRIPTION.yellow,
           healthLabel: TRANSFORM_HEALTH_LABEL.yellow,
-          healthStatus: TRANSFORM_HEALTH.yellow,
+          healthStatus: TRANSFORM_HEALTH_STATUS.yellow,
         },
       },
       {
@@ -92,7 +92,7 @@ export default function ({ getService }: FtrProviderContext) {
         expected: {
           healthDescription: TRANSFORM_HEALTH_DESCRIPTION.green,
           healthLabel: TRANSFORM_HEALTH_LABEL.green,
-          healthStatus: TRANSFORM_HEALTH.green,
+          healthStatus: TRANSFORM_HEALTH_STATUS.green,
         },
       },
       {
@@ -103,7 +103,7 @@ export default function ({ getService }: FtrProviderContext) {
         expected: {
           healthDescription: TRANSFORM_HEALTH_DESCRIPTION.green,
           healthLabel: TRANSFORM_HEALTH_LABEL.green,
-          healthStatus: TRANSFORM_HEALTH.green,
+          healthStatus: TRANSFORM_HEALTH_STATUS.green,
         },
       },
     ];
@@ -114,7 +114,7 @@ export default function ({ getService }: FtrProviderContext) {
 
       for (const testData of testDataList) {
         if (
-          testData.expected.healthStatus === TRANSFORM_HEALTH.yellow &&
+          testData.expected.healthStatus === TRANSFORM_HEALTH_STATUS.yellow &&
           testData.type === 'pivot'
         ) {
           testData.originalConfig.pivot.aggregations['products.base_price.fail'] = {
@@ -126,7 +126,7 @@ export default function ({ getService }: FtrProviderContext) {
           };
         }
         await transform.api.createTransform(testData.originalConfig.id, testData.originalConfig, {
-          deferValidation: testData.expected.healthStatus === TRANSFORM_HEALTH.yellow,
+          deferValidation: testData.expected.healthStatus === TRANSFORM_HEALTH_STATUS.yellow,
         });
       }
       await transform.testResources.setKibanaTimeZoneToUTC();
@@ -167,7 +167,7 @@ export default function ({ getService }: FtrProviderContext) {
 
           await transform.table.assertTransformExpandedRowHealth(
             testData.expected.healthDescription,
-            testData.expected.healthStatus !== TRANSFORM_HEALTH.green
+            testData.expected.healthStatus !== TRANSFORM_HEALTH_STATUS.green
           );
 
           await transform.table.clearSearchString(testDataList.length);