[Transform] Add support for latest function (#85784)

* [Transform] add latest to the schema definition

* [ML] update interfaces, add guards

* [Transform] WIP support latest function

* [Transform] fix request with missing_bucket after merge

* [Transform] fix error in useDeleteTransforms

* [Transform] fix types and fields sorting for pivot

* [Transform] fix types and jest tests

* [Transform] fix error shape

* [Transform] fixed card width, change description

* [Transform] fixed API integration tests

* [Transform] fix config mapper

* [Transform] improve wizard performance

x-pack/plugins/transform/common/api_schemas/common.ts

@@ -40,7 +40,13 @@ export type TransformIdParamSchema = TypeOf<typeof transformIdParamSchema>;
 
 export interface ResponseStatus {
   success: boolean;
-  error?: any;
+  // FIXME error response should have unified shape
+  error?: {
+    type: string;
+    reason: string;
+    root_cause: any[];
+    caused_by: any;
+  } & { response: any };
 }
 
 export interface CommonResponseStatusSchema {

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -35,19 +35,32 @@ export const destSchema = schema.object({
   index: schema.string(),
   pipeline: schema.maybe(schema.string()),
 });
+
 export const pivotSchema = schema.object({
   group_by: schema.any(),
   aggregations: schema.any(),
 });
+
+export const latestFunctionSchema = schema.object({
+  unique_key: schema.arrayOf(schema.string()),
+  sort: schema.string(),
+});
+
+export type PivotConfig = TypeOf<typeof pivotSchema>;
+
+export type LatestFunctionConfig = TypeOf<typeof latestFunctionSchema>;
+
 export const settingsSchema = schema.object({
   max_page_search_size: schema.maybe(schema.number()),
   // The default value is null, which disables throttling.
   docs_per_second: schema.maybe(schema.nullable(schema.number())),
 });
+
 export const sourceSchema = schema.object({
   index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
   query: schema.maybe(schema.recordOf(schema.string(), schema.any())),
 });
+
 export const syncSchema = schema.object({
   time: schema.object({
     delay: schema.maybe(schema.string()),
@@ -55,24 +68,52 @@ export const syncSchema = schema.object({
   }),
 });
 
-// PUT transforms/{transformId}
-export const putTransformsRequestSchema = schema.object({
-  description: schema.maybe(schema.string()),
-  dest: destSchema,
-  frequency: schema.maybe(schema.string()),
-  pivot: pivotSchema,
-  settings: schema.maybe(settingsSchema),
-  source: sourceSchema,
-  sync: schema.maybe(syncSchema),
-});
+function transformConfigPayloadValidator<
+  T extends { pivot?: PivotConfig; latest?: LatestFunctionConfig }
+>(value: T) {
+  if (!value.pivot && !value.latest) {
+    return 'pivot or latest is required for transform configuration';
+  }
+  if (value.pivot && value.latest) {
+    return 'pivot and latest are not allowed together';
+  }
+}
 
-export interface PutTransformsRequestSchema extends TypeOf<typeof putTransformsRequestSchema> {
+// PUT transforms/{transformId}
+export const putTransformsRequestSchema = schema.object(
+  {
+    description: schema.maybe(schema.string()),
+    dest: destSchema,
+    frequency: schema.maybe(schema.string()),
+    /**
+     * Pivot and latest are mutually exclusive, i.e. exactly one must be specified in the transform configuration
+     */
+    pivot: schema.maybe(pivotSchema),
+    /**
+     * Latest and pivot are mutually exclusive, i.e. exactly one must be specified in the transform configuration
+     */
+    latest: schema.maybe(latestFunctionSchema),
+    settings: schema.maybe(settingsSchema),
+    source: sourceSchema,
+    sync: schema.maybe(syncSchema),
+  },
+  {
+    validate: transformConfigPayloadValidator,
+  }
+);
+
+export type PutTransformsRequestSchema = TypeOf<typeof putTransformsRequestSchema>;
+
+export interface PutTransformsPivotRequestSchema
+  extends Omit<PutTransformsRequestSchema, 'latest'> {
   pivot: {
     group_by: PivotGroupByDict;
     aggregations: PivotAggDict;
   };
 }
 
+export type PutTransformsLatestRequestSchema = Omit<PutTransformsRequestSchema, 'pivot'>;
+
 interface TransformCreated {
   transform: TransformId;
 }
@@ -86,18 +127,30 @@ export interface PutTransformsResponseSchema {
 }
 
 // POST transforms/_preview
-export const postTransformsPreviewRequestSchema = schema.object({
-  pivot: pivotSchema,
-  source: sourceSchema,
-});
+export const postTransformsPreviewRequestSchema = schema.object(
+  {
+    pivot: schema.maybe(pivotSchema),
+    latest: schema.maybe(latestFunctionSchema),
+    source: sourceSchema,
+  },
+  {
+    validate: transformConfigPayloadValidator,
+  }
+);
 
-export interface PostTransformsPreviewRequestSchema
-  extends TypeOf<typeof postTransformsPreviewRequestSchema> {
+export type PostTransformsPreviewRequestSchema = TypeOf<typeof postTransformsPreviewRequestSchema>;
+
+export type PivotTransformPreviewRequestSchema = Omit<
+  PostTransformsPreviewRequestSchema,
+  'latest'
+> & {
   pivot: {
     group_by: PivotGroupByDict;
     aggregations: PivotAggDict;
   };
-}
+};
+
+export type LatestTransformPreviewRequestSchema = Omit<PostTransformsPreviewRequestSchema, 'pivot'>;
 
 interface EsMappingType {
   type: ES_FIELD_TYPES;

x-pack/plugins/transform/common/constants.ts

@@ -96,3 +96,10 @@ export const TRANSFORM_MODE = {
 
 const transformModes = Object.values(TRANSFORM_MODE);
 export type TransformMode = typeof transformModes[number];
+
+export const TRANSFORM_FUNCTION = {
+  PIVOT: 'pivot',
+  LATEST: 'latest',
+} as const;
+
+export type TransformFunction = typeof TRANSFORM_FUNCTION[keyof typeof TRANSFORM_FUNCTION];

x-pack/plugins/transform/common/types/transform.ts

@@ -4,14 +4,58 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import type { PutTransformsRequestSchema } from '../api_schemas/transforms';
+import { EuiComboBoxOptionOption } from '@elastic/eui/src/components/combo_box/types';
+import type { LatestFunctionConfig, PutTransformsRequestSchema } from '../api_schemas/transforms';
+import { PivotGroupByDict } from './pivot_group_by';
+import { PivotAggDict } from './pivot_aggs';
 
 export type IndexName = string;
 export type IndexPattern = string;
 export type TransformId = string;
 
-export interface TransformPivotConfig extends PutTransformsRequestSchema {
+/**
+ * Generic type for transform response
+ */
+export type TransformBaseConfig = PutTransformsRequestSchema & {
   id: TransformId;
   create_time?: number;
   version?: string;
+};
+
+export interface PivotConfigDefinition {
+  group_by: PivotGroupByDict;
+  aggregations: PivotAggDict;
+}
+
+/**
+ * Transform with pivot configuration
+ */
+export type TransformPivotConfig = Omit<TransformBaseConfig, 'latest'> & {
+  pivot: PivotConfigDefinition;
+};
+
+/**
+ * Transform with latest function configuration
+ */
+export type TransformLatestConfig = Omit<TransformBaseConfig, 'pivot'> & {
+  latest: LatestFunctionConfig;
+};
+
+export type TransformConfigUnion = TransformPivotConfig | TransformLatestConfig;
+
+export function isPivotTransform(
+  transform: TransformBaseConfig
+): transform is TransformPivotConfig {
+  return transform.hasOwnProperty('pivot');
+}
+
+export function isLatestTransform(
+  transform: TransformBaseConfig
+): transform is TransformLatestConfig {
+  return transform.hasOwnProperty('latest');
+}
+
+export interface LatestFunctionConfigUI {
+  unique_key: Array<EuiComboBoxOptionOption<string>> | undefined;
+  sort: EuiComboBoxOptionOption<string> | undefined;
 }