[Transform] Add support for latest function (#85784)

* [Transform] add latest to the schema definition

* [ML] update interfaces, add guards

* [Transform] WIP support latest function

* [Transform] fix request with missing_bucket after merge

* [Transform] fix error in useDeleteTransforms

* [Transform] fix types and fields sorting for pivot

* [Transform] fix types and jest tests

* [Transform] fix error shape

* [Transform] fixed card width, change description

* [Transform] fixed API integration tests

* [Transform] fix config mapper

* [Transform] improve wizard performance

x-pack/plugins/transform/common/api_schemas/common.ts

@@ -40,7 +40,13 @@ export type TransformIdParamSchema = TypeOf<typeof transformIdParamSchema>;
 
 export interface ResponseStatus {
   success: boolean;
-  error?: any;
+  // FIXME error response should have unified shape
+  error?: {
+    type: string;
+    reason: string;
+    root_cause: any[];
+    caused_by: any;
+  } & { response: any };
 }
 
 export interface CommonResponseStatusSchema {

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -35,19 +35,32 @@ export const destSchema = schema.object({
   index: schema.string(),
   pipeline: schema.maybe(schema.string()),
 });
+
 export const pivotSchema = schema.object({
   group_by: schema.any(),
   aggregations: schema.any(),
 });
+
+export const latestFunctionSchema = schema.object({
+  unique_key: schema.arrayOf(schema.string()),
+  sort: schema.string(),
+});
+
+export type PivotConfig = TypeOf<typeof pivotSchema>;
+
+export type LatestFunctionConfig = TypeOf<typeof latestFunctionSchema>;
+
 export const settingsSchema = schema.object({
   max_page_search_size: schema.maybe(schema.number()),
   // The default value is null, which disables throttling.
   docs_per_second: schema.maybe(schema.nullable(schema.number())),
 });
+
 export const sourceSchema = schema.object({
   index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
   query: schema.maybe(schema.recordOf(schema.string(), schema.any())),
 });
+
 export const syncSchema = schema.object({
   time: schema.object({
     delay: schema.maybe(schema.string()),
@@ -55,24 +68,52 @@ export const syncSchema = schema.object({
   }),
 });
 
-// PUT transforms/{transformId}
+function transformConfigPayloadValidator<
-export const putTransformsRequestSchema = schema.object({
+  T extends { pivot?: PivotConfig; latest?: LatestFunctionConfig }
-  description: schema.maybe(schema.string()),
+>(value: T) {
-  dest: destSchema,
+  if (!value.pivot && !value.latest) {
-  frequency: schema.maybe(schema.string()),
+    return 'pivot or latest is required for transform configuration';
-  pivot: pivotSchema,
+  }
-  settings: schema.maybe(settingsSchema),
+  if (value.pivot && value.latest) {
-  source: sourceSchema,
+    return 'pivot and latest are not allowed together';
-  sync: schema.maybe(syncSchema),
+  }
-});
+}
 
-export interface PutTransformsRequestSchema extends TypeOf<typeof putTransformsRequestSchema> {
+// PUT transforms/{transformId}
+export const putTransformsRequestSchema = schema.object(
+  {
+    description: schema.maybe(schema.string()),
+    dest: destSchema,
+    frequency: schema.maybe(schema.string()),
+    /**
+     * Pivot and latest are mutually exclusive, i.e. exactly one must be specified in the transform configuration
+     */
+    pivot: schema.maybe(pivotSchema),
+    /**
+     * Latest and pivot are mutually exclusive, i.e. exactly one must be specified in the transform configuration
+     */
+    latest: schema.maybe(latestFunctionSchema),
+    settings: schema.maybe(settingsSchema),
+    source: sourceSchema,
+    sync: schema.maybe(syncSchema),
+  },
+  {
+    validate: transformConfigPayloadValidator,
+  }
+);
+
+export type PutTransformsRequestSchema = TypeOf<typeof putTransformsRequestSchema>;
+
+export interface PutTransformsPivotRequestSchema
+  extends Omit<PutTransformsRequestSchema, 'latest'> {
   pivot: {
     group_by: PivotGroupByDict;
     aggregations: PivotAggDict;
   };
 }
 
+export type PutTransformsLatestRequestSchema = Omit<PutTransformsRequestSchema, 'pivot'>;
+
 interface TransformCreated {
   transform: TransformId;
 }
@@ -86,18 +127,30 @@ export interface PutTransformsResponseSchema {
 }
 
 // POST transforms/_preview
-export const postTransformsPreviewRequestSchema = schema.object({
+export const postTransformsPreviewRequestSchema = schema.object(
-  pivot: pivotSchema,
+  {
-  source: sourceSchema,
+    pivot: schema.maybe(pivotSchema),
-});
+    latest: schema.maybe(latestFunctionSchema),
+    source: sourceSchema,
+  },
+  {
+    validate: transformConfigPayloadValidator,
+  }
+);
 
-export interface PostTransformsPreviewRequestSchema
+export type PostTransformsPreviewRequestSchema = TypeOf<typeof postTransformsPreviewRequestSchema>;
-  extends TypeOf<typeof postTransformsPreviewRequestSchema> {
+
+export type PivotTransformPreviewRequestSchema = Omit<
+  PostTransformsPreviewRequestSchema,
+  'latest'
+> & {
   pivot: {
     group_by: PivotGroupByDict;
     aggregations: PivotAggDict;
   };
-}
+};
+
+export type LatestTransformPreviewRequestSchema = Omit<PostTransformsPreviewRequestSchema, 'pivot'>;
 
 interface EsMappingType {
   type: ES_FIELD_TYPES;

x-pack/plugins/transform/common/constants.ts

@@ -96,3 +96,10 @@ export const TRANSFORM_MODE = {
 
 const transformModes = Object.values(TRANSFORM_MODE);
 export type TransformMode = typeof transformModes[number];
+
+export const TRANSFORM_FUNCTION = {
+  PIVOT: 'pivot',
+  LATEST: 'latest',
+} as const;
+
+export type TransformFunction = typeof TRANSFORM_FUNCTION[keyof typeof TRANSFORM_FUNCTION];

x-pack/plugins/transform/common/types/transform.ts

@@ -4,14 +4,58 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import type { PutTransformsRequestSchema } from '../api_schemas/transforms';
+import { EuiComboBoxOptionOption } from '@elastic/eui/src/components/combo_box/types';
+import type { LatestFunctionConfig, PutTransformsRequestSchema } from '../api_schemas/transforms';
+import { PivotGroupByDict } from './pivot_group_by';
+import { PivotAggDict } from './pivot_aggs';
 
 export type IndexName = string;
 export type IndexPattern = string;
 export type TransformId = string;
 
-export interface TransformPivotConfig extends PutTransformsRequestSchema {
+/**
+ * Generic type for transform response
+ */
+export type TransformBaseConfig = PutTransformsRequestSchema & {
   id: TransformId;
   create_time?: number;
   version?: string;
+};
+
+export interface PivotConfigDefinition {
+  group_by: PivotGroupByDict;
+  aggregations: PivotAggDict;
+}
+
+/**
+ * Transform with pivot configuration
+ */
+export type TransformPivotConfig = Omit<TransformBaseConfig, 'latest'> & {
+  pivot: PivotConfigDefinition;
+};
+
+/**
+ * Transform with latest function configuration
+ */
+export type TransformLatestConfig = Omit<TransformBaseConfig, 'pivot'> & {
+  latest: LatestFunctionConfig;
+};
+
+export type TransformConfigUnion = TransformPivotConfig | TransformLatestConfig;
+
+export function isPivotTransform(
+  transform: TransformBaseConfig
+): transform is TransformPivotConfig {
+  return transform.hasOwnProperty('pivot');
+}
+
+export function isLatestTransform(
+  transform: TransformBaseConfig
+): transform is TransformLatestConfig {
+  return transform.hasOwnProperty('latest');
+}
+
+export interface LatestFunctionConfigUI {
+  unique_key: Array<EuiComboBoxOptionOption<string>> | undefined;
+  sort: EuiComboBoxOptionOption<string> | undefined;
 }

x-pack/plugins/transform/public/app/common/data_grid.test.ts

@@ -4,15 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { PIVOT_SUPPORTED_AGGS } from '../../../common/types/pivot_aggs';
+import { getPreviewTransformRequestBody, SimpleQuery } from '../common';
-
-import {
-  getPreviewTransformRequestBody,
-  PivotAggsConfig,
-  PivotGroupByConfig,
-  PIVOT_SUPPORTED_GROUP_BY_AGGS,
-  SimpleQuery,
-} from '../common';
 
 import { getIndexDevConsoleStatement, getPivotPreviewDevConsoleStatement } from './data_grid';
 
@@ -24,24 +16,26 @@ describe('Transform: Data Grid', () => {
         default_operator: 'AND',
       },
     };
-    const groupBy: PivotGroupByConfig = {
+
-      agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS,
+    const request = getPreviewTransformRequestBody('the-index-pattern-title', query, {
-      field: 'the-group-by-field',
+      pivot: {
-      aggName: 'the-group-by-agg-name',
+        group_by: {
-      dropDownName: 'the-group-by-drop-down-name',
+          'the-group-by-agg-name': {
-    };
+            terms: {
-    const agg: PivotAggsConfig = {
+              field: 'the-group-by-field',
-      agg: PIVOT_SUPPORTED_AGGS.AVG,
+            },
-      field: 'the-agg-field',
+          },
-      aggName: 'the-agg-agg-name',
+        },
-      dropDownName: 'the-agg-drop-down-name',
+        aggregations: {
-    };
+          'the-agg-agg-name': {
-    const request = getPreviewTransformRequestBody(
+            avg: {
-      'the-index-pattern-title',
+              field: 'the-agg-field',
-      query,
+            },
-      [groupBy],
+          },
-      [agg]
+        },
-    );
+      },
+    });
+
     const pivotPreviewDevConsoleStatement = getPivotPreviewDevConsoleStatement(request);
 
     expect(pivotPreviewDevConsoleStatement).toBe(`POST _transform/_preview

x-pack/plugins/transform/public/app/common/request.test.ts

@@ -18,7 +18,6 @@ import {
   getPreviewTransformRequestBody,
   getCreateTransformRequestBody,
   getCreateTransformSettingsRequestBody,
-  getMissingBucketConfig,
   getPivotQuery,
   isDefaultQuery,
   isMatchAllQuery,
@@ -26,6 +25,7 @@ import {
   matchAllQuery,
   PivotQuery,
 } from './request';
+import { LatestFunctionConfigUI } from '../../../common/types/transform';
 
 const simpleQuery: PivotQuery = { query_string: { query: 'airline:AAL' } };
 
@@ -62,16 +62,6 @@ describe('Transform: Common', () => {
     expect(isDefaultQuery(simpleQuery)).toBe(false);
   });
 
-  test('getMissingBucketConfig()', () => {
-    expect(getMissingBucketConfig(groupByTerms)).toEqual({});
-    expect(getMissingBucketConfig({ ...groupByTerms, ...{ missing_bucket: true } })).toEqual({
-      missing_bucket: true,
-    });
-    expect(getMissingBucketConfig({ ...groupByTerms, ...{ missing_bucket: false } })).toEqual({
-      missing_bucket: false,
-    });
-  });
-
   test('getPivotQuery()', () => {
     const query = getPivotQuery('the-query');
 
@@ -85,9 +75,13 @@ describe('Transform: Common', () => {
 
   test('getPreviewTransformRequestBody()', () => {
     const query = getPivotQuery('the-query');
-    const groupBy: PivotGroupByConfig[] = [groupByTerms];
+
-    const aggs: PivotAggsConfig[] = [aggsAvg];
+    const request = getPreviewTransformRequestBody('the-index-pattern-title', query, {
-    const request = getPreviewTransformRequestBody('the-index-pattern-title', query, groupBy, aggs);
+      pivot: {
+        aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } },
+        group_by: { 'the-group-by-agg-name': { terms: { field: 'the-group-by-field' } } },
+      },
+    });
 
     expect(request).toEqual({
       pivot: {
@@ -103,13 +97,15 @@ describe('Transform: Common', () => {
 
   test('getPreviewTransformRequestBody() with comma-separated index pattern', () => {
     const query = getPivotQuery('the-query');
-    const groupBy: PivotGroupByConfig[] = [groupByTerms];
-    const aggs: PivotAggsConfig[] = [aggsAvg];
     const request = getPreviewTransformRequestBody(
       'the-index-pattern-title,the-other-title',
       query,
-      groupBy,
+      {
-      aggs
+        pivot: {
+          aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } },
+          group_by: { 'the-group-by-agg-name': { terms: { field: 'the-group-by-field' } } },
+        },
+      }
     );
 
     expect(request).toEqual({
@@ -126,9 +122,14 @@ describe('Transform: Common', () => {
 
   test('getPreviewTransformRequestBody() with missing_buckets config', () => {
     const query = getPivotQuery('the-query');
-    const groupBy: PivotGroupByConfig[] = [{ ...groupByTerms, ...{ missing_bucket: true } }];
+    const request = getPreviewTransformRequestBody('the-index-pattern-title', query, {
-    const aggs: PivotAggsConfig[] = [aggsAvg];
+      pivot: {
-    const request = getPreviewTransformRequestBody('the-index-pattern-title', query, groupBy, aggs);
+        aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } },
+        group_by: {
+          'the-group-by-agg-name': { terms: { field: 'the-group-by-field', missing_bucket: true } },
+        },
+      },
+    });
 
     expect(request).toEqual({
       pivot: {
@@ -155,6 +156,17 @@ describe('Transform: Common', () => {
       searchString: 'the-query',
       searchQuery: 'the-search-query',
       valid: true,
+      transformFunction: 'pivot',
+      latestConfig: {} as LatestFunctionConfigUI,
+      previewRequest: {
+        pivot: {
+          aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } },
+          group_by: { 'the-group-by-agg-name': { terms: { field: 'the-group-by-field' } } },
+        },
+      },
+      validationStatus: {
+        isValid: true,
+      },
     };
     const transformDetailsState: StepDetailsExposedState = {
       continuousModeDateField: 'the-continuous-mode-date-field',

x-pack/plugins/transform/public/app/common/request.ts

@@ -11,31 +11,15 @@ import type { IndexPattern } from '../../../../../../src/plugins/data/public';
 
 import type {
   PostTransformsPreviewRequestSchema,
+  PutTransformsLatestRequestSchema,
+  PutTransformsPivotRequestSchema,
   PutTransformsRequestSchema,
 } from '../../../common/api_schemas/transforms';
-import type {
-  DateHistogramAgg,
-  HistogramAgg,
-  TermsAgg,
-} from '../../../common/types/pivot_group_by';
-import { dictionaryToArray } from '../../../common/types/common';
 
 import type { SavedSearchQuery } from '../hooks/use_search_items';
 import type { StepDefineExposedState } from '../sections/create_transform/components/step_define';
 import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details/step_details_form';
 
-import {
-  getEsAggFromAggConfig,
-  getEsAggFromGroupByConfig,
-  isGroupByDateHistogram,
-  isGroupByHistogram,
-  isGroupByTerms,
-  GroupByConfigWithUiSupport,
-  PivotGroupByConfig,
-} from '../common';
-
-import { PivotAggsConfig } from './pivot_aggs';
-
 export interface SimpleQuery {
   query_string: {
     query: string;
@@ -72,72 +56,20 @@ export function isDefaultQuery(query: PivotQuery): boolean {
   return isSimpleQuery(query) && query.query_string.query === '*';
 }
 
-export const getMissingBucketConfig = (
-  g: GroupByConfigWithUiSupport
-): { missing_bucket?: boolean } => {
-  return g.missing_bucket !== undefined ? { missing_bucket: g.missing_bucket } : {};
-};
-
 export function getPreviewTransformRequestBody(
   indexPatternTitle: IndexPattern['title'],
   query: PivotQuery,
-  groupBy: PivotGroupByConfig[],
+  partialRequest?: StepDefineExposedState['previewRequest'] | undefined
-  aggs: PivotAggsConfig[]
 ): PostTransformsPreviewRequestSchema {
   const index = indexPatternTitle.split(',').map((name: string) => name.trim());
 
-  const request: PostTransformsPreviewRequestSchema = {
+  return {
     source: {
       index,
       ...(!isDefaultQuery(query) && !isMatchAllQuery(query) ? { query } : {}),
     },
-    pivot: {
+    ...(partialRequest ?? {}),
-      group_by: {},
-      aggregations: {},
-    },
   };
-
-  groupBy.forEach((g) => {
-    if (isGroupByTerms(g)) {
-      const termsAgg: TermsAgg = {
-        terms: {
-          field: g.field,
-          ...getMissingBucketConfig(g),
-        },
-      };
-      request.pivot.group_by[g.aggName] = termsAgg;
-    } else if (isGroupByHistogram(g)) {
-      const histogramAgg: HistogramAgg = {
-        histogram: {
-          field: g.field,
-          interval: g.interval,
-          ...getMissingBucketConfig(g),
-        },
-      };
-      request.pivot.group_by[g.aggName] = histogramAgg;
-    } else if (isGroupByDateHistogram(g)) {
-      const dateHistogramAgg: DateHistogramAgg = {
-        date_histogram: {
-          field: g.field,
-          calendar_interval: g.calendar_interval,
-          ...getMissingBucketConfig(g),
-        },
-      };
-      request.pivot.group_by[g.aggName] = dateHistogramAgg;
-    } else {
-      request.pivot.group_by[g.aggName] = getEsAggFromGroupByConfig(g);
-    }
-  });
-
-  aggs.forEach((agg) => {
-    const result = getEsAggFromAggConfig(agg);
-    if (result === null) {
-      return;
-    }
-    request.pivot.aggregations[agg.aggName] = result;
-  });
-
-  return request;
 }
 
 export const getCreateTransformSettingsRequestBody = (
@@ -158,12 +90,11 @@ export const getCreateTransformRequestBody = (
   indexPatternTitle: IndexPattern['title'],
   pivotState: StepDefineExposedState,
   transformDetailsState: StepDetailsExposedState
-): PutTransformsRequestSchema => ({
+): PutTransformsPivotRequestSchema | PutTransformsLatestRequestSchema => ({
   ...getPreviewTransformRequestBody(
     indexPatternTitle,
     getPivotQuery(pivotState.searchQuery),
-    dictionaryToArray(pivotState.groupByList),
+    pivotState.previewRequest
-    dictionaryToArray(pivotState.aggList)
   ),
   // conditionally add optional description
   ...(transformDetailsState.transformDescription !== ''

x-pack/plugins/transform/public/app/common/transform_list.ts

@@ -6,7 +6,7 @@
 
 import { EuiTableActionsColumnType } from '@elastic/eui';
 
-import { TransformId, TransformPivotConfig } from '../../../common/types/transform';
+import { TransformConfigUnion, TransformId } from '../../../common/types/transform';
 import { TransformStats } from '../../../common/types/transform_stats';
 
 // Used to pass on attribute names to table columns
@@ -17,7 +17,7 @@ export enum TRANSFORM_LIST_COLUMN {
 
 export interface TransformListRow {
   id: TransformId;
-  config: TransformPivotConfig;
+  config: TransformConfigUnion;
   mode?: string; // added property on client side to allow filtering by this field
   stats: TransformStats;
 }

x-pack/plugins/transform/public/app/common/transform_stats.ts

@@ -7,8 +7,19 @@
 import { TRANSFORM_STATE } from '../../../common/constants';
 
 import { TransformListRow } from './transform_list';
+import {
+  PutTransformsLatestRequestSchema,
+  PutTransformsPivotRequestSchema,
+} from '../../../common/api_schemas/transforms';
 
-export function getTransformProgress(item: TransformListRow) {
+type TransformItem = Omit<TransformListRow, 'config'> & {
+  config:
+    | TransformListRow['config']
+    | PutTransformsLatestRequestSchema
+    | PutTransformsPivotRequestSchema;
+};
+
+export function getTransformProgress(item: TransformItem) {
   if (isCompletedBatchTransform(item)) {
     return 100;
   }
@@ -17,7 +28,7 @@ export function getTransformProgress(item: TransformListRow) {
   return progress !== undefined ? Math.round(progress) : undefined;
 }
 
-export function isCompletedBatchTransform(item: TransformListRow) {
+export function isCompletedBatchTransform(item: TransformItem) {
   // If `checkpoint=1`, `sync` is missing from the config and state is stopped,
   // then this is a completed batch transform.
   return (

x-pack/plugins/transform/public/app/hooks/use_delete_transform.tsx

@@ -108,10 +108,7 @@ export const useDeleteIndexAndTargetIndex = (items: TransformListRow[]) => {
 type SuccessCountField = keyof Omit<DeleteTransformStatus, 'destinationIndex'>;
 
 export const useDeleteTransforms = () => {
-  const {
+  const { overlays } = useAppDependencies();
-    overlays,
-    ml: { extractErrorMessage },
-  } = useAppDependencies();
   const toastNotifications = useToastNotifications();
   const api = useApi();
 
@@ -188,7 +185,7 @@ export const useDeleteTransforms = () => {
           });
         }
         if (status.transformDeleted?.error) {
-          const error = extractErrorMessage(status.transformDeleted.error);
+          const error = status.transformDeleted.error.reason;
           toastNotifications.addDanger({
             title: i18n.translate('xpack.transform.transformList.deleteTransformErrorMessage', {
               defaultMessage: 'An error occurred deleting the transform {transformId}',
@@ -201,7 +198,7 @@ export const useDeleteTransforms = () => {
         }
 
         if (status.destIndexDeleted?.error) {
-          const error = extractErrorMessage(status.destIndexDeleted.error);
+          const error = status.destIndexDeleted.error.reason;
           toastNotifications.addDanger({
             title: i18n.translate(
               'xpack.transform.deleteTransform.deleteAnalyticsWithIndexErrorMessage',
@@ -217,7 +214,7 @@ export const useDeleteTransforms = () => {
         }
 
         if (status.destIndexPatternDeleted?.error) {
-          const error = extractErrorMessage(status.destIndexPatternDeleted.error);
+          const error = status.destIndexPatternDeleted.error.reason;
           toastNotifications.addDanger({
             title: i18n.translate(
               'xpack.transform.deleteTransform.deleteAnalyticsWithIndexPatternErrorMessage',

x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts

@@ -15,48 +15,51 @@ import { ES_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
 
 import type { PreviewMappingsProperties } from '../../../common/api_schemas/transforms';
 import { isPostTransformsPreviewResponseSchema } from '../../../common/api_schemas/type_guards';
-import { dictionaryToArray } from '../../../common/types/common';
 import { getNestedProperty } from '../../../common/utils/object_utils';
 
 import { RenderCellValue, UseIndexDataReturnType } from '../../shared_imports';
 import { getErrorMessage } from '../../../common/utils/errors';
 
 import { useAppDependencies } from '../app_dependencies';
-import {
+import { getPreviewTransformRequestBody, PivotQuery } from '../common';
-  getPreviewTransformRequestBody,
-  PivotAggsConfigDict,
-  PivotGroupByConfigDict,
-  PivotGroupByConfig,
-  PivotQuery,
-  PivotAggsConfig,
-} from '../common';
-import { isPivotAggsWithExtendedForm } from '../common/pivot_aggs';
 
 import { SearchItems } from './use_search_items';
 import { useApi } from './use_api';
+import { StepDefineExposedState } from '../sections/create_transform/components/step_define';
+import {
+  isLatestPartialRequest,
+  isPivotPartialRequest,
+} from '../sections/create_transform/components/step_define/common/types';
 
-/**
+function sortColumns(groupByArr: string[]) {
- * Checks if the aggregations collection is invalid.
- */
-function isConfigInvalid(aggsArray: PivotAggsConfig[]): boolean {
-  return aggsArray.some((agg) => {
-    return (
-      (isPivotAggsWithExtendedForm(agg) && !agg.isValid()) ||
-      (agg.subAggs && isConfigInvalid(Object.values(agg.subAggs)))
-    );
-  });
-}
-
-function sortColumns(groupByArr: PivotGroupByConfig[]) {
   return (a: string, b: string) => {
     // make sure groupBy fields are always most left columns
-    if (groupByArr.some((d) => d.aggName === a) && groupByArr.some((d) => d.aggName === b)) {
+    if (
+      groupByArr.some((aggName) => aggName === a) &&
+      groupByArr.some((aggName) => aggName === b)
+    ) {
       return a.localeCompare(b);
     }
-    if (groupByArr.some((d) => d.aggName === a)) {
+    if (groupByArr.some((aggName) => aggName === a)) {
       return -1;
     }
-    if (groupByArr.some((d) => d.aggName === b)) {
+    if (groupByArr.some((aggName) => aggName === b)) {
+      return 1;
+    }
+    return a.localeCompare(b);
+  };
+}
+
+function sortColumnsForLatest(sortField: string) {
+  return (a: string, b: string) => {
+    // make sure sort field is always the most left column
+    if (sortField === a && sortField === b) {
+      return a.localeCompare(b);
+    }
+    if (sortField === a) {
+      return -1;
+    }
+    if (sortField === b) {
       return 1;
     }
     return a.localeCompare(b);
@@ -66,8 +69,8 @@ function sortColumns(groupByArr: PivotGroupByConfig[]) {
 export const usePivotData = (
   indexPatternTitle: SearchItems['indexPattern']['title'],
   query: PivotQuery,
-  aggs: PivotAggsConfigDict,
+  validationStatus: StepDefineExposedState['validationStatus'],
-  groupBy: PivotGroupByConfigDict
+  requestPayload: StepDefineExposedState['previewRequest']
 ): UseIndexDataReturnType => {
   const [
     previewMappingsProperties,
@@ -78,14 +81,17 @@ export const usePivotData = (
     ml: { formatHumanReadableDateTimeSeconds, multiColumnSortFactory, useDataGrid, INDEX_STATUS },
   } = useAppDependencies();
 
-  const aggsArr = useMemo(() => dictionaryToArray(aggs), [aggs]);
-  const groupByArr = useMemo(() => dictionaryToArray(groupBy), [groupBy]);
-
   // Filters mapping properties of type `object`, which get returned for nested field parents.
   const columnKeys = Object.keys(previewMappingsProperties).filter(
     (key) => previewMappingsProperties[key].type !== 'object'
   );
-  columnKeys.sort(sortColumns(groupByArr));
+
+  if (isPivotPartialRequest(requestPayload)) {
+    const groupByArr = Object.keys(requestPayload.pivot.group_by);
+    columnKeys.sort(sortColumns(groupByArr));
+  } else if (isLatestPartialRequest(requestPayload)) {
+    columnKeys.sort(sortColumnsForLatest(requestPayload.latest.sort));
+  }
 
   // EuiDataGrid State
   const columns: EuiDataGridColumn[] = columnKeys.map((id) => {
@@ -141,18 +147,10 @@ export const usePivotData = (
   } = dataGrid;
 
   const getPreviewData = async () => {
-    if (aggsArr.length === 0 || groupByArr.length === 0) {
+    if (!validationStatus.isValid) {
       setTableItems([]);
       setRowCount(0);
-      setNoDataMessage(
+      setNoDataMessage(validationStatus.errorMessage!);
-        i18n.translate('xpack.transform.pivotPreview.PivotPreviewIncompleteConfigCalloutBody', {
-          defaultMessage: 'Please choose at least one group-by field and aggregation.',
-        })
-      );
-      return;
-    }
-
-    if (isConfigInvalid(aggsArr)) {
       return;
     }
 
@@ -160,12 +158,7 @@ export const usePivotData = (
     setNoDataMessage('');
     setStatus(INDEX_STATUS.LOADING);
 
-    const previewRequest = getPreviewTransformRequestBody(
+    const previewRequest = getPreviewTransformRequestBody(indexPatternTitle, query, requestPayload);
-      indexPatternTitle,
-      query,
-      groupByArr,
-      aggsArr
-    );
     const resp = await api.getTransformsPreview(previewRequest);
 
     if (!isPostTransformsPreviewResponseSchema(resp)) {
@@ -204,8 +197,7 @@ export const usePivotData = (
     /* eslint-disable react-hooks/exhaustive-deps */
   }, [
     indexPatternTitle,
-    aggsArr,
+    JSON.stringify([requestPayload, query]),
-    JSON.stringify([groupByArr, query]),
     /* eslint-enable react-hooks/exhaustive-deps */
   ]);
 

x-pack/plugins/transform/public/app/hooks/use_start_transform.tsx

@@ -47,7 +47,8 @@ export const useStartTransforms = () => {
     for (const transformId in results) {
       // hasOwnProperty check to ensure only properties on object itself, and not its prototypes
       if (results.hasOwnProperty(transformId)) {
-        if (results[transformId].success === true) {
+        const result = results[transformId];
+        if (result.success === true) {
           toastNotifications.addSuccess(
             i18n.translate('xpack.transform.transformList.startTransformSuccessMessage', {
               defaultMessage: 'Request to start transform {transformId} acknowledged.',
@@ -55,12 +56,13 @@ export const useStartTransforms = () => {
             })
           );
         } else {
-          toastNotifications.addDanger(
+          toastNotifications.addError(new Error(JSON.stringify(result.error!.caused_by, null, 2)), {
-            i18n.translate('xpack.transform.transformList.startTransformErrorMessage', {
+            title: i18n.translate('xpack.transform.transformList.startTransformErrorMessage', {
               defaultMessage: 'An error occurred starting the transform {transformId}',
               values: { transformId },
-            })
+            }),
-          );
+            toastMessage: result.error!.reason,
+          });
         }
       }
     }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx

@@ -25,10 +25,7 @@ import {
 
 import { toMountPoint } from '../../../../../../../../../src/plugins/kibana_react/public';
 
-import type {
+import type { PutTransformsResponseSchema } from '../../../../../../common/api_schemas/transforms';
-  PutTransformsRequestSchema,
-  PutTransformsResponseSchema,
-} from '../../../../../../common/api_schemas/transforms';
 import {
   isGetTransformsStatsResponseSchema,
   isPutTransformsResponseSchema,
@@ -44,6 +41,10 @@ import { useAppDependencies, useToastNotifications } from '../../../../app_depen
 import { RedirectToTransformManagement } from '../../../../common/navigation';
 import { ToastNotificationText } from '../../../../components';
 import { DuplicateIndexPatternError } from '../../../../../../../../../src/plugins/data/public';
+import {
+  PutTransformsLatestRequestSchema,
+  PutTransformsPivotRequestSchema,
+} from '../../../../../../common/api_schemas/transforms';
 
 export interface StepDetailsExposedState {
   created: boolean;
@@ -62,7 +63,7 @@ export function getDefaultStepCreateState(): StepDetailsExposedState {
 export interface StepCreateFormProps {
   createIndexPattern: boolean;
   transformId: string;
-  transformConfig: PutTransformsRequestSchema;
+  transformConfig: PutTransformsPivotRequestSchema | PutTransformsLatestRequestSchema;
   overrides: StepDetailsExposedState;
   timeFieldName?: string | undefined;
   onChange(s: StepDetailsExposedState): void;

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts

@@ -8,7 +8,11 @@ import { isEqual } from 'lodash';
 
 import { Dictionary } from '../../../../../../../common/types/common';
 import { PivotSupportedAggs } from '../../../../../../../common/types/pivot_aggs';
-import { TransformPivotConfig } from '../../../../../../../common/types/transform';
+import {
+  isLatestTransform,
+  isPivotTransform,
+  TransformBaseConfig,
+} from '../../../../../../../common/types/transform';
 
 import {
   matchAllQuery,
@@ -21,13 +25,24 @@ import {
 
 import { StepDefineExposedState } from './types';
 import { getAggConfigFromEsAgg } from '../../../../../common/pivot_aggs';
+import { TRANSFORM_FUNCTION } from '../../../../../../../common/constants';
+import { StepDefineFormProps } from '../step_define_form';
+import { validateLatestConfig } from '../hooks/use_latest_function_config';
+import { validatePivotConfig } from '../hooks/use_pivot_config';
 
 export function applyTransformConfigToDefineState(
   state: StepDefineExposedState,
-  transformConfig?: TransformPivotConfig
+  transformConfig?: TransformBaseConfig,
+  indexPattern?: StepDefineFormProps['searchItems']['indexPattern']
 ): StepDefineExposedState {
-  // apply the transform configuration to wizard DEFINE state
+  if (transformConfig === undefined) {
-  if (transformConfig !== undefined) {
+    return state;
+  }
+
+  if (isPivotTransform(transformConfig)) {
+    state.transformFunction = TRANSFORM_FUNCTION.PIVOT;
+
+    // apply the transform configuration to wizard DEFINE state
     // transform aggregations config to wizard state
     state.aggList = Object.keys(transformConfig.pivot.aggregations).reduce((aggList, aggName) => {
       const aggConfig = transformConfig.pivot.aggregations[
@@ -53,17 +68,44 @@ export function applyTransformConfigToDefineState(
       {} as PivotGroupByConfigDict
     );
 
-    // only apply the query from the transform config to wizard state if it's not the default query
+    state.previewRequest = {
-    const query = transformConfig.source.query;
+      pivot: transformConfig.pivot,
-    if (query !== undefined && !isEqual(query, matchAllQuery)) {
+    };
-      state.isAdvancedSourceEditorEnabled = true;
-      state.searchQuery = query;
-      state.sourceConfigUpdated = true;
-    }
 
-    // applying a transform config to wizard state will always result in a valid configuration
+    state.validationStatus = validatePivotConfig(transformConfig.pivot);
-    state.valid = true;
   }
 
+  if (isLatestTransform(transformConfig)) {
+    state.transformFunction = TRANSFORM_FUNCTION.LATEST;
+    state.latestConfig = {
+      unique_key: transformConfig.latest.unique_key.map((v) => ({
+        value: v,
+        label: indexPattern ? indexPattern.fields.find((f) => f.name === v)?.displayName ?? v : v,
+      })),
+      sort: {
+        value: transformConfig.latest.sort,
+        label: indexPattern
+          ? indexPattern.fields.find((f) => f.name === transformConfig.latest.sort)?.displayName ??
+            transformConfig.latest.sort
+          : transformConfig.latest.sort,
+      },
+    };
+    state.previewRequest = {
+      latest: transformConfig.latest,
+    };
+    state.validationStatus = validateLatestConfig(transformConfig.latest);
+  }
+
+  // only apply the query from the transform config to wizard state if it's not the default query
+  const query = transformConfig.source.query;
+  if (query !== undefined && !isEqual(query, matchAllQuery)) {
+    state.isAdvancedSourceEditorEnabled = true;
+    state.searchQuery = query;
+    state.sourceConfigUpdated = true;
+  }
+
+  // applying a transform config to wizard state will always result in a valid configuration
+  state.valid = true;
+
   return state;
 }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_step_define_state.ts

@@ -9,9 +9,13 @@ import { SearchItems } from '../../../../../hooks/use_search_items';
 
 import { defaultSearch, QUERY_LANGUAGE_KUERY } from './constants';
 import { StepDefineExposedState } from './types';
+import { TRANSFORM_FUNCTION } from '../../../../../../../common/constants';
+import { LatestFunctionConfigUI } from '../../../../../../../common/types/transform';
 
 export function getDefaultStepDefineState(searchItems: SearchItems): StepDefineExposedState {
   return {
+    transformFunction: TRANSFORM_FUNCTION.PIVOT,
+    latestConfig: {} as LatestFunctionConfigUI,
     aggList: {} as PivotAggsConfigDict,
     groupByList: {} as PivotGroupByConfigDict,
     isAdvancedPivotEditorEnabled: false,
@@ -21,5 +25,9 @@ export function getDefaultStepDefineState(searchItems: SearchItems): StepDefineE
     searchQuery: searchItems.savedSearch !== undefined ? searchItems.combinedQuery : defaultSearch,
     sourceConfigUpdated: false,
     valid: false,
+    validationStatus: {
+      isValid: false,
+    },
+    previewRequest: undefined,
   };
 }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts

@@ -12,6 +12,12 @@ import { PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../comm
 import { SavedSearchQuery } from '../../../../../hooks/use_search_items';
 
 import { QUERY_LANGUAGE } from './constants';
+import { TransformFunction } from '../../../../../../../common/constants';
+import {
+  LatestFunctionConfigUI,
+  PivotConfigDefinition,
+} from '../../../../../../../common/types/transform';
+import { LatestFunctionConfig } from '../../../../../../../common/api_schemas/transforms';
 
 export interface ErrorMessage {
   query: string;
@@ -24,8 +30,10 @@ export interface Field {
 }
 
 export interface StepDefineExposedState {
+  transformFunction: TransformFunction;
   aggList: PivotAggsConfigDict;
   groupByList: PivotGroupByConfigDict;
+  latestConfig: LatestFunctionConfigUI;
   isAdvancedPivotEditorEnabled: boolean;
   isAdvancedSourceEditorEnabled: boolean;
   searchLanguage: QUERY_LANGUAGE;
@@ -33,4 +41,17 @@ export interface StepDefineExposedState {
   searchQuery: string | SavedSearchQuery;
   sourceConfigUpdated: boolean;
   valid: boolean;
+  validationStatus: { isValid: boolean; errorMessage?: string };
+  /**
+   * Undefined when the form is incomplete or invalid
+   */
+  previewRequest: { latest: LatestFunctionConfig } | { pivot: PivotConfigDefinition } | undefined;
+}
+
+export function isPivotPartialRequest(arg: any): arg is { pivot: PivotConfigDefinition } {
+  return typeof arg === 'object' && arg.hasOwnProperty('pivot');
+}
+
+export function isLatestPartialRequest(arg: any): arg is { latest: LatestFunctionConfig } {
+  return typeof arg === 'object' && arg.hasOwnProperty('latest');
 }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts

@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { useCallback, useMemo, useState } from 'react';
+import { i18n } from '@kbn/i18n';
+import { EuiComboBoxOptionOption } from '@elastic/eui';
+import { LatestFunctionConfigUI } from '../../../../../../../common/types/transform';
+import { StepDefineFormProps } from '../step_define_form';
+import { StepDefineExposedState } from '../common';
+import { LatestFunctionConfig } from '../../../../../../../common/api_schemas/transforms';
+
+/**
+ * Latest function config mapper between API and UI
+ */
+export const latestConfigMapper = {
+  toAPIConfig(uiConfig: LatestFunctionConfigUI): LatestFunctionConfig | undefined {
+    if (uiConfig.sort === undefined || !uiConfig.unique_key?.length) {
+      return;
+    }
+
+    return {
+      unique_key: uiConfig.unique_key.map((v) => v.value!),
+      sort: uiConfig.sort.value!,
+    };
+  },
+  toUIConfig() {},
+};
+
+/**
+ * Provides available options for unique_key and sort fields
+ * @param indexPattern
+ */
+function getOptions(indexPattern: StepDefineFormProps['searchItems']['indexPattern']) {
+  const uniqueKeyOptions: Array<EuiComboBoxOptionOption<string>> = [];
+  const sortFieldOptions: Array<EuiComboBoxOptionOption<string>> = [];
+
+  const ignoreFieldNames = new Set(['_id', '_index', '_type']);
+
+  for (const field of indexPattern.fields) {
+    if (ignoreFieldNames.has(field.name)) {
+      continue;
+    }
+
+    if (field.aggregatable) {
+      uniqueKeyOptions.push({ label: field.displayName, value: field.name });
+    }
+
+    if (field.sortable) {
+      sortFieldOptions.push({ label: field.displayName, value: field.name });
+    }
+  }
+
+  return { uniqueKeyOptions, sortFieldOptions };
+}
+
+/**
+ * Validates latest function configuration
+ */
+export function validateLatestConfig(config?: LatestFunctionConfig) {
+  const isValid: boolean = !!config?.unique_key?.length && config?.sort !== undefined;
+  return {
+    isValid,
+    ...(isValid
+      ? {}
+      : {
+          errorMessage: i18n.translate(
+            'xpack.transform.latestPreview.latestPreviewIncompleteConfigCalloutBody',
+            {
+              defaultMessage: 'Please choose at least one unique key and sort field.',
+            }
+          ),
+        }),
+  };
+}
+
+export function useLatestFunctionConfig(
+  defaults: StepDefineExposedState['latestConfig'],
+  indexPattern: StepDefineFormProps['searchItems']['indexPattern']
+): {
+  config: LatestFunctionConfigUI;
+  uniqueKeyOptions: Array<EuiComboBoxOptionOption<string>>;
+  sortFieldOptions: Array<EuiComboBoxOptionOption<string>>;
+  updateLatestFunctionConfig: (update: Partial<LatestFunctionConfigUI>) => void;
+  validationStatus: { isValid: boolean; errorMessage?: string };
+  requestPayload: { latest: LatestFunctionConfig } | undefined;
+} {
+  const [config, setLatestFunctionConfig] = useState<LatestFunctionConfigUI>({
+    unique_key: defaults.unique_key,
+    sort: defaults.sort,
+  });
+
+  const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => getOptions(indexPattern), [
+    indexPattern,
+  ]);
+
+  const updateLatestFunctionConfig = useCallback(
+    (update) =>
+      setLatestFunctionConfig({
+        ...config,
+        ...update,
+      }),
+    [config]
+  );
+
+  const requestPayload: { latest: LatestFunctionConfig } | undefined = useMemo(() => {
+    const latest = latestConfigMapper.toAPIConfig(config);
+    return latest ? { latest } : undefined;
+  }, [config]);
+
+  const validationStatus = useMemo(() => validateLatestConfig(requestPayload?.latest), [
+    requestPayload?.latest,
+  ]);
+
+  return {
+    config,
+    uniqueKeyOptions,
+    sortFieldOptions,
+    updateLatestFunctionConfig,
+    validationStatus,
+    requestPayload,
+  };
+}
+
+export type LatestFunctionService = ReturnType<typeof useLatestFunctionConfig>;

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.test.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getMissingBucketConfig } from './use_pivot_config';
+import { PIVOT_SUPPORTED_GROUP_BY_AGGS, PivotGroupByConfig } from '../../../../../common';
+
+const groupByTerms: PivotGroupByConfig = {
+  agg: PIVOT_SUPPORTED_GROUP_BY_AGGS.TERMS,
+  field: 'the-group-by-field',
+  aggName: 'the-group-by-agg-name',
+  dropDownName: 'the-group-by-drop-down-name',
+};
+
+describe('usePivotConfig', () => {
+  test('getMissingBucketConfig()', () => {
+    expect(getMissingBucketConfig(groupByTerms)).toEqual({});
+    expect(getMissingBucketConfig({ ...groupByTerms, ...{ missing_bucket: true } })).toEqual({
+      missing_bucket: true,
+    });
+    expect(getMissingBucketConfig({ ...groupByTerms, ...{ missing_bucket: false } })).toEqual({
+      missing_bucket: false,
+    });
+  });
+});

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts

@@ -5,6 +5,7 @@
  */
 
 import { useCallback, useMemo, useState } from 'react';
+import { i18n } from '@kbn/i18n';
 
 import { AggName } from '../../../../../../../common/types/aggregations';
 import { dictionaryToArray } from '../../../../../../../common/types/common';
@@ -12,6 +13,12 @@ import { dictionaryToArray } from '../../../../../../../common/types/common';
 import { useToastNotifications } from '../../../../../app_dependencies';
 import {
   DropDownLabel,
+  getEsAggFromAggConfig,
+  getEsAggFromGroupByConfig,
+  GroupByConfigWithUiSupport,
+  isGroupByDateHistogram,
+  isGroupByHistogram,
+  isGroupByTerms,
   PivotAggsConfig,
   PivotAggsConfigDict,
   PivotGroupByConfig,
@@ -23,6 +30,14 @@ import {
   StepDefineExposedState,
 } from '../common';
 import { StepDefineFormProps } from '../step_define_form';
+import { isPivotAggsWithExtendedForm } from '../../../../../common/pivot_aggs';
+import {
+  DateHistogramAgg,
+  HistogramAgg,
+  TermsAgg,
+} from '../../../../../../../common/types/pivot_group_by';
+import { PivotTransformPreviewRequestSchema } from '../../../../../../../common/api_schemas/transforms';
+import { TransformPivotConfig } from '../../../../../../../common/types/transform';
 
 /**
  * Clones aggregation configuration and updates parent references
@@ -43,6 +58,37 @@ function cloneAggItem(item: PivotAggsConfig, parentRef?: PivotAggsConfig) {
   return newItem;
 }
 
+/**
+ * Checks if the aggregations collection is invalid.
+ */
+function isConfigInvalid(aggsArray: PivotAggsConfig[]): boolean {
+  return aggsArray.some((agg) => {
+    return (
+      (isPivotAggsWithExtendedForm(agg) && !agg.isValid()) ||
+      (agg.subAggs && isConfigInvalid(Object.values(agg.subAggs)))
+    );
+  });
+}
+
+export function validatePivotConfig(config: TransformPivotConfig['pivot']) {
+  const valid =
+    Object.values(config.aggregations).length > 0 && Object.values(config.group_by).length > 0;
+  const isValid: boolean = valid && !isConfigInvalid(dictionaryToArray(config.aggregations));
+  return {
+    isValid,
+    ...(isValid
+      ? {}
+      : {
+          errorMessage: i18n.translate(
+            'xpack.transform.pivotPreview.PivotPreviewIncompleteConfigCalloutBody',
+            {
+              defaultMessage: 'Please choose at least one group-by field and aggregation.',
+            }
+          ),
+        }),
+  };
+}
+
 /**
  * Returns a root aggregation configuration
  * for provided aggregation item.
@@ -55,6 +101,12 @@ function getRootAggregation(item: PivotAggsConfig) {
   return rootItem;
 }
 
+export const getMissingBucketConfig = (
+  g: GroupByConfigWithUiSupport
+): { missing_bucket?: boolean } => {
+  return g.missing_bucket !== undefined ? { missing_bucket: g.missing_bucket } : {};
+};
+
 export const usePivotConfig = (
   defaults: StepDefineExposedState,
   indexPattern: StepDefineFormProps['searchItems']['indexPattern']
@@ -262,7 +314,60 @@ export const usePivotConfig = (
   const pivotAggsArr = useMemo(() => dictionaryToArray(aggList), [aggList]);
   const pivotGroupByArr = useMemo(() => dictionaryToArray(groupByList), [groupByList]);
 
-  const valid = pivotGroupByArr.length > 0 && pivotAggsArr.length > 0;
+  const requestPayload = useMemo(() => {
+    const request = {
+      pivot: {
+        group_by: {},
+        aggregations: {},
+      } as PivotTransformPreviewRequestSchema['pivot'],
+    };
+
+    pivotGroupByArr.forEach((g) => {
+      if (isGroupByTerms(g)) {
+        const termsAgg: TermsAgg = {
+          terms: {
+            field: g.field,
+          },
+          ...getMissingBucketConfig(g),
+        };
+        request.pivot.group_by[g.aggName] = termsAgg;
+      } else if (isGroupByHistogram(g)) {
+        const histogramAgg: HistogramAgg = {
+          histogram: {
+            field: g.field,
+            interval: g.interval,
+          },
+          ...getMissingBucketConfig(g),
+        };
+        request.pivot.group_by[g.aggName] = histogramAgg;
+      } else if (isGroupByDateHistogram(g)) {
+        const dateHistogramAgg: DateHistogramAgg = {
+          date_histogram: {
+            field: g.field,
+            calendar_interval: g.calendar_interval,
+          },
+          ...getMissingBucketConfig(g),
+        };
+        request.pivot.group_by[g.aggName] = dateHistogramAgg;
+      } else {
+        request.pivot.group_by[g.aggName] = getEsAggFromGroupByConfig(g);
+      }
+    });
+
+    pivotAggsArr.forEach((agg) => {
+      const result = getEsAggFromAggConfig(agg);
+      if (result === null) {
+        return;
+      }
+      request.pivot.aggregations[agg.aggName] = result;
+    });
+
+    return request;
+  }, [pivotAggsArr, pivotGroupByArr]);
+
+  const validationStatus = useMemo(() => {
+    return validatePivotConfig(requestPayload.pivot);
+  }, [requestPayload]);
 
   const actions = useMemo(() => {
     return {
@@ -302,7 +407,8 @@ export const usePivotConfig = (
         groupByOptionsData,
         pivotAggsArr,
         pivotGroupByArr,
-        valid,
+        validationStatus,
+        requestPayload,
       },
     };
   }, [
@@ -315,6 +421,9 @@ export const usePivotConfig = (
     groupByOptionsData,
     pivotAggsArr,
     pivotGroupByArr,
-    valid,
+    validationStatus,
+    requestPayload,
   ]);
 };
+
+export type PivotService = ReturnType<typeof usePivotConfig>;

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { useEffect } from 'react';
+import { useEffect, useState } from 'react';
 
 import { getPreviewTransformRequestBody } from '../../../../../common';
 
@@ -16,6 +16,8 @@ import { useAdvancedPivotEditor } from './use_advanced_pivot_editor';
 import { useAdvancedSourceEditor } from './use_advanced_source_editor';
 import { usePivotConfig } from './use_pivot_config';
 import { useSearchBar } from './use_search_bar';
+import { useLatestFunctionConfig } from './use_latest_function_config';
+import { TRANSFORM_FUNCTION } from '../../../../../../../common/constants';
 
 export type StepDefineFormHook = ReturnType<typeof useStepDefineForm>;
 
@@ -23,14 +25,16 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi
   const defaults = { ...getDefaultStepDefineState(searchItems), ...overrides };
   const { indexPattern } = searchItems;
 
+  const [transformFunction, setTransformFunction] = useState(defaults.transformFunction);
+
   const searchBar = useSearchBar(defaults, indexPattern);
   const pivotConfig = usePivotConfig(defaults, indexPattern);
+  const latestFunctionConfig = useLatestFunctionConfig(defaults.latestConfig, indexPattern);
 
   const previewRequest = getPreviewTransformRequestBody(
     indexPattern.title,
     searchBar.state.pivotQuery,
-    pivotConfig.state.pivotGroupByArr,
+    pivotConfig.state.requestPayload
-    pivotConfig.state.pivotAggsArr
   );
 
   // pivot config hook
@@ -44,8 +48,7 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi
       const previewRequestUpdate = getPreviewTransformRequestBody(
         indexPattern.title,
         searchBar.state.pivotQuery,
-        pivotConfig.state.pivotGroupByArr,
+        pivotConfig.state.requestPayload
-        pivotConfig.state.pivotAggsArr
       );
 
       const stringifiedSourceConfigUpdate = JSON.stringify(
@@ -58,6 +61,8 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi
     }
 
     onChange({
+      transformFunction,
+      latestConfig: latestFunctionConfig.config,
       aggList: pivotConfig.state.aggList,
       groupByList: pivotConfig.state.groupByList,
       isAdvancedPivotEditorEnabled: advancedPivotEditor.state.isAdvancedPivotEditorEnabled,
@@ -66,7 +71,18 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi
       searchString: searchBar.state.searchString,
       searchQuery: searchBar.state.searchQuery,
       sourceConfigUpdated: advancedSourceEditor.state.sourceConfigUpdated,
-      valid: pivotConfig.state.valid,
+      valid:
+        transformFunction === TRANSFORM_FUNCTION.PIVOT
+          ? pivotConfig.state.validationStatus.isValid
+          : latestFunctionConfig.validationStatus.isValid,
+      validationStatus:
+        transformFunction === TRANSFORM_FUNCTION.PIVOT
+          ? pivotConfig.state.validationStatus
+          : latestFunctionConfig.validationStatus,
+      previewRequest:
+        transformFunction === TRANSFORM_FUNCTION.PIVOT
+          ? pivotConfig.state.requestPayload
+          : latestFunctionConfig.requestPayload,
     });
     // custom comparison
     /* eslint-disable react-hooks/exhaustive-deps */
@@ -75,13 +91,18 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi
     JSON.stringify(advancedSourceEditor.state),
     pivotConfig.state,
     JSON.stringify(searchBar.state),
+    latestFunctionConfig.config,
+    transformFunction,
     /* eslint-enable react-hooks/exhaustive-deps */
   ]);
 
   return {
+    transformFunction,
+    setTransformFunction,
     advancedPivotEditor,
     advancedSourceEditor,
     pivotConfig,
+    latestFunctionConfig,
     searchBar,
   };
 };

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/latest_function_form.tsx

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { EuiComboBox, EuiFormRow } from '@elastic/eui';
+import { LatestFunctionService } from './hooks/use_latest_function_config';
+
+interface LatestFunctionFormProps {
+  latestFunctionService: LatestFunctionService;
+}
+
+export const LatestFunctionForm: FC<LatestFunctionFormProps> = ({ latestFunctionService }) => {
+  return (
+    <>
+      <EuiFormRow
+        fullWidth
+        label={
+          <FormattedMessage
+            id="xpack.transform.stepDefineForm.uniqueKeysLabel"
+            defaultMessage="Unique keys"
+          />
+        }
+      >
+        <EuiComboBox<string>
+          fullWidth
+          placeholder={i18n.translate('xpack.transform.stepDefineForm.uniqueKeysPlaceholder', {
+            defaultMessage: 'Add unique keys ...',
+          })}
+          options={latestFunctionService.uniqueKeyOptions}
+          selectedOptions={latestFunctionService.config.unique_key ?? []}
+          onChange={(selected) => {
+            latestFunctionService.updateLatestFunctionConfig({
+              unique_key: selected,
+            });
+          }}
+          isClearable={false}
+          data-test-subj="transformWizardUniqueKeysSelector"
+        />
+      </EuiFormRow>
+      <EuiFormRow
+        fullWidth
+        label={
+          <FormattedMessage
+            id="xpack.transform.stepDefineForm.sortLabel"
+            defaultMessage="Sort field"
+          />
+        }
+      >
+        <EuiComboBox
+          fullWidth
+          placeholder={i18n.translate('xpack.transform.stepDefineForm.sortPlaceholder', {
+            defaultMessage: 'Add a sort field ...',
+          })}
+          singleSelection={{ asPlainText: true }}
+          options={latestFunctionService.sortFieldOptions}
+          selectedOptions={
+            latestFunctionService.config.sort ? [latestFunctionService.config.sort] : []
+          }
+          onChange={(selected) => {
+            latestFunctionService.updateLatestFunctionConfig({
+              sort: { value: selected[0].value, label: selected[0].label as string },
+            });
+          }}
+          isClearable={false}
+          data-test-subj="transformWizardSortFieldSelector"
+        />
+      </EuiFormRow>
+    </>
+  );
+};

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx

@@ -53,6 +53,9 @@ import { SourceSearchBar } from '../source_search_bar';
 import { StepDefineExposedState } from './common';
 import { useStepDefineForm } from './hooks/use_step_define_form';
 import { getAggConfigFromEsAgg } from '../../../../common/pivot_aggs';
+import { TransformFunctionSelector } from './transform_function_selector';
+import { TRANSFORM_FUNCTION } from '../../../../../../common/constants';
+import { LatestFunctionForm } from './latest_function_form';
 
 export interface StepDefineFormProps {
   overrides?: StepDefineExposedState;
@@ -80,7 +83,6 @@ export const StepDefineForm: FC<StepDefineFormProps> = React.memo((props) => {
     isAdvancedSourceEditorEnabled,
     isAdvancedSourceEditorApplyButtonEnabled,
   } = stepDefineForm.advancedSourceEditor.state;
-  const { aggList, groupByList, pivotGroupByArr, pivotAggsArr } = stepDefineForm.pivotConfig.state;
   const pivotQuery = stepDefineForm.searchBar.state.pivotQuery;
 
   const indexPreviewProps = {
@@ -89,15 +91,21 @@ export const StepDefineForm: FC<StepDefineFormProps> = React.memo((props) => {
     toastNotifications,
   };
 
+  const { requestPayload, validationStatus } =
+    stepDefineForm.transformFunction === TRANSFORM_FUNCTION.PIVOT
+      ? stepDefineForm.pivotConfig.state
+      : stepDefineForm.latestFunctionConfig;
+
   const previewRequest = getPreviewTransformRequestBody(
     indexPattern.title,
     pivotQuery,
-    pivotGroupByArr,
+    stepDefineForm.transformFunction === TRANSFORM_FUNCTION.PIVOT
-    pivotAggsArr
+      ? stepDefineForm.pivotConfig.state.requestPayload
+      : stepDefineForm.latestFunctionConfig.requestPayload
   );
 
   const pivotPreviewProps = {
-    ...usePivotData(indexPattern.title, pivotQuery, aggList, groupByList),
+    ...usePivotData(indexPattern.title, pivotQuery, validationStatus, requestPayload),
     dataTestSubj: 'transformPivotPreview',
     toastNotifications,
   };
@@ -171,6 +179,13 @@ export const StepDefineForm: FC<StepDefineFormProps> = React.memo((props) => {
   return (
     <div data-test-subj="transformStepDefineForm">
       <EuiForm>
+        <EuiFormRow fullWidth>
+          <TransformFunctionSelector
+            selectedFunction={stepDefineForm.transformFunction}
+            onChange={stepDefineForm.setTransformFunction}
+          />
+        </EuiFormRow>
+
         {searchItems.savedSearch === undefined && (
           <EuiFormRow
             label={i18n.translate('xpack.transform.stepDefineForm.indexPatternLabel', {
@@ -180,6 +195,7 @@ export const StepDefineForm: FC<StepDefineFormProps> = React.memo((props) => {
             <span>{indexPattern.title}</span>
           </EuiFormRow>
         )}
+
         <EuiFormRow
           fullWidth
           hasEmptyLabelSpace={searchItems?.savedSearch?.id === undefined}
@@ -284,80 +300,85 @@ export const StepDefineForm: FC<StepDefineFormProps> = React.memo((props) => {
       </EuiForm>
       <EuiHorizontalRule margin="m" />
       <EuiForm>
-        <EuiFlexGroup justifyContent="spaceBetween">
+        {stepDefineForm.transformFunction === TRANSFORM_FUNCTION.PIVOT ? (
-          {/* Flex Column #1: Pivot Config Form / Advanced Pivot Config Editor */}
+          <EuiFlexGroup justifyContent="spaceBetween">
-          <EuiFlexItem>
+            {/* Flex Column #1: Pivot Config Form / Advanced Pivot Config Editor */}
-            {!isAdvancedPivotEditorEnabled && (
+            <EuiFlexItem>
-              <PivotConfiguration {...stepDefineForm.pivotConfig} />
+              {!isAdvancedPivotEditorEnabled && (
-            )}
+                <PivotConfiguration {...stepDefineForm.pivotConfig} />
-            {isAdvancedPivotEditorEnabled && (
-              <AdvancedPivotEditor {...stepDefineForm.advancedPivotEditor} />
-            )}
-          </EuiFlexItem>
-          <EuiFlexItem grow={false} style={{ width: advancedEditorsSidebarWidth }}>
-            <EuiFlexGroup gutterSize="xs" direction="column" justifyContent="spaceBetween">
-              <EuiFlexItem grow={false}>
-                <EuiFormRow hasEmptyLabelSpace>
-                  <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
-                    <EuiFlexItem grow={false}>
-                      <AdvancedPivotEditorSwitch {...stepDefineForm} />
-                    </EuiFlexItem>
-                    <EuiFlexItem grow={false}>
-                      <EuiCopy
-                        beforeMessage={copyToClipboardPivotDescription}
-                        textToCopy={copyToClipboardPivot}
-                      >
-                        {(copy: () => void) => (
-                          <EuiButtonIcon
-                            onClick={copy}
-                            iconType="copyClipboard"
-                            aria-label={copyToClipboardPivotDescription}
-                          />
-                        )}
-                      </EuiCopy>
-                    </EuiFlexItem>
-                  </EuiFlexGroup>
-                </EuiFormRow>
-              </EuiFlexItem>
-              {isAdvancedPivotEditorEnabled && (
-                <EuiFlexItem style={{ width: advancedEditorsSidebarWidth }}>
-                  <EuiSpacer size="s" />
-                  <EuiText size="xs">
-                    <>
-                      {i18n.translate('xpack.transform.stepDefineForm.advancedEditorHelpText', {
-                        defaultMessage:
-                          'The advanced editor allows you to edit the pivot configuration of the transform.',
-                      })}{' '}
-                      <EuiLink href={esTransformPivot} target="_blank">
-                        {i18n.translate(
-                          'xpack.transform.stepDefineForm.advancedEditorHelpTextLink',
-                          {
-                            defaultMessage: 'Learn more about available options.',
-                          }
-                        )}
-                      </EuiLink>
-                    </>
-                  </EuiText>
-                  <EuiSpacer size="s" />
-                  <EuiButton
-                    style={{ width: 'fit-content' }}
-                    size="s"
-                    fill
-                    onClick={applyPivotChangesHandler}
-                    disabled={!isAdvancedPivotEditorApplyButtonEnabled}
-                  >
-                    {i18n.translate(
-                      'xpack.transform.stepDefineForm.advancedEditorApplyButtonText',
-                      {
-                        defaultMessage: 'Apply changes',
-                      }
-                    )}
-                  </EuiButton>
-                </EuiFlexItem>
               )}
-            </EuiFlexGroup>
+              {isAdvancedPivotEditorEnabled && (
-          </EuiFlexItem>
+                <AdvancedPivotEditor {...stepDefineForm.advancedPivotEditor} />
-        </EuiFlexGroup>
+              )}
+            </EuiFlexItem>
+            <EuiFlexItem grow={false} style={{ width: advancedEditorsSidebarWidth }}>
+              <EuiFlexGroup gutterSize="xs" direction="column" justifyContent="spaceBetween">
+                <EuiFlexItem grow={false}>
+                  <EuiFormRow hasEmptyLabelSpace>
+                    <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
+                      <EuiFlexItem grow={false}>
+                        <AdvancedPivotEditorSwitch {...stepDefineForm} />
+                      </EuiFlexItem>
+                      <EuiFlexItem grow={false}>
+                        <EuiCopy
+                          beforeMessage={copyToClipboardPivotDescription}
+                          textToCopy={copyToClipboardPivot}
+                        >
+                          {(copy: () => void) => (
+                            <EuiButtonIcon
+                              onClick={copy}
+                              iconType="copyClipboard"
+                              aria-label={copyToClipboardPivotDescription}
+                            />
+                          )}
+                        </EuiCopy>
+                      </EuiFlexItem>
+                    </EuiFlexGroup>
+                  </EuiFormRow>
+                </EuiFlexItem>
+                {isAdvancedPivotEditorEnabled && (
+                  <EuiFlexItem style={{ width: advancedEditorsSidebarWidth }}>
+                    <EuiSpacer size="s" />
+                    <EuiText size="xs">
+                      <>
+                        {i18n.translate('xpack.transform.stepDefineForm.advancedEditorHelpText', {
+                          defaultMessage:
+                            'The advanced editor allows you to edit the pivot configuration of the transform.',
+                        })}{' '}
+                        <EuiLink href={esTransformPivot} target="_blank">
+                          {i18n.translate(
+                            'xpack.transform.stepDefineForm.advancedEditorHelpTextLink',
+                            {
+                              defaultMessage: 'Learn more about available options.',
+                            }
+                          )}
+                        </EuiLink>
+                      </>
+                    </EuiText>
+                    <EuiSpacer size="s" />
+                    <EuiButton
+                      style={{ width: 'fit-content' }}
+                      size="s"
+                      fill
+                      onClick={applyPivotChangesHandler}
+                      disabled={!isAdvancedPivotEditorApplyButtonEnabled}
+                    >
+                      {i18n.translate(
+                        'xpack.transform.stepDefineForm.advancedEditorApplyButtonText',
+                        {
+                          defaultMessage: 'Apply changes',
+                        }
+                      )}
+                    </EuiButton>
+                  </EuiFlexItem>
+                )}
+              </EuiFlexGroup>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        ) : null}
+        {stepDefineForm.transformFunction === TRANSFORM_FUNCTION.LATEST ? (
+          <LatestFunctionForm latestFunctionService={stepDefineForm.latestFunctionConfig} />
+        ) : null}
       </EuiForm>
       <EuiSpacer size="m" />
       <DataGrid {...pivotPreviewProps} />

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.test.tsx

@@ -59,6 +59,21 @@ describe('Transform: <DefinePivotSummary />', () => {
       searchString: 'the-query',
       searchQuery: 'the-search-query',
       valid: true,
+      validationStatus: {
+        isValid: true,
+      },
+      transformFunction: 'pivot',
+      previewRequest: {
+        pivot: {
+          aggregations: {
+            // @ts-ignore
+            'the-agg-name': agg,
+          },
+          group_by: {
+            'the-group-by-name': groupBy,
+          },
+        },
+      },
     };
 
     const { getByText } = render(

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx

@@ -6,11 +6,10 @@
 
 import React, { Fragment, FC } from 'react';
 
+import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
 
-import { EuiCodeBlock, EuiForm, EuiFormRow, EuiSpacer } from '@elastic/eui';
+import { EuiBadge, EuiCodeBlock, EuiForm, EuiFormRow, EuiSpacer, EuiText } from '@elastic/eui';
-
-import { dictionaryToArray } from '../../../../../../common/types/common';
 
 import { useAppDependencies, useToastNotifications } from '../../../../app_dependencies';
 import {
@@ -27,6 +26,8 @@ import { AggListSummary } from '../aggregation_list';
 import { GroupByListSummary } from '../group_by_list';
 
 import { StepDefineExposedState } from './common';
+import { TRANSFORM_FUNCTION } from '../../../../../../common/constants';
+import { isLatestPartialRequest } from './common/types';
 
 interface Props {
   formState: StepDefineExposedState;
@@ -34,29 +35,35 @@ interface Props {
 }
 
 export const StepDefineSummary: FC<Props> = ({
-  formState: { searchString, searchQuery, groupByList, aggList },
+  formState: {
+    searchString,
+    searchQuery,
+    groupByList,
+    aggList,
+    transformFunction,
+    previewRequest: partialPreviewRequest,
+    validationStatus,
+  },
   searchItems,
 }) => {
   const {
     ml: { DataGrid },
   } = useAppDependencies();
   const toastNotifications = useToastNotifications();
-  const pivotAggsArr = dictionaryToArray(aggList);
+
-  const pivotGroupByArr = dictionaryToArray(groupByList);
   const pivotQuery = getPivotQuery(searchQuery);
 
   const previewRequest = getPreviewTransformRequestBody(
     searchItems.indexPattern.title,
     pivotQuery,
-    pivotGroupByArr,
+    partialPreviewRequest
-    pivotAggsArr
   );
 
   const pivotPreviewProps = usePivotData(
     searchItems.indexPattern.title,
     pivotQuery,
-    aggList,
+    validationStatus,
-    groupByList
+    partialPreviewRequest
   );
 
   const isModifiedQuery =
@@ -64,6 +71,13 @@ export const StepDefineSummary: FC<Props> = ({
     !isDefaultQuery(pivotQuery) &&
     !isMatchAllQuery(pivotQuery);
 
+  let uniqueKeys: string[] = [];
+  let sortField = '';
+  if (isLatestPartialRequest(previewRequest)) {
+    uniqueKeys = previewRequest.latest.unique_key;
+    sortField = previewRequest.latest.sort;
+  }
+
   return (
     <div data-test-subj="transformStepDefineSummary">
       <EuiForm>
@@ -116,21 +130,55 @@ export const StepDefineSummary: FC<Props> = ({
           </EuiFormRow>
         )}
 
-        <EuiFormRow
+        {transformFunction === TRANSFORM_FUNCTION.PIVOT ? (
-          label={i18n.translate('xpack.transform.stepDefineSummary.groupByLabel', {
+          <>
-            defaultMessage: 'Group by',
+            <EuiFormRow
-          })}
+              label={i18n.translate('xpack.transform.stepDefineSummary.groupByLabel', {
-        >
+                defaultMessage: 'Group by',
-          <GroupByListSummary list={groupByList} />
+              })}
-        </EuiFormRow>
+            >
+              <GroupByListSummary list={groupByList} />
+            </EuiFormRow>
 
-        <EuiFormRow
+            <EuiFormRow
-          label={i18n.translate('xpack.transform.stepDefineSummary.aggregationsLabel', {
+              label={i18n.translate('xpack.transform.stepDefineSummary.aggregationsLabel', {
-            defaultMessage: 'Aggregations',
+                defaultMessage: 'Aggregations',
-          })}
+              })}
-        >
+            >
-          <AggListSummary list={aggList} />
+              <AggListSummary list={aggList} />
-        </EuiFormRow>
+            </EuiFormRow>
+          </>
+        ) : (
+          <>
+            <EuiFormRow
+              label={
+                <FormattedMessage
+                  id="xpack.transform.stepDefineForm.uniqueKeysLabel"
+                  defaultMessage="Unique keys"
+                />
+              }
+            >
+              <>
+                {uniqueKeys.map((k) => (
+                  <EuiBadge color="hollow" key={k}>
+                    {k}
+                  </EuiBadge>
+                ))}
+              </>
+            </EuiFormRow>
+
+            <EuiFormRow
+              label={
+                <FormattedMessage
+                  id="xpack.transform.stepDefineForm.sortLabel"
+                  defaultMessage="Sort field"
+                />
+              }
+            >
+              <EuiText>{sortField}</EuiText>
+            </EuiFormRow>
+          </>
+        )}
 
         <EuiSpacer size="m" />
         <DataGrid

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/transform_function_selector.tsx

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+import { i18n } from '@kbn/i18n';
+import { EuiCard, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiSpacer } from '@elastic/eui';
+import { TRANSFORM_FUNCTION, TransformFunction } from '../../../../../../common/constants';
+
+interface TransformFunctionSelectorProps {
+  selectedFunction: TransformFunction;
+  onChange: (update: TransformFunction) => void;
+}
+
+export const TransformFunctionSelector: FC<TransformFunctionSelectorProps> = ({
+  selectedFunction,
+  onChange,
+}) => {
+  const transformFunctions = [
+    {
+      name: TRANSFORM_FUNCTION.PIVOT,
+      helpText: i18n.translate('xpack.transform.stepDefineForm.pivotHelperText', {
+        defaultMessage: 'Aggregate and group your data',
+      }),
+      icon: 'aggregate',
+      title: i18n.translate('xpack.transform.stepDefineForm.pivotLabel', {
+        defaultMessage: 'Pivot',
+      }),
+    },
+    {
+      name: TRANSFORM_FUNCTION.LATEST,
+      helpText: i18n.translate('xpack.transform.stepDefineForm.latestHelperText', {
+        defaultMessage: 'Keep track of your most recent data',
+      }),
+      icon: 'clock',
+      title: i18n.translate('xpack.transform.stepDefineForm.latestLabel', {
+        defaultMessage: 'Latest',
+      }),
+    },
+  ];
+
+  return (
+    <>
+      <EuiFlexGroup gutterSize="m" data-test-subj="transformFunctionSelection">
+        {transformFunctions.map(({ helpText, icon, name, title }) => (
+          <EuiFlexItem key={name} style={{ width: 320 }} grow={false}>
+            <EuiCard
+              icon={<EuiIcon size="xl" type={icon} />}
+              title={title}
+              description={helpText}
+              data-test-subj={`transformCreation-${name}-option${
+                selectedFunction === name ? ' selectedFunction' : ''
+              }`}
+              selectable={{
+                onClick: () => {
+                  // Only allow one function selected at a time and don't allow deselection
+                  if (selectedFunction === name) {
+                    return;
+                  }
+                  onChange(name);
+                },
+                isSelected: selectedFunction === name,
+              }}
+            />
+          </EuiFlexItem>
+        ))}
+      </EuiFlexGroup>
+      <EuiSpacer size="m" />
+    </>
+  );
+};

x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx

@@ -50,7 +50,6 @@ import {
   transformSettingsMaxPageSearchSizeValidator,
 } from '../../../../common/validators';
 import { StepDefineExposedState } from '../step_define/common';
-import { dictionaryToArray } from '../../../../../../common/types/common';
 
 export interface StepDetailsExposedState {
   continuousModeDateField: string;
@@ -179,15 +178,12 @@ export const StepDetailsForm: FC<Props> = React.memo(
     useEffect(() => {
       // use an IIFE to avoid returning a Promise to useEffect.
       (async function () {
-        const { searchQuery, groupByList, aggList } = stepDefineState;
+        const { searchQuery, previewRequest: partialPreviewRequest } = stepDefineState;
-        const pivotAggsArr = dictionaryToArray(aggList);
-        const pivotGroupByArr = dictionaryToArray(groupByList);
         const pivotQuery = getPivotQuery(searchQuery);
         const previewRequest = getPreviewTransformRequestBody(
           searchItems.indexPattern.title,
           pivotQuery,
-          pivotGroupByArr,
+          partialPreviewRequest
-          pivotAggsArr
         );
 
         const transformPreview = await api.getTransformsPreview(previewRequest);

x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { Fragment, FC, useEffect, useRef, useState, createContext } from 'react';
+import React, { Fragment, FC, useEffect, useRef, useState, createContext, useMemo } from 'react';
 
 import { i18n } from '@kbn/i18n';
 
@@ -100,7 +100,11 @@ export const Wizard: FC<WizardProps> = React.memo(({ cloneConfig, searchItems })
 
   // The DEFINE state
   const [stepDefineState, setStepDefineState] = useState(
-    applyTransformConfigToDefineState(getDefaultStepDefineState(searchItems), cloneConfig)
+    applyTransformConfigToDefineState(
+      getDefaultStepDefineState(searchItems),
+      cloneConfig,
+      indexPattern
+    )
   );
 
   // The DETAILS state
@@ -108,18 +112,6 @@ export const Wizard: FC<WizardProps> = React.memo(({ cloneConfig, searchItems })
     applyTransformConfigToDetailsState(getDefaultStepDetailsState(), cloneConfig)
   );
 
-  const stepDetails =
-    currentStep === WIZARD_STEPS.DETAILS ? (
-      <StepDetailsForm
-        onChange={setStepDetailsState}
-        overrides={stepDetailsState}
-        searchItems={searchItems}
-        stepDefineState={stepDefineState}
-      />
-    ) : (
-      <StepDetailsSummary {...stepDetailsState} />
-    );
-
   // The CREATE state
   const [stepCreateState, setStepCreateState] = useState(getDefaultStepCreateState);
 
@@ -157,22 +149,8 @@ export const Wizard: FC<WizardProps> = React.memo(({ cloneConfig, searchItems })
     stepDetailsState
   );
 
-  const stepCreate =
+  const stepDefine = useMemo(() => {
-    currentStep === WIZARD_STEPS.CREATE ? (
+    return {
-      <StepCreateForm
-        createIndexPattern={stepDetailsState.createIndexPattern}
-        transformId={stepDetailsState.transformId}
-        transformConfig={transformConfig}
-        onChange={setStepCreateState}
-        overrides={stepCreateState}
-        timeFieldName={stepDetailsState.indexPatternTimeField}
-      />
-    ) : (
-      <StepCreateSummary />
-    );
-
-  const stepsConfig = [
-    {
       title: i18n.translate('xpack.transform.transformsWizard.stepConfigurationTitle', {
         defaultMessage: 'Configuration',
       }),
@@ -185,14 +163,26 @@ export const Wizard: FC<WizardProps> = React.memo(({ cloneConfig, searchItems })
           searchItems={searchItems}
         />
       ),
-    },
+    };
-    {
+  }, [currentStep, stepDefineState, setCurrentStep, setStepDefineState, searchItems]);
+
+  const stepDetails = useMemo(() => {
+    return {
       title: i18n.translate('xpack.transform.transformsWizard.stepDetailsTitle', {
         defaultMessage: 'Transform details',
       }),
       children: (
         <Fragment>
-          {stepDetails}
+          {currentStep === WIZARD_STEPS.DETAILS ? (
+            <StepDetailsForm
+              onChange={setStepDetailsState}
+              overrides={stepDetailsState}
+              searchItems={searchItems}
+              stepDefineState={stepDefineState}
+            />
+          ) : (
+            <StepDetailsSummary {...stepDetailsState} />
+          )}
           {currentStep === WIZARD_STEPS.DETAILS && (
             <WizardNav
               previous={() => {
@@ -205,22 +195,47 @@ export const Wizard: FC<WizardProps> = React.memo(({ cloneConfig, searchItems })
         </Fragment>
       ),
       status: currentStep >= WIZARD_STEPS.DETAILS ? undefined : ('incomplete' as EuiStepStatus),
-    },
+    };
-    {
+  }, [currentStep, setStepDetailsState, stepDetailsState, searchItems, stepDefineState]);
+
+  const stepCreate = useMemo(() => {
+    return {
       title: i18n.translate('xpack.transform.transformsWizard.stepCreateTitle', {
         defaultMessage: 'Create',
       }),
       children: (
         <Fragment>
-          {stepCreate}
+          {currentStep === WIZARD_STEPS.CREATE ? (
+            <StepCreateForm
+              createIndexPattern={stepDetailsState.createIndexPattern}
+              transformId={stepDetailsState.transformId}
+              transformConfig={transformConfig}
+              onChange={setStepCreateState}
+              overrides={stepCreateState}
+              timeFieldName={stepDetailsState.indexPatternTimeField}
+            />
+          ) : (
+            <StepCreateSummary />
+          )}
           {currentStep === WIZARD_STEPS.CREATE && !stepCreateState.created && (
             <WizardNav previous={() => setCurrentStep(WIZARD_STEPS.DETAILS)} />
           )}
         </Fragment>
       ),
       status: currentStep >= WIZARD_STEPS.CREATE ? undefined : ('incomplete' as EuiStepStatus),
-    },
+    };
-  ];
+  }, [
+    currentStep,
+    setCurrentStep,
+    stepDetailsState.createIndexPattern,
+    stepDetailsState.transformId,
+    transformConfig,
+    setStepCreateState,
+    stepCreateState,
+    stepDetailsState.indexPatternTimeField,
+  ]);
+
+  const stepsConfig = [stepDefine, stepDetails, stepCreate];
 
   return (
     <CreateTransformWizardContext.Provider value={{ indexPattern }}>

x-pack/plugins/transform/public/app/sections/transform_management/components/action_edit/use_edit_action.tsx

@@ -6,7 +6,7 @@
 
 import React, { useContext, useMemo, useState } from 'react';
 
-import { TransformPivotConfig } from '../../../../../../common/types/transform';
+import { TransformConfigUnion } from '../../../../../../common/types/transform';
 
 import { TransformListAction, TransformListRow } from '../../../../common';
 import { AuthorizationContext } from '../../../../lib/authorization';
@@ -16,10 +16,10 @@ import { editActionNameText, EditActionName } from './edit_action_name';
 export const useEditAction = (forceDisable: boolean) => {
   const { canCreateTransform } = useContext(AuthorizationContext).capabilities;
 
-  const [config, setConfig] = useState<TransformPivotConfig>();
+  const [config, setConfig] = useState<TransformConfigUnion>();
   const [isFlyoutVisible, setIsFlyoutVisible] = useState(false);
   const closeFlyout = () => setIsFlyoutVisible(false);
-  const showFlyout = (newConfig: TransformPivotConfig) => {
+  const showFlyout = (newConfig: TransformConfigUnion) => {
     setConfig(newConfig);
     setIsFlyoutVisible(true);
   };

x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_flyout.tsx

@@ -24,7 +24,7 @@ import {
 } from '@elastic/eui';
 
 import { isPostTransformsUpdateResponseSchema } from '../../../../../../common/api_schemas/type_guards';
-import { TransformPivotConfig } from '../../../../../../common/types/transform';
+import { TransformConfigUnion } from '../../../../../../common/types/transform';
 
 import { getErrorMessage } from '../../../../../../common/utils/errors';
 
@@ -42,7 +42,7 @@ import {
 
 interface EditTransformFlyoutProps {
   closeFlyout: () => void;
-  config: TransformPivotConfig;
+  config: TransformConfigUnion;
 }
 
 export const EditTransformFlyout: FC<EditTransformFlyoutProps> = ({ closeFlyout, config }) => {

x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/use_edit_transform_flyout.ts

@@ -12,7 +12,7 @@ import { useReducer } from 'react';
 import { i18n } from '@kbn/i18n';
 
 import { PostTransformsUpdateRequestSchema } from '../../../../../../common/api_schemas/update_transforms';
-import { TransformPivotConfig } from '../../../../../../common/types/transform';
+import { TransformConfigUnion } from '../../../../../../common/types/transform';
 import { getNestedProperty, setNestedProperty } from '../../../../../../common/utils/object_utils';
 
 // This custom hook uses nested reducers to provide a generic framework to manage form state
@@ -167,7 +167,7 @@ const validate = {
 export const initializeField = (
   formFieldName: string,
   configFieldName: string,
-  config: TransformPivotConfig,
+  config: TransformConfigUnion,
   overloads?: Partial<FormField>
 ): FormField => {
   const defaultValue = overloads?.defaultValue !== undefined ? overloads.defaultValue : '';
@@ -207,7 +207,7 @@ interface Action {
 // Considers options like if a value is nullable or optional.
 const getUpdateValue = (
   attribute: keyof EditTransformFlyoutFieldsState,
-  config: TransformPivotConfig,
+  config: TransformConfigUnion,
   formState: EditTransformFlyoutFieldsState,
   enforceFormValue = false
 ) => {
@@ -245,7 +245,7 @@ const getUpdateValue = (
 // request object suitable to be sent to the
 // transform update API endpoint.
 export const applyFormFieldsToTransformConfig = (
-  config: TransformPivotConfig,
+  config: TransformConfigUnion,
   formState: EditTransformFlyoutFieldsState
 ): PostTransformsUpdateRequestSchema =>
   // Iterates over all form fields and only if necessary applies them to
@@ -257,7 +257,7 @@ export const applyFormFieldsToTransformConfig = (
 
 // Takes in a transform configuration and returns
 // the default state to populate the form.
-export const getDefaultState = (config: TransformPivotConfig): EditTransformFlyoutState => ({
+export const getDefaultState = (config: TransformConfigUnion): EditTransformFlyoutState => ({
   formFields: {
     // top level attributes
     description: initializeField('description', 'description', config),
@@ -319,7 +319,7 @@ const formFieldReducer = (state: FormField, value: string): FormField => {
 // - `formFieldReducer` to update the actions field
 // - compares the most recent state against the original one to update `isFormTouched`
 // - sets `isFormValid` to have a flag if any of the form fields contains an error.
-export const formReducerFactory = (config: TransformPivotConfig) => {
+export const formReducerFactory = (config: TransformConfigUnion) => {
   const defaultState = getDefaultState(config);
   const defaultFieldValues = Object.values(defaultState.formFields).map((f) => f.value);
 
@@ -341,7 +341,7 @@ export const formReducerFactory = (config: TransformPivotConfig) => {
   };
 };
 
-export const useEditTransformFlyout = (config: TransformPivotConfig) => {
+export const useEditTransformFlyout = (config: TransformConfigUnion) => {
   return useReducer(formReducerFactory(config), getDefaultState(config));
 };
 

x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx

@@ -6,7 +6,7 @@
 
 import React, { useMemo, FC } from 'react';
 
-import { TransformPivotConfig } from '../../../../../../common/types/transform';
+import { TransformConfigUnion } from '../../../../../../common/types/transform';
 
 import { useAppDependencies, useToastNotifications } from '../../../../app_dependencies';
 import { getPivotQuery } from '../../../../common';
@@ -19,7 +19,7 @@ import {
 } from '../../../create_transform/components/step_define/';
 
 interface ExpandedRowPreviewPaneProps {
-  transformConfig: TransformPivotConfig;
+  transformConfig: TransformConfigUnion;
 }
 
 export const ExpandedRowPreviewPane: FC<ExpandedRowPreviewPaneProps> = ({ transformConfig }) => {
@@ -28,7 +28,7 @@ export const ExpandedRowPreviewPane: FC<ExpandedRowPreviewPaneProps> = ({ transf
   } = useAppDependencies();
   const toastNotifications = useToastNotifications();
 
-  const { aggList, groupByList, searchQuery } = useMemo(
+  const { searchQuery, validationStatus, previewRequest } = useMemo(
     () =>
       applyTransformConfigToDefineState(
         getDefaultStepDefineState({} as SearchItems),
@@ -43,7 +43,12 @@ export const ExpandedRowPreviewPane: FC<ExpandedRowPreviewPaneProps> = ({ transf
     ? transformConfig.source.index.join(',')
     : transformConfig.source.index;
 
-  const pivotPreviewProps = usePivotData(indexPatternTitle, pivotQuery, aggList, groupByList);
+  const pivotPreviewProps = usePivotData(
+    indexPatternTitle,
+    pivotQuery,
+    validationStatus,
+    previewRequest
+  );
 
   return (
     <DataGrid

x-pack/plugins/transform/server/routes/api/error_utils.ts

@@ -38,26 +38,25 @@ export function fillResultsWithTimeouts({ results, id, items, action }: Params)
         )
       : '';
 
-  const error = {
+  const reason = i18n.translate(
-    response: {
+    'xpack.transform.models.transformService.requestToActionTimedOutErrorMessage',
-      error: {
+    {
-        root_cause: [
+      defaultMessage: `Request to {action} '{id}' timed out. {extra}`,
-          {
+      values: {
-            reason: i18n.translate(
+        id,
-              'xpack.transform.models.transformService.requestToActionTimedOutErrorMessage',
+        action,
-              {
+        extra,
-                defaultMessage: `Request to {action} '{id}' timed out. {extra}`,
-                values: {
-                  id,
-                  action,
-                  extra,
-                },
-              }
-            ),
-          },
-        ],
       },
-    },
+    }
+  );
+
+  const error = {
+    reason,
+    root_cause: [
+      {
+        reason,
+      },
+    ],
   };
 
   const newResults: CommonResponseStatusSchema | DeleteTransformsResponseSchema = {};
@@ -66,6 +65,7 @@ export function fillResultsWithTimeouts({ results, id, items, action }: Params)
     if (results[currentVal.id] === undefined) {
       accumResults[currentVal.id] = {
         success: false,
+        // @ts-ignore
         error,
       };
     } else {

x-pack/plugins/transform/server/routes/api/transforms.ts

@@ -450,7 +450,7 @@ async function deleteTransforms(
           ? transformConfig.dest.index[0]
           : transformConfig.dest.index;
       } catch (getTransformConfigError) {
-        transformDeleted.error = wrapError(getTransformConfigError);
+        transformDeleted.error = getTransformConfigError.meta.body.error;
         results[transformId] = {
           transformDeleted,
           destIndexDeleted,
@@ -471,7 +471,7 @@ async function deleteTransforms(
           });
           destIndexDeleted.success = true;
         } catch (deleteIndexError) {
-          destIndexDeleted.error = wrapError(deleteIndexError);
+          destIndexDeleted.error = deleteIndexError.meta.body.error;
         }
       }
 
@@ -487,7 +487,7 @@ async function deleteTransforms(
             destIndexPatternDeleted.success = true;
           }
         } catch (deleteDestIndexPatternError) {
-          destIndexPatternDeleted.error = wrapError(deleteDestIndexPatternError);
+          destIndexPatternDeleted.error = deleteDestIndexPatternError.meta.body.error;
         }
       }
 
@@ -498,7 +498,7 @@ async function deleteTransforms(
         });
         transformDeleted.success = true;
       } catch (deleteTransformJobError) {
-        transformDeleted.error = wrapError(deleteTransformJobError);
+        transformDeleted.error = deleteTransformJobError.meta.body.error;
         if (deleteTransformJobError.statusCode === 403) {
           return response.forbidden();
         }
@@ -519,7 +519,7 @@ async function deleteTransforms(
           action: TRANSFORM_ACTIONS.DELETE,
         });
       }
-      results[transformId] = { transformDeleted: { success: false, error: JSON.stringify(e) } };
+      results[transformId] = { transformDeleted: { success: false, error: e.meta.body.error } };
     }
   }
   return results;
@@ -579,7 +579,7 @@ async function startTransforms(
           action: TRANSFORM_ACTIONS.START,
         });
       }
-      results[transformId] = { success: false, error: JSON.stringify(e) };
+      results[transformId] = { success: false, error: e.meta.body.error };
     }
   }
   return results;
@@ -628,7 +628,7 @@ async function stopTransforms(
           action: TRANSFORM_ACTIONS.STOP,
         });
       }
-      results[transformId] = { success: false, error: JSON.stringify(e) };
+      results[transformId] = { success: false, error: e.meta.body.error };
     }
   }
   return results;

x-pack/test/api_integration/apis/transform/index.ts

@@ -34,5 +34,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./transforms_preview'));
     loadTestFile(require.resolve('./transforms_stats'));
     loadTestFile(require.resolve('./transforms_update'));
+    loadTestFile(require.resolve('./transforms_create'));
   });
 }

x-pack/test/api_integration/apis/transform/start_transforms.ts

@@ -75,7 +75,7 @@ export default ({ getService }: FtrProviderContext) => {
           .expect(200);
 
         expect(body[transformId].success).to.eql(false);
-        expect(typeof body[transformId].error).to.eql('string');
+        expect(typeof body[transformId].error).to.eql('object');
 
         await transform.api.waitForTransformState(transformId, TRANSFORM_STATE.STOPPED);
         await transform.api.waitForIndicesNotToExist(destinationIndex);

x-pack/test/api_integration/apis/transform/stop_transforms.ts

@@ -97,7 +97,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         expect(isStopTransformsResponseSchema(body)).to.eql(true);
         expect(body[transformId].success).to.eql(false);
-        expect(typeof body[transformId].error).to.eql('string');
+        expect(typeof body[transformId].error).to.eql('object');
 
         await transform.api.waitForTransformStateNotToBe(transformId, TRANSFORM_STATE.STOPPED);
         await transform.api.waitForIndicesToExist(destinationIndex);

x-pack/test/api_integration/apis/transform/transforms_create.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { COMMON_REQUEST_HEADERS } from '../../../functional/services/ml/common_api';
+import { USER } from '../../../functional/services/transform/security_common';
+
+import { generateTransformConfig } from './common';
+
+export default ({ getService }: FtrProviderContext) => {
+  const esArchiver = getService('esArchiver');
+  const supertest = getService('supertestWithoutAuth');
+  const transform = getService('transform');
+
+  describe('/api/transform/transforms/{transformId}/ create', function () {
+    before(async () => {
+      await esArchiver.loadIfNeeded('ml/farequote');
+      await transform.testResources.setKibanaTimeZoneToUTC();
+    });
+
+    after(async () => {
+      await transform.api.cleanTransformIndices();
+    });
+
+    it('should not allow pivot and latest configs is same transform', async () => {
+      const transformId = 'test_transform_id';
+
+      const { body } = await supertest
+        .put(`/api/transform/transforms/${transformId}`)
+        .auth(
+          USER.TRANSFORM_POWERUSER,
+          transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER)
+        )
+        .set(COMMON_REQUEST_HEADERS)
+        .send({
+          ...generateTransformConfig(transformId),
+          latest: {
+            unique_key: ['country', 'gender'],
+            sort: 'infected',
+          },
+        })
+        .expect(400);
+
+      expect(body.message).to.eql('[request body]: pivot and latest are not allowed together');
+    });
+
+    it('should ensure if pivot or latest is provided', async () => {
+      const transformId = 'test_transform_id';
+
+      const { pivot, ...config } = generateTransformConfig(transformId);
+
+      const { body } = await supertest
+        .put(`/api/transform/transforms/${transformId}`)
+        .auth(
+          USER.TRANSFORM_POWERUSER,
+          transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER)
+        )
+        .set(COMMON_REQUEST_HEADERS)
+        .send(config)
+        .expect(400);
+
+      expect(body.message).to.eql(
+        '[request body]: pivot or latest is required for transform configuration'
+      );
+    });
+  });
+};