[ML] Transforms: Create optional data view on server side as part of transform create API call. (#160513)

When creating a transform, users have the option to create a Kibana data
view as part of the creation process. So far we've done that as a
separate call from the client side. This PR adds query param options to
the API endpoint to optionally create the data view server side.
API integration tests have been extended to test the new feature.

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -148,16 +148,29 @@ export interface PutTransformsPivotRequestSchema
 
 export type PutTransformsLatestRequestSchema = Omit<PutTransformsRequestSchema, 'pivot'>;
 
+export const putTransformsQuerySchema = schema.object({
+  createDataView: schema.boolean({ defaultValue: false }),
+  timeFieldName: schema.maybe(schema.string()),
+});
+
+export type PutTransformsQuerySchema = TypeOf<typeof putTransformsQuerySchema>;
+
 interface TransformCreated {
   transform: TransformId;
 }
-interface TransformCreatedError {
-  id: TransformId;
+interface DataViewCreated {
+  id: string;
+}
+interface CreatedError {
+  id: string;
   error: any;
 }
+
 export interface PutTransformsResponseSchema {
   transformsCreated: TransformCreated[];
-  errors: TransformCreatedError[];
+  dataViewsCreated: DataViewCreated[];
+  dataViewsErrors: CreatedError[];
+  errors: CreatedError[];
 }
 
 // POST transforms/_preview

x-pack/plugins/transform/public/app/hooks/use_create_transform.tsx

@@ -27,6 +27,8 @@ import { useRefreshTransformList } from './use_refresh_transform_list';
 interface CreateTransformArgs {
   transformId: TransformId;
   transformConfig: PutTransformsRequestSchema;
+  createDataView: boolean;
+  timeFieldName?: string;
 }
 
 export const useCreateTransform = () => {
@@ -48,10 +50,16 @@ export const useCreateTransform = () => {
   }
 
   const mutation = useMutation({
-    mutationFn: ({ transformId, transformConfig }: CreateTransformArgs) => {
+    mutationFn: ({
+      transformId,
+      transformConfig,
+      createDataView = false,
+      timeFieldName,
+    }: CreateTransformArgs) => {
       return http.put<PutTransformsResponseSchema>(
         addInternalBasePath(`transforms/${transformId}`),
         {
+          query: { createDataView, timeFieldName },
           body: JSON.stringify(transformConfig),
           version: '1',
         }

x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx

@@ -28,9 +28,6 @@ import { toMountPoint } from '@kbn/react-kibana-mount';
 
 import { DISCOVER_APP_LOCATOR } from '@kbn/discover-plugin/common';
 
-import { DuplicateDataViewError } from '@kbn/data-plugin/public';
-import type { RuntimeField } from '@kbn/data-views-plugin/common';
-import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import { PROGRESS_REFRESH_INTERVAL_MS } from '../../../../../../common/constants';
 
 import { getErrorMessage } from '../../../../../../common/utils/errors';
@@ -44,7 +41,7 @@ import {
   PutTransformsLatestRequestSchema,
   PutTransformsPivotRequestSchema,
 } from '../../../../../../common/api_schemas/transforms';
-import { isContinuousTransform, isLatestTransform } from '../../../../../../common/types/transform';
+import { isContinuousTransform } from '../../../../../../common/types/transform';
 import { TransformAlertFlyout } from '../../../../../alerting/transform_alerting_flyout';
 
 export interface StepDetailsExposedState {
@@ -87,8 +84,7 @@ export const StepCreateForm: FC<StepCreateFormProps> = React.memo(
     const [discoverLink, setDiscoverLink] = useState<string>();
 
     const toastNotifications = useToastNotifications();
-    const { application, data, i18n: i18nStart, share, theme } = useAppDependencies();
-    const dataViews = data.dataViews;
+    const { application, i18n: i18nStart, share, theme } = useAppDependencies();
     const isDiscoverAvailable = application.capabilities.discover?.show ?? false;
 
     useEffect(() => {
@@ -128,13 +124,13 @@ export const StepCreateForm: FC<StepCreateFormProps> = React.memo(
       setLoading(true);
 
       createTransform(
-        { transformId, transformConfig },
+        { transformId, transformConfig, createDataView, timeFieldName },
         {
           onError: () => setCreated(false),
-          onSuccess: () => {
+          onSuccess: (resp) => {
             setCreated(true);
-            if (createDataView) {
-              createKibanaDataView();
+            if (resp.dataViewsCreated.length === 1) {
+              setDataViewId(resp.dataViewsCreated[0].id);
             }
             if (startAfterCreation) {
               startTransform();
@@ -155,57 +151,6 @@ export const StepCreateForm: FC<StepCreateFormProps> = React.memo(
       });
     }
 
-    const createKibanaDataView = async () => {
-      setLoading(true);
-      const dataViewName = transformConfig.dest.index;
-      const runtimeMappings = transformConfig.source.runtime_mappings as Record<
-        string,
-        RuntimeField
-      >;
-
-      try {
-        const newDataView = await dataViews.createAndSave(
-          {
-            title: dataViewName,
-            timeFieldName,
-            ...(isPopulatedObject(runtimeMappings) && isLatestTransform(transformConfig)
-              ? { runtimeFieldMap: runtimeMappings }
-              : {}),
-            allowNoIndex: true,
-          },
-          false,
-          true
-        );
-
-        setDataViewId(newDataView.id);
-        setLoading(false);
-        return true;
-      } catch (e) {
-        if (e instanceof DuplicateDataViewError) {
-          toastNotifications.addDanger(
-            i18n.translate('xpack.transform.stepCreateForm.duplicateDataViewErrorMessage', {
-              defaultMessage:
-                'An error occurred creating the Kibana data view {dataViewName}: The data view already exists.',
-              values: { dataViewName },
-            })
-          );
-        } else {
-          toastNotifications.addDanger({
-            title: i18n.translate('xpack.transform.stepCreateForm.createDataViewErrorMessage', {
-              defaultMessage: 'An error occurred creating the Kibana data view {dataViewName}:',
-              values: { dataViewName },
-            }),
-            text: toMountPoint(<ToastNotificationText text={getErrorMessage(e)} />, {
-              theme,
-              i18n: i18nStart,
-            }),
-          });
-          setLoading(false);
-          return false;
-        }
-      }
-    };
-
     const isBatchTransform = typeof transformConfig.sync === 'undefined';
 
     useEffect(() => {

x-pack/plugins/transform/server/routes/api/transforms_create/register_route.ts

@@ -11,15 +11,19 @@ import {
 } from '../../../../common/api_schemas/common';
 import {
   putTransformsRequestSchema,
+  putTransformsQuerySchema,
   type PutTransformsRequestSchema,
+  type PutTransformsQuerySchema,
 } from '../../../../common/api_schemas/transforms';
 import { addInternalBasePath } from '../../../../common/constants';
 
 import type { RouteDependencies } from '../../../types';
 
-import { routeHandler } from './route_handler';
+import { routeHandlerFactory } from './route_handler_factory';
+
+export function registerRoute(routeDependencies: RouteDependencies) {
+  const { router, license } = routeDependencies;
 
-export function registerRoute({ router, license }: RouteDependencies) {
   /**
    * @apiGroup Transforms
    *
@@ -28,6 +32,7 @@ export function registerRoute({ router, license }: RouteDependencies) {
    * @apiDescription Creates a transform
    *
    * @apiSchema (params) transformIdParamSchema
+   * @apiSchema (query) transformIdParamSchema
    * @apiSchema (body) putTransformsRequestSchema
    */
   router.versioned
@@ -35,18 +40,21 @@ export function registerRoute({ router, license }: RouteDependencies) {
       path: addInternalBasePath('transforms/{transformId}'),
       access: 'internal',
     })
-    .addVersion<TransformIdParamSchema, undefined, PutTransformsRequestSchema>(
+    .addVersion<TransformIdParamSchema, PutTransformsQuerySchema, PutTransformsRequestSchema>(
       {
         version: '1',
         validate: {
           request: {
             params: transformIdParamSchema,
+            query: putTransformsQuerySchema,
             body: putTransformsRequestSchema,
           },
         },
       },
-      license.guardApiRoute<TransformIdParamSchema, undefined, PutTransformsRequestSchema>(
-        routeHandler
-      )
+      license.guardApiRoute<
+        TransformIdParamSchema,
+        PutTransformsQuerySchema,
+        PutTransformsRequestSchema
+      >(routeHandlerFactory(routeDependencies))
     );
 }

x-pack/plugins/transform/server/routes/api/transforms_create/route_handler.ts

@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import type { RequestHandler } from '@kbn/core/server';
-
-import type { TransformIdParamSchema } from '../../../../common/api_schemas/common';
-import type {
-  PutTransformsRequestSchema,
-  PutTransformsResponseSchema,
-} from '../../../../common/api_schemas/transforms';
-
-import type { TransformRequestHandlerContext } from '../../../services/license';
-
-import { wrapEsError } from '../../utils/error_utils';
-
-export const routeHandler: RequestHandler<
-  TransformIdParamSchema,
-  undefined,
-  PutTransformsRequestSchema,
-  TransformRequestHandlerContext
-> = async (ctx, req, res) => {
-  const { transformId } = req.params;
-
-  const response: PutTransformsResponseSchema = {
-    transformsCreated: [],
-    errors: [],
-  };
-
-  const esClient = (await ctx.core).elasticsearch.client;
-
-  try {
-    const resp = await esClient.asCurrentUser.transform.putTransform({
-      // @ts-expect-error @elastic/elasticsearch group_by is expected to be optional in TransformPivot
-      body: req.body,
-      transform_id: transformId,
-    });
-
-    if (resp.acknowledged) {
-      response.transformsCreated.push({ transform: transformId });
-    } else {
-      response.errors.push({
-        id: transformId,
-        error: wrapEsError(resp),
-      });
-    }
-  } catch (e) {
-    response.errors.push({
-      id: transformId,
-      error: wrapEsError(e),
-    });
-  }
-
-  return res.ok({ body: response });
-};

x-pack/plugins/transform/server/routes/api/transforms_create/route_handler_factory.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { RequestHandler } from '@kbn/core/server';
+import type { RuntimeField } from '@kbn/data-views-plugin/common';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object';
+
+import type { TransformIdParamSchema } from '../../../../common/api_schemas/common';
+import type {
+  PutTransformsRequestSchema,
+  PutTransformsQuerySchema,
+  PutTransformsResponseSchema,
+} from '../../../../common/api_schemas/transforms';
+import { isLatestTransform } from '../../../../common/types/transform';
+
+import type { RouteDependencies } from '../../../types';
+import type { TransformRequestHandlerContext } from '../../../services/license';
+
+import { wrapEsError } from '../../utils/error_utils';
+
+export const routeHandlerFactory: (
+  routeDependencies: RouteDependencies
+) => RequestHandler<
+  TransformIdParamSchema,
+  PutTransformsQuerySchema,
+  PutTransformsRequestSchema,
+  TransformRequestHandlerContext
+> = (routeDependencies) => async (ctx, req, res) => {
+  const { coreStart, dataViews } = routeDependencies;
+  const { transformId } = req.params;
+  const { createDataView, timeFieldName } = req.query;
+
+  const response: PutTransformsResponseSchema = {
+    dataViewsCreated: [],
+    dataViewsErrors: [],
+    transformsCreated: [],
+    errors: [],
+  };
+
+  const esClient = (await ctx.core).elasticsearch.client;
+
+  try {
+    const resp = await esClient.asCurrentUser.transform.putTransform({
+      // @ts-expect-error @elastic/elasticsearch group_by is expected to be optional in TransformPivot
+      body: req.body,
+      transform_id: transformId,
+    });
+
+    if (resp.acknowledged) {
+      response.transformsCreated.push({ transform: transformId });
+    } else {
+      response.errors.push({
+        id: transformId,
+        error: wrapEsError(resp),
+      });
+    }
+  } catch (e) {
+    response.errors.push({
+      id: transformId,
+      error: wrapEsError(e),
+    });
+  }
+
+  if (createDataView) {
+    const { savedObjects, elasticsearch } = coreStart;
+    const dataViewsService = await dataViews.dataViewsServiceFactory(
+      savedObjects.getScopedClient(req),
+      elasticsearch.client.asScoped(req).asCurrentUser,
+      req
+    );
+
+    const dataViewName = req.body.dest.index;
+    const runtimeMappings = req.body.source.runtime_mappings as Record<string, RuntimeField>;
+
+    try {
+      const dataViewsResp = await dataViewsService.createAndSave(
+        {
+          title: dataViewName,
+          timeFieldName,
+          // Adding runtime mappings for transforms of type latest only here
+          // since only they will want to replicate the source index mapping.
+          // Pivot type transforms have index mappings that cannot be
+          // inferred from the source index.
+          ...(isPopulatedObject(runtimeMappings) && isLatestTransform(req.body)
+            ? { runtimeFieldMap: runtimeMappings }
+            : {}),
+          allowNoIndex: true,
+        },
+        false,
+        true
+      );
+
+      if (dataViewsResp.id) {
+        response.dataViewsCreated = [{ id: dataViewsResp.id }];
+      }
+    } catch (error) {
+      // For the error id we use the transform id
+      // because in case of an error we don't get a data view id.
+      response.dataViewsErrors = [{ id: transformId, error }];
+    }
+  }
+
+  return res.ok({ body: response });
+};

x-pack/plugins/translations/translations/fr-FR.json

@@ -40449,9 +40449,7 @@
     "xpack.transform.forceDeleteTransformMessage": "Supprimer {count} {count, plural, one {transformation} many {transformations} other {transformations}}",
     "xpack.transform.managedTransformsWarningCallout": "{count, plural, one {Cette transformation} many {Au moins l''une de ces transformations} other {Au moins l''une de ces transformations}} est préconfigurée par Elastic. Le fait de {action} {count, plural, one {la} many {les} other {les}} avec une heure de fin spécifique peut avoir un impact sur d'autres éléments du produit.",
     "xpack.transform.multiTransformActionsMenu.transformsCount": "Sélection effectuée de {count} {count, plural, one {transformation} many {transformations} other {transformations}}",
-    "xpack.transform.stepCreateForm.createDataViewErrorMessage": "Une erreur est survenue lors de la création de la vue de données Kibana {dataViewName} :",
     "xpack.transform.stepCreateForm.createTransformErrorMessage": "Une erreur s'est produite lors de la création de la transformation {transformId} :",
-    "xpack.transform.stepCreateForm.duplicateDataViewErrorMessage": "Une erreur est survenue lors de la création de la vue de données Kibana {dataViewName} : La vue de données existe déjà.",
     "xpack.transform.stepDefineForm.invalidKuerySyntaxErrorMessageQueryBar": "Requête non valide : {queryErrorMessage}",
     "xpack.transform.stepDefineForm.queryPlaceholderKql": "Par exemple, {example}",
     "xpack.transform.stepDefineForm.queryPlaceholderLucene": "Par exemple, {example}",

x-pack/plugins/translations/translations/ja-JP.json

@@ -40447,9 +40447,7 @@
     "xpack.transform.forceDeleteTransformMessage": "{count}{count, plural, other {変換}}の削除",
     "xpack.transform.managedTransformsWarningCallout": "{count, plural, other {これらの変換のうちの少なくとも1個の変換}}はElasticによってあらかじめ構成されています。{count, plural, other {それらを}}{action}すると、製品の他の部分に影響する可能性があります。",
     "xpack.transform.multiTransformActionsMenu.transformsCount": "{count}個の{count, plural, other {変換}}を選択済み",
-    "xpack.transform.stepCreateForm.createDataViewErrorMessage": "Kibanaデータビュー{dataViewName}の作成中にエラーが発生しました：",
     "xpack.transform.stepCreateForm.createTransformErrorMessage": "変換 {transformId} の取得中にエラーが発生しました。",
-    "xpack.transform.stepCreateForm.duplicateDataViewErrorMessage": "Kibanaデータビュー{dataViewName}の作成中にエラーが発生しました：データビューはすでに存在します。",
     "xpack.transform.stepDefineForm.invalidKuerySyntaxErrorMessageQueryBar": "無効なクエリ：{queryErrorMessage}",
     "xpack.transform.stepDefineForm.queryPlaceholderKql": "例: {example}.",
     "xpack.transform.stepDefineForm.queryPlaceholderLucene": "例: {example}.",

x-pack/plugins/translations/translations/zh-CN.json

@@ -40441,9 +40441,7 @@
     "xpack.transform.forceDeleteTransformMessage": "删除 {count} {count, plural, other {转换}}",
     "xpack.transform.managedTransformsWarningCallout": "{count, plural, other {至少一个此类转换}}由 Elastic 预配置；{action} {count, plural, other {这些转换}}可能会影响该产品的其他部分。",
     "xpack.transform.multiTransformActionsMenu.transformsCount": "已选定 {count} 个{count, plural, other {转换}}",
-    "xpack.transform.stepCreateForm.createDataViewErrorMessage": "创建 Kibana 数据视图 {dataViewName} 时发生错误：",
     "xpack.transform.stepCreateForm.createTransformErrorMessage": "创建转换 {transformId} 时出错：",
-    "xpack.transform.stepCreateForm.duplicateDataViewErrorMessage": "创建 Kibana 数据视图 {dataViewName} 时发生错误：数据视图已存在。",
     "xpack.transform.stepDefineForm.invalidKuerySyntaxErrorMessageQueryBar": "无效查询：{queryErrorMessage}",
     "xpack.transform.stepDefineForm.queryPlaceholderKql": "例如，{example}",
     "xpack.transform.stepDefineForm.queryPlaceholderLucene": "例如，{example}",

x-pack/test/api_integration/apis/transform/transforms_create.ts

@@ -10,7 +10,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 import { getCommonRequestHeader } from '../../../functional/services/ml/common_api';
 import { USER } from '../../../functional/services/transform/security_common';
 
-import { generateTransformConfig } from './common';
+import { generateTransformConfig, generateDestIndex } from './common';
 
 export default ({ getService }: FtrProviderContext) => {
   const esArchiver = getService('esArchiver');
@@ -27,8 +27,102 @@ export default ({ getService }: FtrProviderContext) => {
       await transform.api.cleanTransformIndices();
     });
 
+    it('should create a transform', async () => {
+      const transformId = 'test_transform_id_create';
+
+      const { body, status } = await supertest
+        .put(`/internal/transform/transforms/${transformId}`)
+        .auth(
+          USER.TRANSFORM_POWERUSER,
+          transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER)
+        )
+        .set(getCommonRequestHeader('1'))
+        .send({
+          ...generateTransformConfig(transformId),
+        });
+
+      transform.api.assertResponseStatusCode(200, status, body);
+
+      expect(body).to.eql({
+        dataViewsCreated: [],
+        dataViewsErrors: [],
+        errors: [],
+        transformsCreated: [
+          {
+            transform: transformId,
+          },
+        ],
+      });
+    });
+
+    it('should create a transform with data view', async () => {
+      const transformId = 'test_transform_id_create_with_data_view';
+      const destinationIndex = generateDestIndex(transformId);
+
+      const { body, status } = await supertest
+        .put(`/internal/transform/transforms/${transformId}?createDataView=true`)
+        .auth(
+          USER.TRANSFORM_POWERUSER,
+          transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER)
+        )
+        .set(getCommonRequestHeader('1'))
+        .send({
+          ...generateTransformConfig(transformId),
+        });
+
+      transform.api.assertResponseStatusCode(200, status, body);
+
+      // The data view id will be returned as a non-deterministic uuid
+      // so we cannot assert the actual id returned. We'll just assert
+      // that a data view has been created a no errors were returned.
+      expect(body.dataViewsCreated.length).to.be(1);
+      expect(body.dataViewsErrors.length).to.be(0);
+      expect(body.errors.length).to.be(0);
+      expect(body.transformsCreated).to.eql([
+        {
+          transform: transformId,
+        },
+      ]);
+
+      await transform.testResources.deleteIndexPatternByTitle(destinationIndex);
+    });
+
+    it('should create a transform with data view and time field', async () => {
+      const transformId = 'test_transform_id_create_with_data_view_and_time_field';
+      const destinationIndex = generateDestIndex(transformId);
+
+      const { body, status } = await supertest
+        .put(
+          `/internal/transform/transforms/${transformId}?createDataView=true&timeFieldName=@timestamp`
+        )
+        .auth(
+          USER.TRANSFORM_POWERUSER,
+          transform.securityCommon.getPasswordForUser(USER.TRANSFORM_POWERUSER)
+        )
+        .set(getCommonRequestHeader('1'))
+        .send({
+          ...generateTransformConfig(transformId),
+        });
+
+      transform.api.assertResponseStatusCode(200, status, body);
+
+      // The data view id will be returned as a non-deterministic uuid
+      // so we cannot assert the actual id returned. We'll just assert
+      // that a data view has been created a no errors were returned.
+      expect(body.dataViewsCreated.length).to.be(1);
+      expect(body.dataViewsErrors.length).to.be(0);
+      expect(body.errors.length).to.be(0);
+      expect(body.transformsCreated).to.eql([
+        {
+          transform: transformId,
+        },
+      ]);
+
+      await transform.testResources.deleteIndexPatternByTitle(destinationIndex);
+    });
+
     it('should not allow pivot and latest configs in same transform', async () => {
-      const transformId = 'test_transform_id';
+      const transformId = 'test_transform_id_fail';
 
       const { body, status } = await supertest
         .put(`/internal/transform/transforms/${transformId}`)
@@ -50,7 +144,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     it('should ensure if pivot or latest is provided', async () => {
-      const transformId = 'test_transform_id';
+      const transformId = 'test_transform_id_fail';
 
       const { pivot, ...config } = generateTransformConfig(transformId);