[ml] migrate file_data_visualizer/import route to file_upload plugin (#89640) (#90075)

* migrate file_upload plugin to maps_file_upload

* update plugins list

* migrate ml import endpoint

* migrate ml telemetry to file_upload plugin

* add fileUpload plugin to ml

* add TS project

* update ML to use file_upload endpoint

* move types to file_upload plugin

* ignore error

* clean up

* i18n clean-up

* remove schemas from ml

* remove usageCollection from ml

* node scripts/build_plugin_list_docs

* update telemety collector

* revert changes to ingestPipeline schema

* change name of TELEMETRY_DOC_ID to unique value

* remove ImportFile from ml/server/routes/apidoc.json

* fix typo in x=pack/tsconfig.json

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
# Conflicts:
#	src/plugins/telemetry/schema/oss_plugins.json
#	x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

docs/developer/plugin-list.asciidoc

@@ -366,6 +366,10 @@ and actions.
 |The features plugin enhance Kibana with a per-feature privilege system.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload]
+|WARNING: Missing README.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/fleet/README.md[fleet]
 |Fleet needs to have Elasticsearch API keys enabled, and also to have TLS enabled on kibana, (if you want to run Kibana without TLS you can provide the following config flag --xpack.fleet.agents.tlsCheckDisabled=false)
 

x-pack/plugins/file_upload/common/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './constants';
+export * from './types';

x-pack/plugins/file_upload/common/types.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface ImportResponse {
+  success: boolean;
+  id: string;
+  index?: string;
+  pipelineId?: string;
+  docCount: number;
+  failures: ImportFailure[];
+  error?: any;
+  ingestError?: boolean;
+}
+
+export interface ImportFailure {
+  item: number;
+  reason: string;
+  doc: ImportDoc;
+}
+
+export interface Doc {
+  message: string;
+}
+
+export type ImportDoc = Doc | string;
+
+export interface Settings {
+  pipeline?: string;
+  index: string;
+  body: any[];
+  [key: string]: any;
+}
+
+export interface Mappings {
+  _meta?: {
+    created_by: string;
+  };
+  properties: {
+    [key: string]: any;
+  };
+}
+
+export interface IngestPipelineWrapper {
+  id: string;
+  pipeline: IngestPipeline;
+}
+
+export interface IngestPipeline {
+  description: string;
+  processors: any[];
+}

x-pack/plugins/file_upload/jest.config.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/plugins/file_upload'],
+};

x-pack/plugins/file_upload/kibana.json

@@ -0,0 +1,8 @@
+{
+  "id": "fileUpload",
+  "version": "8.0.0",
+  "kibanaVersion": "kibana",
+  "server": true,
+  "ui": false,
+  "requiredPlugins": ["usageCollection"]
+}

x-pack/plugins/file_upload/server/error_wrapper.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { boomify, isBoom } from '@hapi/boom';
+import { ResponseError, CustomHttpResponseOptions } from 'kibana/server';
+
+export function wrapError(error: any): CustomHttpResponseOptions<ResponseError> {
+  const boom = isBoom(error)
+    ? error
+    : boomify(error, { statusCode: error.status ?? error.statusCode });
+  const statusCode = boom.output.statusCode;
+  return {
+    body: {
+      message: boom,
+      ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}),
+    },
+    headers: boom.output.headers as { [key: string]: string },
+    statusCode,
+  };
+}

x-pack/plugins/file_upload/server/import_data.ts

@@ -5,19 +5,20 @@
  */
 
 import { IScopedClusterClient } from 'kibana/server';
-import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer';
+import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
 import {
   ImportResponse,
   ImportFailure,
   Settings,
   Mappings,
   IngestPipelineWrapper,
-} from '../../../common/types/file_datavisualizer';
-import { InputData } from './file_data_visualizer';
+} from '../common';
+
+export type InputData = any[];
 
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
-    id: string,
+    id: string | undefined,
     index: string,
     settings: Settings,
     mappings: Mappings,
@@ -77,7 +78,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
     } catch (error) {
       return {
         success: false,
-        id,
+        id: id!,
         index: createdIndex,
         pipelineId: createdPipelineId,
         error: error.body !== undefined ? error.body : error,

x-pack/plugins/file_upload/server/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FileUploadPlugin } from './plugin';
+
+export const plugin = () => new FileUploadPlugin();

x-pack/plugins/file_upload/server/plugin.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CoreSetup, CoreStart, Plugin } from 'src/core/server';
+import { fileUploadRoutes } from './routes';
+import { initFileUploadTelemetry } from './telemetry';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
+
+interface SetupDeps {
+  usageCollection: UsageCollectionSetup;
+}
+
+export class FileUploadPlugin implements Plugin {
+  async setup(coreSetup: CoreSetup, plugins: SetupDeps) {
+    fileUploadRoutes(coreSetup.http.createRouter());
+
+    initFileUploadTelemetry(coreSetup, plugins.usageCollection);
+  }
+
+  start(core: CoreStart) {}
+}

x-pack/plugins/file_upload/server/routes.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter, IScopedClusterClient } from 'kibana/server';
+import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common';
+import { wrapError } from './error_wrapper';
+import { InputData, importDataProvider } from './import_data';
+
+import { updateTelemetry } from './telemetry';
+import { importFileBodySchema, importFileQuerySchema } from './schemas';
+
+function importData(
+  client: IScopedClusterClient,
+  id: string | undefined,
+  index: string,
+  settings: Settings,
+  mappings: Mappings,
+  ingestPipeline: IngestPipelineWrapper,
+  data: InputData
+) {
+  const { importData: importDataFunc } = importDataProvider(client);
+  return importDataFunc(id, index, settings, mappings, ingestPipeline, data);
+}
+
+/**
+ * Routes for the file upload.
+ */
+export function fileUploadRoutes(router: IRouter) {
+  /**
+   * @apiGroup FileDataVisualizer
+   *
+   * @api {post} /api/file_upload/import Import file data
+   * @apiName ImportFile
+   * @apiDescription Imports file data into elasticsearch index.
+   *
+   * @apiSchema (query) importFileQuerySchema
+   * @apiSchema (body) importFileBodySchema
+   */
+  router.post(
+    {
+      path: '/api/file_upload/import',
+      validate: {
+        query: importFileQuerySchema,
+        body: importFileBodySchema,
+      },
+      options: {
+        body: {
+          accepts: ['application/json'],
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+        tags: ['access:ml:canFindFileStructure'],
+      },
+    },
+    async (context, request, response) => {
+      try {
+        const { id } = request.query;
+        const { index, data, settings, mappings, ingestPipeline } = request.body;
+
+        // `id` being `undefined` tells us that this is a new import due to create a new index.
+        // follow-up import calls to just add additional data will include the `id` of the created
+        // index, we'll ignore those and don't increment the counter.
+        if (id === undefined) {
+          await updateTelemetry();
+        }
+
+        const result = await importData(
+          context.core.elasticsearch.client,
+          id,
+          index,
+          settings,
+          mappings,
+          // @ts-expect-error
+          ingestPipeline,
+          data
+        );
+        return response.ok({ body: result });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    }
+  );
+}

x-pack/plugins/file_upload/server/schemas.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+export const importFileQuerySchema = schema.object({
+  id: schema.maybe(schema.string()),
+});
+
+export const importFileBodySchema = schema.object({
+  index: schema.string(),
+  data: schema.arrayOf(schema.any()),
+  settings: schema.maybe(schema.any()),
+  /** Mappings */
+  mappings: schema.any(),
+  /** Ingest pipeline definition */
+  ingestPipeline: schema.object({
+    id: schema.maybe(schema.string()),
+    pipeline: schema.maybe(schema.any()),
+  }),
+});

x-pack/plugins/file_upload/server/telemetry/index.ts

@@ -4,5 +4,5 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { initMlTelemetry } from './ml_usage_collector';
+export { initFileUploadTelemetry } from './usage_collector';
 export { updateTelemetry } from './telemetry';

x-pack/plugins/file_upload/server/telemetry/mappings.ts

@@ -7,13 +7,13 @@
 import { SavedObjectsType } from 'src/core/server';
 import { TELEMETRY_DOC_ID } from './telemetry';
 
-export const mlTelemetryMappingsType: SavedObjectsType = {
+export const telemetryMappingsType: SavedObjectsType = {
   name: TELEMETRY_DOC_ID,
   hidden: false,
   namespaceType: 'agnostic',
   mappings: {
     properties: {
-      file_data_visualizer: {
+      file_upload: {
         properties: {
           index_creation_count: {
             type: 'long',

x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts

@@ -34,7 +34,7 @@ describe('ml plugin telemetry', () => {
     it('should update existing telemetry', async () => {
       const internalRepo = mockInit({
         attributes: {
-          file_data_visualizer: {
+          file_upload: {
             index_creation_count: 2,
           },
         },

x-pack/plugins/file_upload/server/telemetry/telemetry.ts

@@ -9,10 +9,10 @@ import { ISavedObjectsRepository } from 'kibana/server';
 
 import { getInternalRepository } from './internal_repository';
 
-export const TELEMETRY_DOC_ID = 'ml-telemetry';
+export const TELEMETRY_DOC_ID = 'file-upload-usage-collection-telemetry';
 
 export interface Telemetry {
-  file_data_visualizer: {
+  file_upload: {
     index_creation_count: number;
   };
 }
@@ -23,7 +23,7 @@ export interface TelemetrySavedObject {
 
 export function initTelemetry(): Telemetry {
   return {
-    file_data_visualizer: {
+    file_upload: {
       index_creation_count: 0,
     },
   };
@@ -74,8 +74,8 @@ export async function updateTelemetry(internalRepo?: ISavedObjectsRepository) {
 
 function incrementCounts(telemetry: Telemetry) {
   return {
-    file_data_visualizer: {
-      index_creation_count: telemetry.file_data_visualizer.index_creation_count + 1,
+    file_upload: {
+      index_creation_count: telemetry.file_upload.index_creation_count + 1,
     },
   };
 }

x-pack/plugins/file_upload/server/telemetry/usage_collector.ts

@@ -8,23 +8,26 @@ import { CoreSetup } from 'kibana/server';
 
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { getTelemetry, initTelemetry, Telemetry } from './telemetry';
-import { mlTelemetryMappingsType } from './mappings';
+import { telemetryMappingsType } from './mappings';
 import { setInternalRepository } from './internal_repository';
 
-export function initMlTelemetry(coreSetup: CoreSetup, usageCollection: UsageCollectionSetup) {
-  coreSetup.savedObjects.registerType(mlTelemetryMappingsType);
-  registerMlUsageCollector(usageCollection);
+export function initFileUploadTelemetry(
+  coreSetup: CoreSetup,
+  usageCollection: UsageCollectionSetup
+) {
+  coreSetup.savedObjects.registerType(telemetryMappingsType);
+  registerUsageCollector(usageCollection);
   coreSetup.getStartServices().then(([core]) => {
     setInternalRepository(core.savedObjects.createInternalRepository);
   });
 }
 
-function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void {
-  const mlUsageCollector = usageCollection.makeUsageCollector<Telemetry>({
-    type: 'mlTelemetry',
+function registerUsageCollector(usageCollectionSetup: UsageCollectionSetup): void {
+  const usageCollector = usageCollectionSetup.makeUsageCollector<Telemetry>({
+    type: 'fileUpload',
     isReady: () => true,
     schema: {
-      file_data_visualizer: {
+      file_upload: {
         index_creation_count: { type: 'long' },
       },
     },
@@ -38,5 +41,5 @@ function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void {
     },
   });
 
-  usageCollection.registerCollector(mlUsageCollector);
+  usageCollectionSetup.registerCollector(usageCollector);
 }

x-pack/plugins/file_upload/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "outDir": "./target/types",
+    "emitDeclarationOnly": true,
+    "declaration": true,
+    "declarationMap": true
+  },
+  "include": ["common/**/*", "public/**/*", "server/**/*"],
+  "references": [
+    { "path": "../../../src/core/tsconfig.json" },
+    { "path": "../../../src/plugins/usage_collection/tsconfig.json" }
+  ]
+}

x-pack/plugins/maps_file_upload/public/util/indexing_service.js

@@ -119,7 +119,7 @@ async function writeToIndex(indexingDetails) {
   const { appName, index, data, settings, mappings, ingestPipeline } = indexingDetails;
 
   return await httpService({
-    url: `/api/fileupload/import`,
+    url: `/api/maps/fileupload/import`,
     method: 'POST',
     ...(query ? { query } : {}),
     data: {

x-pack/plugins/maps_file_upload/server/routes/file_upload.js

@@ -9,7 +9,7 @@ import { updateTelemetry } from '../telemetry/telemetry';
 import { MAX_BYTES } from '../../common/constants/file_import';
 import { schema } from '@kbn/config-schema';
 
-export const IMPORT_ROUTE = '/api/fileupload/import';
+export const IMPORT_ROUTE = '/api/maps/fileupload/import';
 
 export const querySchema = schema.maybe(
   schema.object({

x-pack/plugins/ml/common/types/file_datavisualizer.ts

@@ -68,52 +68,3 @@ export interface FindFileStructureResponse {
   timestamp_field?: string;
   should_trim_fields?: boolean;
 }
-
-export interface ImportResponse {
-  success: boolean;
-  id: string;
-  index?: string;
-  pipelineId?: string;
-  docCount: number;
-  failures: ImportFailure[];
-  error?: any;
-  ingestError?: boolean;
-}
-
-export interface ImportFailure {
-  item: number;
-  reason: string;
-  doc: ImportDoc;
-}
-
-export interface Doc {
-  message: string;
-}
-
-export type ImportDoc = Doc | string;
-
-export interface Settings {
-  pipeline?: string;
-  index: string;
-  body: any[];
-  [key: string]: any;
-}
-
-export interface Mappings {
-  _meta?: {
-    created_by: string;
-  };
-  properties: {
-    [key: string]: any;
-  };
-}
-
-export interface IngestPipelineWrapper {
-  id: string;
-  pipeline: IngestPipeline;
-}
-
-export interface IngestPipeline {
-  description: string;
-  processors: any[];
-}

x-pack/plugins/ml/kibana.json

@@ -10,8 +10,8 @@
     "data",
     "cloud",
     "features",
+    "fileUpload",
     "licensing",
-    "usageCollection",
     "share",
     "embeddable",
     "uiActions",

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts

@@ -8,11 +8,8 @@ import { i18n } from '@kbn/i18n';
 import { cloneDeep } from 'lodash';
 import uuid from 'uuid/v4';
 import { CombinedField } from './types';
-import {
-  FindFileStructureResponse,
-  IngestPipeline,
-  Mappings,
-} from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common';
 
 const COMMON_LAT_NAMES = ['latitude', 'lat'];
 const COMMON_LON_NAMES = ['longitude', 'long', 'lon'];

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx

@@ -11,7 +11,7 @@ import { EuiCallOut, EuiSpacer, EuiButtonEmpty, EuiHorizontalRule } from '@elast
 
 import numeral from '@elastic/numeral';
 import { ErrorResponse } from '../../../../../../common/types/errors';
-import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../common/constants/file_datavisualizer';
+import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../../file_upload/common';
 
 interface FileTooLargeProps {
   fileSize: number;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts

@@ -15,7 +15,7 @@ import {
   Mappings,
   Settings,
   IngestPipeline,
-} from '../../../../../../../common/types/file_datavisualizer';
+} from '../../../../../../../../file_upload/common';
 
 const CHUNK_SIZE = 5000;
 const MAX_CHUNK_CHAR_COUNT = 1000000;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts

@@ -5,10 +5,8 @@
  */
 
 import { Importer, ImportConfig, CreateDocsResponse } from './importer';
-import {
-  Doc,
-  FindFileStructureResponse,
-} from '../../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer';
+import { Doc } from '../../../../../../../../file_upload/common';
 
 export class MessageImporter extends Importer {
   private _excludeLinesRegex: RegExp | null;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts

@@ -14,7 +14,7 @@ import {
   MAX_FILE_SIZE_BYTES,
   ABSOLUTE_MAX_FILE_SIZE_BYTES,
   FILE_SIZE_DISPLAY_FORMAT,
-} from '../../../../../../common/constants/file_datavisualizer';
+} from '../../../../../../../file_upload/common';
 import { getUiSettings } from '../../../../util/dependency_cache';
 import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../../../../../common/constants/settings';
 

x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts

@@ -7,7 +7,7 @@
 import { http } from '../http_service';
 
 import { basePath } from './index';
-import { ImportResponse } from '../../../../common/types/file_datavisualizer';
+import { ImportResponse } from '../../../../../file_upload/common';
 
 export const fileDatavisualizer = {
   analyzeFile(file: string, params: Record<string, string> = {}) {
@@ -45,7 +45,7 @@ export const fileDatavisualizer = {
     });
 
     return http<ImportResponse>({
-      path: `${basePath()}/file_data_visualizer/import`,
+      path: `/api/file_upload/import`,
       method: 'POST',
       query,
       body,

x-pack/plugins/ml/public/plugin.ts

@@ -21,7 +21,6 @@ import type {
   SharePluginStart,
   UrlGeneratorContract,
 } from 'src/plugins/share/public';
-import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import type { DataPublicPluginStart } from 'src/plugins/data/public';
 import type { HomePublicPluginSetup } from 'src/plugins/home/public';
 import type { IndexPatternManagementSetup } from 'src/plugins/index_pattern_management/public';
@@ -60,7 +59,6 @@ export interface MlSetupDependencies {
   security?: SecurityPluginSetup;
   licensing: LicensingPluginSetup;
   management?: ManagementSetup;
-  usageCollection: UsageCollectionSetup;
   licenseManagement?: LicenseManagementUIPluginSetup;
   home?: HomePublicPluginSetup;
   embeddable: EmbeddableSetup;
@@ -102,7 +100,6 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
             security: pluginsSetup.security,
             licensing: pluginsSetup.licensing,
             management: pluginsSetup.management,
-            usageCollection: pluginsSetup.usageCollection,
             licenseManagement: pluginsSetup.licenseManagement,
             home: pluginsSetup.home,
             embeddable: { ...pluginsSetup.embeddable, ...pluginsStart.embeddable },

x-pack/plugins/ml/server/lib/register_settings.ts

@@ -14,7 +14,7 @@ import {
   DEFAULT_AD_RESULTS_TIME_FILTER,
   DEFAULT_ENABLE_AD_RESULTS_TIME_FILTER,
 } from '../../common/constants/settings';
-import { MAX_FILE_SIZE } from '../../common/constants/file_datavisualizer';
+import { MAX_FILE_SIZE } from '../../../file_upload/common';
 
 export function registerKibanaSettings(coreSetup: CoreSetup) {
   coreSetup.uiSettings.register({

x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts

@@ -18,7 +18,7 @@ import {
   DataFrameAnalyticsStats,
   MapElements,
 } from '../../../common/types/data_frame_analytics';
-import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer';
+import { INDEX_META_DATA_CREATED_BY } from '../../../../file_upload/common';
 import { getAnalysisType } from '../../../common/util/analytics_utils';
 import {
   ExtendAnalyticsMapArgs,

x-pack/plugins/ml/server/models/file_data_visualizer/index.ts

@@ -5,5 +5,3 @@
  */
 
 export { fileDataVisualizerProvider, InputData } from './file_data_visualizer';
-
-export { importDataProvider } from './import_data';

x-pack/plugins/ml/server/plugin.ts

@@ -23,7 +23,6 @@ import { SpacesPluginSetup } from '../../spaces/server';
 import { PLUGIN_ID } from '../common/constants/app';
 import { MlCapabilities } from '../common/types/capabilities';
 
-import { initMlTelemetry } from './lib/telemetry';
 import { initMlServerLog } from './lib/log';
 import { initSampleDataSets } from './lib/sample_data_sets';
 
@@ -190,7 +189,6 @@ export class MlServerPlugin
     trainedModelsRoutes(routeInit);
 
     initMlServerLog({ log: this.log });
-    initMlTelemetry(coreSetup, plugins.usageCollection);
 
     return {
       ...createSharedServices(

x-pack/plugins/ml/server/routes/apidoc.json

@@ -43,7 +43,6 @@
 
     "FileDataVisualizer",
     "AnalyzeFile",
-    "ImportFile",
 
     "ResultsService",
     "GetAnomaliesTableData",

x-pack/plugins/ml/server/routes/file_data_visualizer.ts

@@ -5,28 +5,13 @@
  */
 
 import { schema } from '@kbn/config-schema';
-import { IScopedClusterClient } from 'kibana/server';
-import { MAX_FILE_SIZE_BYTES } from '../../common/constants/file_datavisualizer';
-import {
-  InputOverrides,
-  Settings,
-  IngestPipelineWrapper,
-  Mappings,
-} from '../../common/types/file_datavisualizer';
+import { MAX_FILE_SIZE_BYTES } from '../../../file_upload/common';
+import { InputOverrides } from '../../common/types/file_datavisualizer';
 import { wrapError } from '../client/error_wrapper';
-import {
-  InputData,
-  fileDataVisualizerProvider,
-  importDataProvider,
-} from '../models/file_data_visualizer';
+import { InputData, fileDataVisualizerProvider } from '../models/file_data_visualizer';
 
 import { RouteInitialization } from '../types';
-import { updateTelemetry } from '../lib/telemetry';
-import {
-  analyzeFileQuerySchema,
-  importFileBodySchema,
-  importFileQuerySchema,
-} from './schemas/file_data_visualizer_schema';
+import { analyzeFileQuerySchema } from './schemas/file_data_visualizer_schema';
 import type { MlClient } from '../lib/ml_client';
 
 function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverrides) {
@@ -34,19 +19,6 @@ function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverr
   return analyzeFile(data, overrides);
 }
 
-function importData(
-  client: IScopedClusterClient,
-  id: string,
-  index: string,
-  settings: Settings,
-  mappings: Mappings,
-  ingestPipeline: IngestPipelineWrapper,
-  data: InputData
-) {
-  const { importData: importDataFunc } = importDataProvider(client);
-  return importDataFunc(id, index, settings, mappings, ingestPipeline, data);
-}
-
 /**
  * Routes for the file data visualizer.
  */
@@ -84,57 +56,4 @@ export function fileDataVisualizerRoutes({ router, routeGuard }: RouteInitializa
       }
     })
   );
-
-  /**
-   * @apiGroup FileDataVisualizer
-   *
-   * @api {post} /api/ml/file_data_visualizer/import Import file data
-   * @apiName ImportFile
-   * @apiDescription Imports file data into elasticsearch index.
-   *
-   * @apiSchema (query) importFileQuerySchema
-   * @apiSchema (body) importFileBodySchema
-   */
-  router.post(
-    {
-      path: '/api/ml/file_data_visualizer/import',
-      validate: {
-        query: importFileQuerySchema,
-        body: importFileBodySchema,
-      },
-      options: {
-        body: {
-          accepts: ['application/json'],
-          maxBytes: MAX_FILE_SIZE_BYTES,
-        },
-        tags: ['access:ml:canFindFileStructure'],
-      },
-    },
-    routeGuard.basicLicenseAPIGuard(async ({ client, request, response }) => {
-      try {
-        const { id } = request.query;
-        const { index, data, settings, mappings, ingestPipeline } = request.body;
-
-        // `id` being `undefined` tells us that this is a new import due to create a new index.
-        // follow-up import calls to just add additional data will include the `id` of the created
-        // index, we'll ignore those and don't increment the counter.
-        if (id === undefined) {
-          await updateTelemetry();
-        }
-
-        const result = await importData(
-          client,
-          id,
-          index,
-          settings,
-          mappings,
-          ingestPipeline,
-          data
-        );
-        return response.ok({ body: result });
-      } catch (e) {
-        return response.customError(wrapError(e));
-      }
-    })
-  );
 }

x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts

@@ -24,20 +24,3 @@ export const analyzeFileQuerySchema = schema.maybe(
     timestamp_format: schema.maybe(schema.string()),
   })
 );
-
-export const importFileQuerySchema = schema.object({
-  id: schema.maybe(schema.string()),
-});
-
-export const importFileBodySchema = schema.object({
-  index: schema.maybe(schema.string()),
-  data: schema.arrayOf(schema.any()),
-  settings: schema.maybe(schema.any()),
-  /** Mappings */
-  mappings: schema.any(),
-  /** Ingest pipeline definition */
-  ingestPipeline: schema.object({
-    id: schema.maybe(schema.string()),
-    pipeline: schema.maybe(schema.any()),
-  }),
-});

x-pack/plugins/ml/server/types.ts

@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import type { HomeServerPluginSetup } from 'src/plugins/home/server';
 import type { IRouter } from 'kibana/server';
 import type { CloudSetup } from '../../cloud/server';
@@ -43,7 +42,6 @@ export interface PluginsSetup {
   licensing: LicensingPluginSetup;
   security?: SecurityPluginSetup;
   spaces?: SpacesPluginSetup;
-  usageCollection: UsageCollectionSetup;
 }
 
 export interface PluginsStart {

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -1771,6 +1771,17 @@
         }
       }
     },
+    "fileUpload": {
+      "properties": {
+        "file_upload": {
+          "properties": {
+            "index_creation_count": {
+              "type": "long"
+            }
+          }
+        }
+      }
+    },
     "fleet": {
       "properties": {
         "agents_enabled": {
@@ -2308,17 +2319,6 @@
         }
       }
     },
-    "mlTelemetry": {
-      "properties": {
-        "file_data_visualizer": {
-          "properties": {
-            "index_creation_count": {
-              "type": "long"
-            }
-          }
-        }
-      }
-    },
     "monitoring": {
       "properties": {
         "hasMonitoringData": {

x-pack/tsconfig.json

@@ -17,6 +17,7 @@
     "plugins/global_search_providers/**/*",
     "plugins/graph/**/*",
     "plugins/features/**/*",
+    "plugins/file_upload/**/*",
     "plugins/embeddable_enhanced/**/*",
     "plugins/event_log/**/*",
     "plugins/enterprise_search/**/*",
@@ -103,6 +104,7 @@
     { "path": "./plugins/enterprise_search/tsconfig.json" },
     { "path": "./plugins/event_log/tsconfig.json" },
     { "path": "./plugins/features/tsconfig.json" },
+    { "path": "./plugins/file_upload/tsconfig.json" },
     { "path": "./plugins/global_search_bar/tsconfig.json" },
     { "path": "./plugins/global_search_providers/tsconfig.json" },
     { "path": "./plugins/global_search/tsconfig.json" },

x-pack/tsconfig.refs.json

@@ -16,6 +16,7 @@
     { "path": "./plugins/enterprise_search/tsconfig.json" },
     { "path": "./plugins/event_log/tsconfig.json" },
     { "path": "./plugins/features/tsconfig.json" },
+    { "path": "./plugins/file_upload/tsconfig.json" },
     { "path": "./plugins/global_search_bar/tsconfig.json" },
     { "path": "./plugins/global_search_providers/tsconfig.json" },
     { "path": "./plugins/global_search/tsconfig.json" },