[ml] migrate file_data_visualizer/analyze_file to file_upload plugin (#94259)

* [ml] migrate file_data_visualizer/analyze_file to file_upload plugin

* tslint

* give analyze route access to ml user

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/file_upload/common/types.ts

@@ -5,6 +5,73 @@
  * 2.0.
  */
 
+import { ES_FIELD_TYPES } from '../../../../src/plugins/data/common';
+
+export interface InputOverrides {
+  [key: string]: string | undefined;
+}
+
+export type FormattedOverrides = InputOverrides & {
+  column_names: string[];
+  has_header_row: boolean;
+  should_trim_fields: boolean;
+};
+
+export interface AnalysisResult {
+  results: FindFileStructureResponse;
+  overrides?: FormattedOverrides;
+}
+
+export interface FindFileStructureResponse {
+  charset: string;
+  has_header_row: boolean;
+  has_byte_order_marker: boolean;
+  format: string;
+  field_stats: {
+    [fieldName: string]: {
+      count: number;
+      cardinality: number;
+      top_hits: Array<{ count: number; value: any }>;
+      mean_value?: number;
+      median_value?: number;
+      max_value?: number;
+      min_value?: number;
+      earliest?: string;
+      latest?: string;
+    };
+  };
+  sample_start: string;
+  num_messages_analyzed: number;
+  mappings: {
+    properties: {
+      [fieldName: string]: {
+        // including all possible Elasticsearch types
+        // since find_file_structure API can be enhanced to include new fields in the future
+        type: Exclude<
+          ES_FIELD_TYPES,
+          ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE
+        >;
+        format?: string;
+      };
+    };
+  };
+  quote: string;
+  delimiter: string;
+  need_client_timezone: boolean;
+  num_lines_analyzed: number;
+  column_names?: string[];
+  explanation?: string[];
+  grok_pattern?: string;
+  multiline_start_pattern?: string;
+  exclude_lines_pattern?: string;
+  java_timestamp_formats?: string[];
+  joda_timestamp_formats?: string[];
+  timestamp_field?: string;
+  should_trim_fields?: boolean;
+}
+
+export type InputData = any[];
+
 export interface ImportResponse {
   success: boolean;
   id: string;

x-pack/plugins/file_upload/server/analyze_file.tsx

@@ -9,32 +9,29 @@ import { IScopedClusterClient } from 'kibana/server';
 import {
   AnalysisResult,
   FormattedOverrides,
+  InputData,
   InputOverrides,
   FindFileStructureResponse,
-} from '../../../common/types/file_datavisualizer';
+} from '../common';
 
-export type InputData = any[];
+export async function analyzeFile(
+  client: IScopedClusterClient,
+  data: InputData,
+  overrides: InputOverrides
+): Promise<AnalysisResult> {
+  overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
+  const {
+    body,
+  } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
+    body: data,
+    ...overrides,
+  });
 
-export function fileDataVisualizerProvider(client: IScopedClusterClient) {
-  async function analyzeFile(data: InputData, overrides: InputOverrides): Promise<AnalysisResult> {
-    overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
-    const {
-      body,
-    } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
-      body: data,
-      ...overrides,
-    });
-
-    const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
-
-    return {
-      ...(hasOverrides && { overrides: reducedOverrides }),
-      results: body,
-    };
-  }
+  const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
 
   return {
-    analyzeFile,
+    ...(hasOverrides && { overrides: reducedOverrides }),
+    results: body,
   };
 }
 
@@ -42,8 +39,8 @@ function formatOverrides(overrides: InputOverrides) {
   let hasOverrides = false;
 
   const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => {
-    const overrideValue: string = overrides[overrideKey];
-    if (overrideValue !== '') {
+    const overrideValue: string | undefined = overrides[overrideKey];
+    if (overrideValue !== undefined && overrideValue !== '') {
       if (overrideKey === 'column_names') {
         acc.column_names = overrideValue.split(',');
       } else if (overrideKey === 'has_header_row') {

x-pack/plugins/file_upload/server/import_data.ts

@@ -10,13 +10,12 @@ import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
 import {
   ImportResponse,
   ImportFailure,
+  InputData,
   Settings,
   Mappings,
   IngestPipelineWrapper,
 } from '../common';
 
-export type InputData = any[];
-
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
     id: string | undefined,

x-pack/plugins/file_upload/server/routes.ts

@@ -5,13 +5,21 @@
  * 2.0.
  */
 
+import { schema } from '@kbn/config-schema';
 import { IRouter, IScopedClusterClient } from 'kibana/server';
-import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common';
+import {
+  MAX_FILE_SIZE_BYTES,
+  IngestPipelineWrapper,
+  InputData,
+  Mappings,
+  Settings,
+} from '../common';
 import { wrapError } from './error_wrapper';
-import { InputData, importDataProvider } from './import_data';
+import { analyzeFile } from './analyze_file';
+import { importDataProvider } from './import_data';
 
 import { updateTelemetry } from './telemetry';
-import { importFileBodySchema, importFileQuerySchema } from './schemas';
+import { analyzeFileQuerySchema, importFileBodySchema, importFileQuerySchema } from './schemas';
 
 function importData(
   client: IScopedClusterClient,
@@ -30,6 +38,44 @@ function importData(
  * Routes for the file upload.
  */
 export function fileUploadRoutes(router: IRouter) {
+  /**
+   * @apiGroup FileDataVisualizer
+   *
+   * @api {post} /api/file_upload/analyze_file Analyze file data
+   * @apiName AnalyzeFile
+   * @apiDescription Performs analysis of the file data.
+   *
+   * @apiSchema (query) analyzeFileQuerySchema
+   */
+  router.post(
+    {
+      path: '/api/file_upload/analyze_file',
+      validate: {
+        body: schema.any(),
+        query: analyzeFileQuerySchema,
+      },
+      options: {
+        body: {
+          accepts: ['text/*', 'application/json'],
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+        tags: ['access:fileUpload:analyzeFile'],
+      },
+    },
+    async (context, request, response) => {
+      try {
+        const result = await analyzeFile(
+          context.core.elasticsearch.client,
+          request.body,
+          request.query
+        );
+        return response.ok({ body: result });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    }
+  );
+
   /**
    * @apiGroup FileDataVisualizer
    *

x-pack/plugins/file_upload/server/schemas.ts

@@ -7,6 +7,23 @@
 
 import { schema } from '@kbn/config-schema';
 
+export const analyzeFileQuerySchema = schema.object({
+  charset: schema.maybe(schema.string()),
+  column_names: schema.maybe(schema.string()),
+  delimiter: schema.maybe(schema.string()),
+  explain: schema.maybe(schema.string()),
+  format: schema.maybe(schema.string()),
+  grok_pattern: schema.maybe(schema.string()),
+  has_header_row: schema.maybe(schema.string()),
+  line_merge_size_limit: schema.maybe(schema.string()),
+  lines_to_sample: schema.maybe(schema.string()),
+  quote: schema.maybe(schema.string()),
+  should_trim_fields: schema.maybe(schema.string()),
+  timeout: schema.maybe(schema.string()),
+  timestamp_field: schema.maybe(schema.string()),
+  timestamp_format: schema.maybe(schema.string()),
+});
+
 export const importFileQuerySchema = schema.object({
   id: schema.maybe(schema.string()),
 });

x-pack/plugins/ml/common/types/capabilities.ts

@@ -102,7 +102,11 @@ export function getPluginPrivileges() {
   return {
     admin: {
       ...privilege,
-      api: ['fileUpload:import', ...allMlCapabilitiesKeys.map((k) => `ml:${k}`)],
+      api: [
+        'fileUpload:import',
+        'fileUpload:analyzeFile',
+        ...allMlCapabilitiesKeys.map((k) => `ml:${k}`),
+      ],
       catalogue: [PLUGIN_ID, `${PLUGIN_ID}_file_data_visualizer`],
       ui: allMlCapabilitiesKeys,
       savedObject: {
@@ -116,7 +120,7 @@ export function getPluginPrivileges() {
     },
     user: {
       ...privilege,
-      api: userMlCapabilitiesKeys.map((k) => `ml:${k}`),
+      api: ['fileUpload:analyzeFile', ...userMlCapabilitiesKeys.map((k) => `ml:${k}`)],
       catalogue: [PLUGIN_ID],
       management: { insightsAndAlerting: [] },
       ui: userMlCapabilitiesKeys,

x-pack/plugins/ml/common/types/file_datavisualizer.ts

@@ -1,71 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common';
-
-export interface InputOverrides {
-  [key: string]: string;
-}
-
-export type FormattedOverrides = InputOverrides & {
-  column_names: string[];
-  has_header_row: boolean;
-  should_trim_fields: boolean;
-};
-
-export interface AnalysisResult {
-  results: FindFileStructureResponse;
-  overrides?: FormattedOverrides;
-}
-
-export interface FindFileStructureResponse {
-  charset: string;
-  has_header_row: boolean;
-  has_byte_order_marker: boolean;
-  format: string;
-  field_stats: {
-    [fieldName: string]: {
-      count: number;
-      cardinality: number;
-      top_hits: Array<{ count: number; value: any }>;
-      mean_value?: number;
-      median_value?: number;
-      max_value?: number;
-      min_value?: number;
-      earliest?: string;
-      latest?: string;
-    };
-  };
-  sample_start: string;
-  num_messages_analyzed: number;
-  mappings: {
-    properties: {
-      [fieldName: string]: {
-        // including all possible Elasticsearch types
-        // since find_file_structure API can be enhanced to include new fields in the future
-        type: Exclude<
-          ES_FIELD_TYPES,
-          ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE
-        >;
-        format?: string;
-      };
-    };
-  };
-  quote: string;
-  delimiter: string;
-  need_client_timezone: boolean;
-  num_lines_analyzed: number;
-  column_names?: string[];
-  explanation?: string[];
-  grok_pattern?: string;
-  multiline_start_pattern?: string;
-  exclude_lines_pattern?: string;
-  java_timestamp_formats?: string[];
-  joda_timestamp_formats?: string[];
-  timestamp_field?: string;
-  should_trim_fields?: boolean;
-}

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx

@@ -9,7 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import React, { FC } from 'react';
 
 import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => {
   const items = createDisplayItems(results);

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx

@@ -29,7 +29,7 @@ import {
   removeCombinedFieldsFromMappings,
   removeCombinedFieldsFromPipeline,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   mappingsString: string;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx

@@ -29,7 +29,7 @@ import {
   getFieldNames,
   getNameCollisionMsg,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   addCombinedField: (combinedField: CombinedField) => void;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts

@@ -9,8 +9,11 @@ import { i18n } from '@kbn/i18n';
 import { cloneDeep } from 'lodash';
 import uuid from 'uuid/v4';
 import { CombinedField } from './types';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
-import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common';
+import {
+  FindFileStructureResponse,
+  IngestPipeline,
+  Mappings,
+} from '../../../../../../../file_upload/common';
 
 const COMMON_LAT_NAMES = ['latitude', 'lat'];
 const COMMON_LON_NAMES = ['longitude', 'long', 'lon'];

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx

@@ -20,7 +20,7 @@ import {
   EuiText,
   EuiSubSteps,
 } from '@elastic/eui';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   results: FindFileStructureResponse;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 import { getFieldNames, getSupportedFieldType } from './get_field_names';
 import { FileBasedFieldVisConfig } from '../../../stats_table/types';
 import { ML_JOB_FIELD_TYPES } from '../../../../../../common/constants/field_types';

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/fields_stats_grid.tsx

@@ -7,7 +7,7 @@
 
 import React, { useMemo, FC } from 'react';
 import { EuiFlexGroup, EuiSpacer } from '@elastic/eui';
-import type { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import type { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../../../stats_table';
 import type { FileBasedFieldVisConfig } from '../../../stats_table/types/field_vis_config';
 import { FileBasedDataVisualizerExpandedRow } from '../expanded_row';

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/get_field_names.ts

@@ -6,7 +6,7 @@
  */
 
 import { difference } from 'lodash';
-import type { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import type { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 import { MlJobFieldType } from '../../../../../../common/types/field_types';
 import { ML_JOB_FIELD_TYPES } from '../../../../../../common/constants/field_types';
 import { ES_FIELD_TYPES } from '../../../../../../../../../src/plugins/data/common';

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts

@@ -6,7 +6,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 export function createFilebeatConfig(
   index: string,

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx

@@ -23,7 +23,7 @@ import {
 } from '@elastic/eui';
 import { createFilebeatConfig } from './filebeat_config';
 import { useMlKibana } from '../../../../contexts/kibana';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 export enum EDITOR_MODE {
   HIDDEN,

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx

@@ -20,7 +20,7 @@ import {
 
 import { CombinedField, CombinedFieldsForm } from '../combined_fields';
 import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 const EDITOR_HEIGHT = '300px';
 
 interface Props {

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx

@@ -13,7 +13,7 @@ import { EuiTabbedContent, EuiSpacer } from '@elastic/eui';
 import { SimpleSettings } from './simple';
 import { AdvancedSettings } from './advanced';
 import { CombinedField } from '../combined_fields';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   index: string;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx

@@ -20,7 +20,7 @@ import {
   DISCOVER_APP_URL_GENERATOR,
   DiscoverUrlGeneratorState,
 } from '../../../../../../../../../src/plugins/discover/public';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 const RECHECK_DELAY_MS = 3000;
 

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx

@@ -20,7 +20,7 @@ import {
   EuiFlexGroup,
   EuiFlexItem,
 } from '@elastic/eui';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 import { FileContents } from '../file_contents';
 import { AnalysisSummary } from '../analysis_summary';

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts

@@ -7,7 +7,7 @@
 
 import { isEqual } from 'lodash';
 import { ml } from '../../../../services/ml_api_service';
-import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer';
+import { AnalysisResult, InputOverrides } from '../../../../../../../file_upload/common';
 import { MB } from '../../../../../../../file_upload/public';
 
 export const DEFAULT_LINES_TO_SAMPLE = 1000;

x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts

@@ -7,13 +7,11 @@
 
 import { http } from '../http_service';
 
-import { basePath } from './index';
-
 export const fileDatavisualizer = {
   analyzeFile(file: string, params: Record<string, string> = {}) {
     const body = JSON.stringify(file);
     return http<any>({
-      path: `${basePath()}/file_data_visualizer/analyze_file`,
+      path: '/api/file_upload/analyze_file',
       method: 'POST',
       body,
       query: params,

x-pack/plugins/ml/server/models/file_data_visualizer/index.ts

@@ -1,8 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-export { fileDataVisualizerProvider, InputData } from './file_data_visualizer';

x-pack/plugins/ml/server/plugin.ts

@@ -34,7 +34,6 @@ import { dataFrameAnalyticsRoutes } from './routes/data_frame_analytics';
 import { dataRecognizer } from './routes/modules';
 import { dataVisualizerRoutes } from './routes/data_visualizer';
 import { fieldsService } from './routes/fields_service';
-import { fileDataVisualizerRoutes } from './routes/file_data_visualizer';
 import { filtersRoutes } from './routes/filters';
 import { indicesRoutes } from './routes/indices';
 import { jobAuditMessagesRoutes } from './routes/job_audit_messages';
@@ -172,7 +171,6 @@ export class MlServerPlugin
     dataRecognizer(routeInit);
     dataVisualizerRoutes(routeInit);
     fieldsService(routeInit);
-    fileDataVisualizerRoutes(routeInit);
     filtersRoutes(routeInit);
     indicesRoutes(routeInit);
     jobAuditMessagesRoutes(routeInit);

x-pack/plugins/ml/server/routes/file_data_visualizer.ts

@@ -1,60 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { IScopedClusterClient } from 'kibana/server';
-import { schema } from '@kbn/config-schema';
-import { MAX_FILE_SIZE_BYTES } from '../../../file_upload/common';
-import { InputOverrides } from '../../common/types/file_datavisualizer';
-import { wrapError } from '../client/error_wrapper';
-import { InputData, fileDataVisualizerProvider } from '../models/file_data_visualizer';
-
-import { RouteInitialization } from '../types';
-import { analyzeFileQuerySchema } from './schemas/file_data_visualizer_schema';
-
-function analyzeFiles(client: IScopedClusterClient, data: InputData, overrides: InputOverrides) {
-  const { analyzeFile } = fileDataVisualizerProvider(client);
-  return analyzeFile(data, overrides);
-}
-
-/**
- * Routes for the file data visualizer.
- */
-export function fileDataVisualizerRoutes({ router, routeGuard }: RouteInitialization) {
-  /**
-   * @apiGroup FileDataVisualizer
-   *
-   * @api {post} /api/ml/file_data_visualizer/analyze_file Analyze file data
-   * @apiName AnalyzeFile
-   * @apiDescription Performs analysis of the file data.
-   *
-   * @apiSchema (query) analyzeFileQuerySchema
-   */
-  router.post(
-    {
-      path: '/api/ml/file_data_visualizer/analyze_file',
-      validate: {
-        body: schema.any(),
-        query: analyzeFileQuerySchema,
-      },
-      options: {
-        body: {
-          accepts: ['text/*', 'application/json'],
-          maxBytes: MAX_FILE_SIZE_BYTES,
-        },
-        tags: ['access:ml:canFindFileStructure'],
-      },
-    },
-    routeGuard.basicLicenseAPIGuard(async ({ client, request, response }) => {
-      try {
-        const result = await analyzeFiles(client, request.body, request.query);
-        return response.ok({ body: result });
-      } catch (e) {
-        return response.customError(wrapError(e));
-      }
-    })
-  );
-}

x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts

@@ -1,27 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { schema } from '@kbn/config-schema';
-
-export const analyzeFileQuerySchema = schema.maybe(
-  schema.object({
-    charset: schema.maybe(schema.string()),
-    column_names: schema.maybe(schema.string()),
-    delimiter: schema.maybe(schema.string()),
-    explain: schema.maybe(schema.string()),
-    format: schema.maybe(schema.string()),
-    grok_pattern: schema.maybe(schema.string()),
-    has_header_row: schema.maybe(schema.string()),
-    line_merge_size_limit: schema.maybe(schema.string()),
-    lines_to_sample: schema.maybe(schema.string()),
-    quote: schema.maybe(schema.string()),
-    should_trim_fields: schema.maybe(schema.string()),
-    timeout: schema.maybe(schema.string()),
-    timestamp_field: schema.maybe(schema.string()),
-    timestamp_format: schema.maybe(schema.string()),
-  })
-);