[ML] Make script_fields available on dropdowns of the advanced wirzard's detectors modal. (#21205) (#21230)

- If a job configuration's datafeed contains script_fields, they are now available in the field dropdowns of the advanced wizard's detector modal.
- Additionally, job validation now considers script_fields and doesn't report them as non aggregatable anymore.

x-pack/plugins/ml/public/jobs/new_job/advanced/new_job_controller.js

@@ -351,6 +351,16 @@ module.controller('MlNewJob',
                 });
               });
 
+              // Add script fields from the job configuration to $scope.fields
+              // so they're available from within the dropdown in the detector modal.
+              const scriptFields = Object.keys(_.get($scope.job, 'datafeed_config.script_fields', {}));
+              // This type information is retrieved via fieldCaps for regular fields,
+              // here we're creating a similar object so the script field is usable further on.
+              const scriptType = { type: 'script_fields', searchable: false, aggregatable: true };
+              scriptFields.forEach((fieldName) => {
+                $scope.fields[fieldName] = scriptType;
+              });
+
               if (Object.keys($scope.fields).length) {
                 $scope.ui.indexTextOk = true;
               }

x-pack/plugins/ml/server/models/job_validation/__tests__/job_validation.js

@@ -325,6 +325,81 @@ describe('ML - validateJob', () => {
     );
   });
 
+  it('non-existent field reported as non aggregatable', () => {
+    const payload = {
+      job: {
+        job_id: 'categorization_test',
+        analysis_config: {
+          bucket_span: '15m',
+          detectors: [{
+            function: 'count',
+            partition_field_name: 'ailine'
+          }],
+          influencers: []
+        },
+        data_description: { time_field: '@timestamp' },
+        datafeed_config: { indices: [] }
+      },
+      fields: { testField: {} }
+    };
+
+    return validateJob(callWithRequest, payload).then(
+      (messages) => {
+        const ids = messages.map(m => m.id);
+        expect(ids).to.eql([
+          'job_id_valid',
+          'detectors_function_not_empty',
+          'index_fields_valid',
+          'field_not_aggregatable',
+          'time_field_invalid'
+        ]);
+      }
+    );
+  });
+
+  it('script field not reported as non aggregatable', () => {
+    const payload = {
+      job: {
+        job_id: 'categorization_test',
+        analysis_config: {
+          bucket_span: '15m',
+          detectors: [{
+            function: 'count',
+            partition_field_name: 'custom_script_field'
+          }],
+          influencers: []
+        },
+        data_description: { time_field: '@timestamp' },
+        datafeed_config: {
+          indices: [],
+          script_fields: {
+            custom_script_field: {
+              script: {
+                source: `'some script'`,
+                lang: 'painless'
+              }
+            }
+          },
+        }
+      },
+      fields: { testField: {} }
+    };
+
+    return validateJob(callWithRequest, payload).then(
+      (messages) => {
+        const ids = messages.map(m => m.id);
+        expect(ids).to.eql([
+          'job_id_valid',
+          'detectors_function_not_empty',
+          'index_fields_valid',
+          'success_cardinality',
+          'time_field_invalid',
+          'influencer_low_suggestion'
+        ]);
+      }
+    );
+  });
+
   // the following two tests validate the correct template rendering of
   // urls in messages with {{version}} in them to be replaced with the
   // specified version. (defaulting to 'current')

x-pack/plugins/ml/server/models/job_validation/validate_cardinality.js

@@ -12,6 +12,18 @@ import { DataVisualizer } from '../data_visualizer';
 
 import { validateJobObject } from './validate_job_object';
 
+function isValidCategorizationConfig(job, fieldName) {
+  return (
+    typeof job.analysis_config.categorization_field_name !== 'undefined' &&
+    fieldName === 'mlcategory'
+  );
+}
+
+function isScriptField(job, fieldName) {
+  const scriptFields = Object.keys(_.get(job, 'datafeed_config.script_fields', {}));
+  return scriptFields.includes(fieldName);
+}
+
 // Thresholds to determine whether cardinality is
 // too high or low for certain fields analysis
 const OVER_FIELD_CARDINALITY_THRESHOLD_LOW = 10;
@@ -84,12 +96,12 @@ const validateFactory = (callWithRequest, job) => {
               });
             }
           } else {
-            // when the job is using categorization and the field name is set to 'mlcategory',
+            // only report uniqueFieldName as not aggregatable if it's not part
-            // then don't report the field as not being able to be aggregated.
+            // of a valid categorization configuration and if it's not a scripted field.
-            if (!(
+            if (
-              typeof job.analysis_config.categorization_field_name !== 'undefined' &&
+              !isValidCategorizationConfig(job, uniqueFieldName) &&
-              uniqueFieldName === 'mlcategory'
+              !isScriptField(job, uniqueFieldName)
-            )) {
+            ) {
               messages.push({
                 id: 'field_not_aggregatable',
                 fieldName: uniqueFieldName