[Rest Api Compatibility] Enable tests after types and cat api fixed (#75179)

Some tests are fixed after typed api is available with compatible api.
Also cat api returning text fixed some tests
relates #51816

modules/analysis-common/build.gradle

@@ -25,11 +25,18 @@ dependencies {
   compileOnly project(':modules:lang-painless')
 }
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens',
-    'indices.analyze/10_analyze/htmlStrip_deprecated',
-    'analysis-common/40_token_filters/delimited_payload_filter_error',
-    'analysis-common/20_analyzers/standard_html_strip'
-  ].join(',')
+def v7compatibilityNotSupportedTests = {
+  return [
+     //marked as not needing compatible api
+     'indices.analyze/10_analyze/htmlStrip_deprecated', // Cleanup versioned deprecations in analysis #41560
+     'analysis-common/40_token_filters/delimited_payload_filter_error', //Remove preconfigured delimited_payload_filter #43686
+     'analysis-common/20_analyzers/standard_html_strip' // Cleanup versioned deprecations in analysis #41560
+  ]
+}
+
+tasks.named("yamlRestCompatTest").configure {
+  systemProperty 'tests.rest.blacklist', ([
+    'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 - not sure, remove `common` query
+  ]+ v7compatibilityNotSupportedTests())
+  .join(',')
 }

modules/ingest-common/build.gradle

@@ -44,10 +44,6 @@ tasks.named("thirdPartyAudit").configure {
   )
 }
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'ingest/80_foreach/Test foreach Processor',
-    'ingest/230_change_target_index/Test Change Target Index with Explicit Pipeline',
-    'ingest/230_change_target_index/Test Change Target Index with Default Pipeline'
-  ].join(',')
-}
+tasks.named("transformV7RestTests").configure({ task ->
+  task.addAllowedWarningRegex("\\[types removal\\].*")
+})

modules/ingest-user-agent/build.gradle

@@ -23,9 +23,6 @@ testClusters.all {
   extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml')
 }
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'ingest-useragent/30_custom_regex/Test user agent processor with custom regex file',
-    'ingest-useragent/20_useragent_processor/Test user agent processor with parameters'
-  ].join(',')
-}
+tasks.named("transformV7RestTests").configure({ task ->
+  task.addAllowedWarningRegex("setting \\[ecs\\] is deprecated as ECS format is the default and only option")
+})

modules/lang-mustache/build.gradle

@@ -26,9 +26,6 @@ restResources {
   }
 }
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'lang_mustache/60_typed_keys/Multisearch template with typed_keys parameter',
-    'lang_mustache/60_typed_keys/Search template with typed_keys parameter'
-  ].join(',')
-}
+tasks.named("transformV7RestTests").configure({ task ->
+  task.addAllowedWarningRegex("\\[types removal\\].*")
+})

modules/repository-url/build.gradle

@@ -63,10 +63,3 @@ testClusters.all {
   }, PropertyNormalization.IGNORE_VALUE
 }
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'repository_url/10_basic/Get a non existing snapshot',
-    'repository_url/10_basic/Restore with repository-url using http://',
-    'repository_url/10_basic/Restore with repository-url using file://'
-  ].join(',')
-}

plugins/analysis-icu/build.gradle

@@ -36,11 +36,15 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
+def v7compatibilityNotSupportedTests = {
+  return [
+    //marked as not needing compatible api
+    'analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter' // Cleanup versioned deprecations in analysis #41560
+  ]
+}
 
 tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter'
-  ].join(',')
+  systemProperty 'tests.rest.blacklist', v7compatibilityNotSupportedTests().join(',')
 }
 
 tasks.named('splitPackagesAudit').configure {

rest-api-spec/build.gradle

@@ -71,7 +71,15 @@ def v7compatibilityNotSupportedTests = {
           'indices.stats/20_translog/Translog stats on closed indices without soft-deletes',
 
           // upgrade api will only get a dummy endpoint returning an exception suggesting to use _reindex
-          'indices.upgrade/*/*'
+          'indices.upgrade/*/*',
+
+          'search.aggregation/20_terms/*profiler*', // The profiler results aren't backwards compatible.
+          'search.aggregation/370_doc_count_field/Test filters agg with doc_count', // Uses profiler for assertions which is not backwards compatible
+
+          'indices.create/10_basic/Create index without soft deletes', //Make soft-deletes mandatory in 8.0 #51122 - settings changes are note supported in Rest Api compatibility
+
+
+          'field_caps/30_filter/Field caps with index filter', //behaviour change after #63692 4digits dates are parsed as epoch and in quotes as year
   ]
 }
 tasks.named("yamlRestCompatTest").configure {
@@ -79,12 +87,9 @@ tasks.named("yamlRestCompatTest").configure {
     // Skip these tests on Windows since the blacklist exceeds Windows CLI limits
     OS.current() != OS.WINDOWS
   }
-
   systemProperty 'tests.rest.blacklist', ([
     'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion and specifying both node_ids and node_names',
     'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion without specifying nodes',
-    'field_caps/30_filter/Field caps with index filter',
-    'indices.create/10_basic/Create index without soft deletes',
     'indices.flush/10_basic/Index synced flush rest test',
     'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set',
     'indices.open/10_basic/?wait_for_active_shards default is deprecated',
@@ -92,15 +97,14 @@ tasks.named("yamlRestCompatTest").configure {
     // not fixing this in #70966
     'indices.put_template/11_basic_with_types/Put template with empty mappings',
     'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers',
-    'search.aggregation/20_terms/*profiler*', // The profiler results aren't backwards compatible.
     'search.aggregation/51_filter_with_types/Filter aggs with terms lookup and ensure it\'s cached',
-    'search.aggregation/370_doc_count_field/Test filters agg with doc_count', // Uses profiler for assertions which is not backwards compatible
     'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', //terms_lookup
     'search/260_parameter_validation/test size=-1 is deprecated', //size=-1 change
     'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency
     'search/340_type_query/type query', // type_query - probably should behave like match_all
   ] + v7compatibilityNotSupportedTests())
     .join(',')
+
 }
 
 tasks.named("transformV7RestTests").configure({ task ->

x-pack/plugin/watcher/qa/rest/build.gradle

@@ -54,16 +54,3 @@ tasks.named("transformV7RestTests").configure({ task ->
 
 })
 
-tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', [
-    'mustache/25_array_compare/Basic array_compare watch',
-    'mustache/30_search_input/Test search input mustache integration (using request body and rest_total_hits_as_int)',
-    'mustache/30_search_input/Test search input mustache integration (using request body)',
-    'mustache/30_search_input/Test search input mustache integration (using request template and rest_total_hits_as_int)',
-    'mustache/30_search_input/Test search input mustache integration (using request template)',
-    'mustache/40_search_transform/Test search transform mustache integration (using request body)',
-    'mustache/40_search_transform/Test search transform mustache integration (using request template)',
-  ].join(',')
-}
-
-