[CI tools] Use ES 8.x (#213056)

## Summary

After upgrading the ES client to 9.0
(https://github.com/elastic/kibana/pull/208776), we noticed that the CI
fails to upload the results of the tests to the CI cluster:

```
ERROR ResponseError: media_type_header_exception
      	Caused by:
      		status_exception: Accept version must be either version 8 or 7, but found 9. Accept=application/vnd.elasticsearch+json; compatible-with=9
      	Root causes:
      		media_type_header_exception: Invalid media-type value on headers [Content-Type, Accept]
```

This PR makes sure that the CI client is still using v8.x until we
upgrade that cluster.

package.json

@@ -1736,6 +1736,7 @@
     "dependency-check": "^4.1.0",
     "dependency-cruiser": "^16.8.0",
     "ejs": "^3.1.10",
+    "elasticsearch-8.x": "npm:@elastic/elasticsearch@8",
     "enzyme": "^3.11.0",
     "enzyme-to-json": "^3.6.2",
     "eslint": "^8.57.0",

packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_es.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Client, HttpConnection } from '@elastic/elasticsearch';
+import { Client, HttpConnection } from 'elasticsearch-8.x'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import { createFailError } from '@kbn/dev-cli-errors';
 import { ToolingLog } from '@kbn/tooling-log';
 

packages/kbn-performance-testing-dataset-extractor/src/es_client.ts

@@ -7,9 +7,12 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { Client, HttpConnection } from '@elastic/elasticsearch';
+import { Client, HttpConnection } from 'elasticsearch-8.x'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
-import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import {
-import { SearchRequest, MsearchRequestItem } from '@elastic/elasticsearch/lib/api/types';
+  QueryDslQueryContainer,
+  SearchRequest,
+  MsearchRequestItem,
+} from 'elasticsearch-8.x/lib/api/types'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import { ToolingLog } from '@kbn/tooling-log';
 
 interface ClientOptions {

renovate.json

@@ -952,6 +952,27 @@
       ],
       "enabled": true
     },
+    {
+      "groupName": "@elastic/elasticsearch (CI clients)",
+      "matchDepNames": [
+        "elasticsearch-8.x"
+      ],
+      "reviewers": [
+        "team:kibana-operations",
+        "team:appex-qa"
+      ],
+      "allowedVersions": "<9.0.0",
+      "matchBaseBranches": [
+        "main"
+      ],
+      "labels": [
+        "release_note:skip",
+        "backport:prev-minor",
+        "Team:Operations",
+        "Team:QA"
+      ],
+      "enabled": true
+    },
     {
       "groupName": "@elastic/elasticsearch",
       "matchDepNames": [
@@ -961,6 +982,28 @@
         "team:kibana-operations",
         "team:kibana-core"
       ],
+      "allowedVersions": "<9.0.0",
+      "matchBaseBranches": [
+        "8.x"
+      ],
+      "labels": [
+        "release_note:skip",
+        "backport:skip",
+        "Team:Operations",
+        "Team:Core"
+      ],
+      "enabled": true
+    },
+    {
+      "groupName": "@elastic/elasticsearch",
+      "matchDepNames": [
+        "@elastic/elasticsearch"
+      ],
+      "reviewers": [
+        "team:kibana-operations",
+        "team:kibana-core"
+      ],
+      "allowedVersions": "<8.0.0",
       "matchBaseBranches": [
         "7.17"
       ],

src/dev/code_coverage/ingest_coverage/ingest.js

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-const { Client, HttpConnection } = require('@elastic/elasticsearch');
+const { Client, HttpConnection } = require('elasticsearch-8.x'); // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import { RESEARCH_CI_JOB_NAME } from './constants';
 import { whichIndex } from './ingest_helpers';
 import { fromNullable } from './either';

src/platform/packages/private/kbn-scout-reporting/src/helpers/elasticsearch.ts

@@ -11,7 +11,7 @@ import {
   Client as ESClient,
   ClientOptions as ESClientOptions,
   HttpConnection,
-} from '@elastic/elasticsearch';
+} from 'elasticsearch-8.x'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import { ToolingLog } from '@kbn/tooling-log';
 import { createFailError } from '@kbn/dev-cli-errors';
 

src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/component_templates.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import { ClusterPutComponentTemplateRequest } from 'elasticsearch-8.x/lib/api/types'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import {
   buildkiteProperties,
   reporterProperties,

src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/index.ts

@@ -11,7 +11,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import readline from 'node:readline';
 import { ToolingLog } from '@kbn/tooling-log';
-import { Client as ESClient } from '@elastic/elasticsearch';
+import { Client as ESClient } from 'elasticsearch-8.x'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import {
   SCOUT_REPORTER_ES_API_KEY,
   SCOUT_REPORTER_ES_URL,

src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/index_templates.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types';
+import { IndicesPutIndexTemplateRequest } from 'elasticsearch-8.x/lib/api/types'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 import { SCOUT_TEST_EVENTS_TEMPLATE_NAME, SCOUT_TEST_EVENTS_INDEX_PATTERN } from '@kbn/scout-info';
 import * as componentTemplates from './component_templates';
 

src/platform/packages/private/kbn-scout-reporting/src/reporting/report/events/persistence/mappings.ts

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-import { PropertyName, MappingProperty } from '@elastic/elasticsearch/lib/api/types';
+import { PropertyName, MappingProperty } from 'elasticsearch-8.x/lib/api/types'; // Switch to `@elastic/elasticsearch` when the CI cluster is upgraded.
 
 export const buildkiteProperties: Record<PropertyName, MappingProperty> = {
   branch: {

src/platform/plugins/shared/data/server/search/strategies/sql_search/sql_search_strategy.ts

@@ -10,7 +10,7 @@
 import type { IncomingHttpHeaders } from 'http';
 import type { IScopedClusterClient, Logger } from '@kbn/core/server';
 import { catchError, tap } from 'rxjs';
-import type { DiagnosticResult } from '@elastic/transport';
+import type { DiagnosticResult } from '@elastic/elasticsearch';
 import { SqlQueryResponse, type SqlQuerySqlFormat } from '@elastic/elasticsearch/lib/api/types';
 import { getKbnServerError } from '@kbn/kibana-utils-plugin/server';
 import { getKbnSearchError } from '../../report_search_error';

x-pack/solutions/security/plugins/security_solution/server/endpoint/services/actions/mocks.ts

@@ -5,11 +5,10 @@
  * 2.0.
  */
 
-import type { estypes } from '@elastic/elasticsearch';
+import type { estypes, TransportRequestOptions } from '@elastic/elasticsearch';
 import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
 import { AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common';
 import { Readable } from 'stream';
-import type { TransportRequestOptions } from '@elastic/transport';
 import { applyEsClientSearchMock } from '../../mocks/utils.mock';
 import type { HapiReadableStream } from '../../../types';
 import { EndpointActionGenerator } from '../../../../common/endpoint/data_generators/endpoint_action_generator';

yarn.lock

@@ -2471,18 +2471,18 @@
     tslib "^2.8.1"
     undici "^7.2.3"
 
-"@elastic/transport@^8.3.1":
+"@elastic/transport@^8.3.1", "@elastic/transport@^8.9.1":
-  version "8.9.1"
+  version "8.9.4"
-  resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.9.1.tgz#1acc090ac45903a3d5a8b7269f6ba6410108dc0a"
+  resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.9.4.tgz#8acdc0b95cd0be8022e4e16ec5a755ab65ac5ea0"
-  integrity sha512-jasKNQeOb1vNf9aEYg+8zXmetaFjApDTSCC4QTl6aTixvyiRiSLcCiB8P6Q0lY9JIII/BhqNl8WbpFnsKitntw==
+  integrity sha512-y6kjy5s0MQE3MQx9ItmvQ8th7GlGcZfzZ7ZDvI8bUhaKua2dJk01k9ia/bdJ4dnPpWpOyFTRgkgBZS31ZTLpcg==
   dependencies:
     "@opentelemetry/api" "1.x"
-    debug "^4.3.4"
+    debug "^4.3.7"
-    hpagent "^1.0.0"
+    hpagent "^1.2.0"
     ms "^2.1.3"
-    secure-json-parse "^2.4.0"
+    secure-json-parse "^3.0.1"
-    tslib "^2.4.0"
+    tslib "^2.8.1"
-    undici "^6.12.0"
+    undici "^6.21.1"
 
 "@emotion/babel-plugin-jsx-pragmatic@^0.2.1":
   version "0.2.1"
@@ -18137,6 +18137,15 @@ elastic-apm-node@^4.10.0, elastic-apm-node@^4.11.0:
     stream-chopper "^3.0.1"
     unicode-byte-truncate "^1.0.0"
 
+"elasticsearch-8.x@npm:@elastic/elasticsearch@8":
+  version "8.17.1"
+  resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.17.1.tgz#97baa241f1f7bdd9b9b5c7479c64996e4c54adea"
+  integrity sha512-EaDP4/jfNu0nhnHZjxk9bL9ofKWKX9QUdEJ8QsGa+/KMPBEwD+HMyYXH4FSRlg7YONI0UbdO/mMZobvcEnMFBA==
+  dependencies:
+    "@elastic/transport" "^8.9.1"
+    apache-arrow "^18.0.0"
+    tslib "^2.4.0"
+
 elasticsearch@^16.4.0:
   version "16.7.0"
   resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-16.7.0.tgz#9055e3f586934d8de5fd407b04050e9d54173333"
@@ -21121,7 +21130,7 @@ hpack.js@^2.1.6:
     readable-stream "^2.0.1"
     wbuf "^1.1.0"
 
-hpagent@^1.0.0, hpagent@^1.2.0:
+hpagent@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903"
   integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==
@@ -29916,7 +29925,7 @@ screenfull@^5.0.0:
   resolved "https://registry.yarnpkg.com/screenfull/-/screenfull-5.0.0.tgz#5c2010c0e84fd4157bf852877698f90b8cbe96f6"
   integrity sha512-yShzhaIoE9OtOhWVyBBffA6V98CDCoyHTsp8228blmqYy1Z5bddzE/4FPiJKlr8DVR4VBiiUyfPzIQPIYDkeMA==
 
-secure-json-parse@^2.4.0, secure-json-parse@^2.7.0:
+secure-json-parse@^2.7.0:
   version "2.7.0"
   resolved "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz"
   integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==
@@ -32576,10 +32585,10 @@ undici@^5.28.5:
   dependencies:
     "@fastify/busboy" "^2.0.0"
 
-undici@^6.12.0:
+undici@^6.21.1:
-  version "6.19.2"
+  version "6.21.1"
-  resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.2.tgz#231bc5de78d0dafb6260cf454b294576c2f3cd31"
+  resolved "https://registry.yarnpkg.com/undici/-/undici-6.21.1.tgz#336025a14162e6837e44ad7b819b35b6c6af0e05"
-  integrity sha512-JfjKqIauur3Q6biAtHJ564e3bWa8VvT+7cSiOJHFbX4Erv6CLGDpg8z+Fmg/1OI/47RA+GI2QZaF48SSaLvyBA==
+  integrity sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==
 
 undici@^7.2.3:
   version "7.3.0"