[8.x] GenAI FTR tests: switch to daily job (#203369) (#203454)

# Backport

This will backport the following commits from `main` to `8.x`:
- [GenAI FTR tests: switch to daily job
(#203369)](https://github.com/elastic/kibana/pull/203369)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Pierre
Gayvallet","email":"pierre.gayvallet@elastic.co"},"sourceCommit":{"committedDate":"2024-12-09T15:09:29Z","message":"GenAI
FTR tests: switch to daily job (#203369)\n\n## Summary\r\n\r\nPart of
https://github.com/elastic/kibana-team/issues/1271\r\nFollow-up of
https://github.com/elastic/kibana/pull/198000\r\n\r\nFix
https://github.com/elastic/kibana/issues/203205\r\nFix
https://github.com/elastic/kibana/issues/203062\r\n\r\n- Fix the problem
with the Gemini token count events \r\n- Unskip the test suite\r\n-
Remove the suite from the `on-merge` pipeline\r\n- Add a dedicated
pipeline to run the tests
daily","sha":"1f2a76a44bb6e339441e1ba10c861d5a2b7b9667","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:version","Team:AI
Infra","v8.18.0"],"number":203369,"url":"https://github.com/elastic/kibana/pull/203369","mergeCommit":{"message":"GenAI
FTR tests: switch to daily job (#203369)\n\n## Summary\r\n\r\nPart of
https://github.com/elastic/kibana-team/issues/1271\r\nFollow-up of
https://github.com/elastic/kibana/pull/198000\r\n\r\nFix
https://github.com/elastic/kibana/issues/203205\r\nFix
https://github.com/elastic/kibana/issues/203062\r\n\r\n- Fix the problem
with the Gemini token count events \r\n- Unskip the test suite\r\n-
Remove the suite from the `on-merge` pipeline\r\n- Add a dedicated
pipeline to run the tests
daily","sha":"1f2a76a44bb6e339441e1ba10c861d5a2b7b9667"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/203369","number":203369,"mergeCommit":{"message":"GenAI
FTR tests: switch to daily job (#203369)\n\n## Summary\r\n\r\nPart of
https://github.com/elastic/kibana-team/issues/1271\r\nFollow-up of
https://github.com/elastic/kibana/pull/198000\r\n\r\nFix
https://github.com/elastic/kibana/issues/203205\r\nFix
https://github.com/elastic/kibana/issues/203062\r\n\r\n- Fix the problem
with the Gemini token count events \r\n- Unskip the test suite\r\n-
Remove the suite from the `on-merge` pipeline\r\n- Add a dedicated
pipeline to run the tests
daily","sha":"1f2a76a44bb6e339441e1ba10c861d5a2b7b9667"}},{"branch":"8.x","label":"v8.18.0","labelRegex":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml

@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-gen-ai-daily
+  description: Runs the GenAI FTR tests daily
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-gen-ai-tests-daily'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:appex-ai-infra'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / gen-ai-tests / daily
+      description: Runs the GenAI FTR tests daily
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#appex-ai-infra-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      cancel_intermediate_builds: true
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/gen_ai_testing.yml
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        appex-ai-infra:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 0 * * * America/New_York
+          message: Daily build
+          branch: main
+      tags:
+        - kibana

.buildkite/pipeline-resource-definitions/locations.yml

@@ -45,3 +45,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml

.buildkite/pipelines/gen_ai_testing.yml

@@ -0,0 +1,48 @@
+env:
+  FTR_GEN_AI: "1"
+steps:
+  - label: '👨‍🔧 Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - label: '🧑‍🏭 Build Kibana Distribution'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
+      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+      FTR_GEN_AI: "1"
+    label: AppEx AI-Infra Inference APIs FTR tests
+    key: ai-infra-gen-ai-inference-api
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+

.buildkite/pipelines/on_merge.yml

@@ -169,28 +169,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/ftr_configs.sh
-    env:
-      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
-      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
-      FTR_GEN_AI: "1"
-    label: AppEx AI-Infra Inference APIs FTR tests
-    key: ai-infra-gen-ai-inference-api
-    timeout_in_minutes: 50
-    parallelism: 1
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
     label: 'Serverless Explore - Security Solution Cypress Tests'
     agents:

x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts

@@ -38,8 +38,9 @@ export function processVertexStream() {
           });
         }
 
-        // completion: only present on last chunk
-        if (value.usageMetadata) {
+        // 'usageMetadata' can be present as an empty object on chunks
+        // only the last chunk will have its fields populated
+        if (value.usageMetadata?.totalTokenCount) {
           subscriber.next({
             type: ChatCompletionEventType.ChatCompletionTokenCount,
             tokens: {

x-pack/test/functional_gen_ai/inference/tests/index.ts

@@ -11,8 +11,7 @@ import { chatCompleteSuite } from './chat_complete';
 
 // eslint-disable-next-line import/no-default-export
 export default function (providerContext: FtrProviderContext) {
-  // Failing: See https://github.com/elastic/kibana/issues/203205
-  describe.skip('Inference plugin - API integration tests', async () => {
+  describe('Inference plugin - API integration tests', async () => {
     getAvailableConnectors().forEach((connector) => {
       describe(`Connector ${connector.id}`, () => {
         chatCompleteSuite(connector, providerContext);