[Lens] Query based annotations (#138753)

* ⚗️ Initial code for query based annotations

* 🐛 Solved more conflicts

* ⚗️ More scaffolding layout

* ⚗️ Initial indexpatetrn move into frame

* ⚗️ Make field selection work

* 🚧 Fixed almost all dataViews occurrencies, but changeIndexPattern

* 🚧 More work on change index pattern

* Move lens dataViews state into main state

* 🔥 Remove some old cruft from the code

* 🐛 Fix dataViews layer change

* 🐛 Fix datasourceLayers refs

* 🔥 Remove more old cruft

* 🐛 Fix bug when loading SO

* 🐛 Fix initial existence flag

* 🏷️ Fix type issues

* 🏷️ Fix types and tests

* 🏷️ Fix types issues

* ✅ Fix more tests

* ✅ Fix with new dataViews structure

* ✅ Fix more test mocks

* ✅ More tests fixed

* 🔥 Removed unused prop

* ✅ Down to single broken test suite

* 🏷️ Fix type issue

* 👌 Integrate selector feedback

* ✅ Fix remaining unit tests

* 🏷️ fix type issues

* 🐛 Fix bug when creating dataview in place

* ✨ Update with latest dataview state + fix dataviews picker for annotations

* 🐛 Fix edit + remove field flow

* Update x-pack/plugins/lens/public/visualizations/xy/types.ts

* 📸 Fix snapshot

* 🐛 Fix the dataViews switch bug

* 🔥 remove old cruft

* ♻️ Revert removal from dataviews state branch

* ♻️ Load all at once

* 🔧 working on persistent state + fix new layer bug

* 🔥 remove unused stuff

* 🏷️ Fix some typings

* 🔧 Fix expression issue

* ✅ Add service unit tests

* 👌 Integrated feedback

* ✨ Add migration code for manual annotations

* 🏷️ Fix type issue

* ✅ Add some other unit test

* 🏷️ Fix more type issues

* 🐛 Fix importing issue

* ♻️ Make range default color dependant on opint one

* 🐛 Fix duplicate fields selection in tooltip section

* ✅ Add more unit tests

* ✅ Fix broken test

* 🏷️ Mute ts error for now

* ✅ Fix tests

* 🔥 Reduce plugin weight

* 🐛 prevent layout shift on panel open

* 🐛 Fix extract + inject visualization references

* 🏷️ fix type issues

* ✨ Add dataview reference migration for annotations

* 🔧 Add migration to embedadble

* 🏷️ Fix type export

* 🐛 Fix more conflicts with main

* ✅ Fix tests

* 🏷️ Make textField optional

* ♻️ Refactor query input to be a shared component

* 🐛 Fix missing import

* 🐛 fix more import issues

* 🔥 remove duplicate code

* 🐛 Fix dataView switch bug

* 🏷️ Fix type issue

* annotations with fetching_event_annotations

* portal for kql input fix

* timeField goes for default if not filled

* limit changes

* handle ad-hoc data view references correctly

* fix types

* adjust tests to datatable format (remove isHidden tests as it's filtered before)

* small refactors

* fix loading on dashboard

* empty is invalid (?) tbd

* new tooltip

* emptyDatatable

* ♻️ Flip field + query inputs

* 🏷️ Fix type issue

* ✨ Add field validation for text and tooltip fields

* tooltip for single annotation

* fix tests

* fix for non--timefilter dataview

* fix annotations test - the cause was that we now don't display label for aggregated annotations ever

* use eui elements

* newline problem solved

* ✅ Add more error tests

* 👌 Rename migration state version type

* fix types for expression chart

* 🐛 Fix i18n id

* 🏷️ Fix type issue

* fix hidden all annotations

* ✅ Fix tests after ishidden removal

* 🐛 Revert references migration to an in app solution

Co-authored-by: Joe Reuter <johannes.reuter@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Marta Bondyra <4283304+mbondyra@users.noreply.github.com>
Co-authored-by: Marta Bondyra <marta.bondyra@elastic.co>

x-pack/plugins/lens/server/embeddable/make_lens_embeddable_factory.ts

@@ -27,6 +27,7 @@ import {
   commonUpdateVisLayerType,
   getLensCustomVisualizationMigrations,
   getLensFilterMigrations,
+  commonExplicitAnnotationType,
   getLensDataViewMigrations,
   commonMigrateMetricIds,
   commonMigratePartitionChartGroups,
@@ -36,13 +37,14 @@ import {
   LensDocShape713,
   LensDocShape715,
   LensDocShape810,
-  LensDocShape840,
+  LensDocShape850,
   LensDocShapePre712,
   VisState716,
   VisState810,
-  VisState840,
+  VisState850,
   VisStatePre715,
   VisStatePre830,
+  XYVisState850,
 } from '../migrations/types';
 import { extract, inject } from '../../common/embeddable_factory';
 
@@ -134,10 +136,16 @@ export const makeLensEmbeddableFactory =
                 } as unknown as SerializableRecord;
               },
               '8.5.0': (state) => {
-                const lensState = state as unknown as { attributes: LensDocShape840<VisState840> };
+                const lensState = state as unknown as {
+                  attributes: LensDocShape850<VisState850>;
+                };
+
                 let migratedLensState = commonMigrateMetricIds(lensState.attributes);
+                migratedLensState = commonExplicitAnnotationType(
+                  migratedLensState as LensDocShape850<XYVisState850>
+                );
                 migratedLensState = commonMigratePartitionChartGroups(
-                  migratedLensState as LensDocShape840<{
+                  migratedLensState as LensDocShape850<{
                     shape: string;
                     layers: Array<{ groups?: string[] }>;
                   }>

x-pack/plugins/lens/server/migrations/common_migrations.ts

@@ -30,7 +30,8 @@ import {
   LensDocShape810,
   LensDocShape830,
   VisStatePre830,
-  LensDocShape840,
+  XYVisStatePre850,
+  VisState850,
   LensDocShape850,
 } from './types';
 import { DOCUMENT_FIELD_NAME, layerTypes, LegacyMetricState, isPartitionShape } from '../../common';
@@ -422,16 +423,49 @@ export const commonFixValueLabelsInXY = (
   };
 };
 
+export const commonExplicitAnnotationType = (
+  attributes: LensDocShape850<XYVisStatePre850>
+): LensDocShape850<VisState850> => {
+  // Skip the migration heavy part if not XY or it does not contain annotations
+  if (
+    attributes.visualizationType !== 'lnsXY' ||
+    attributes.state.visualization.layers.every((l) => l.layerType !== 'annotations')
+  ) {
+    return attributes as LensDocShape850<VisState850>;
+  }
+  const newAttributes = cloneDeep(attributes);
+  const { visualization } = newAttributes.state;
+  const { layers } = visualization;
+  return {
+    ...newAttributes,
+    state: {
+      ...newAttributes.state,
+      visualization: {
+        ...visualization,
+        layers: layers.map((l) => {
+          if (l.layerType !== 'annotations') {
+            return l;
+          }
+          return {
+            ...l,
+            annotations: l.annotations.map((a) => ({ ...a, type: 'manual' })),
+          };
+        }),
+      },
+    },
+  };
+};
+
 export const commonMigrateMetricIds = (
-  attributes: LensDocShape840<unknown>
-): LensDocShape840<unknown> => {
+  attributes: LensDocShape850<unknown>
+): LensDocShape850<unknown> => {
   const typeMappings = {
     lnsMetric: 'lnsLegacyMetric',
     lnsMetricNew: 'lnsMetric',
   } as Record<string, string>;
 
   if (!attributes.visualizationType || !(attributes.visualizationType in typeMappings)) {
-    return attributes as LensDocShape840<unknown>;
+    return attributes as LensDocShape850<unknown>;
   }
 
   const newAttributes = cloneDeep(attributes);
@@ -441,7 +475,7 @@ export const commonMigrateMetricIds = (
 };
 
 export const commonMigratePartitionChartGroups = (
-  attributes: LensDocShape840<{
+  attributes: LensDocShape850<{
     shape: string;
     layers: Array<{ groups?: string[] }>;
   }>

x-pack/plugins/lens/server/migrations/saved_object_migrations.test.ts

@@ -22,6 +22,9 @@ import {
   VisState810,
   VisState820,
   VisState830,
+  LensDocShape850,
+  XYVisStatePre850,
+  VisState850,
 } from './types';
 import { layerTypes, LegacyMetricState } from '../../common';
 import { Filter } from '@kbn/es-query';
@@ -2291,6 +2294,44 @@ describe('Lens migrations', () => {
     });
   });
 
+  describe('8.5.0 Add Annotation event type and dataView references', () => {
+    const context = { log: { warn: () => {} } } as unknown as SavedObjectMigrationContext;
+    const example = {
+      type: 'lens',
+      id: 'mocked-saved-object-id',
+      attributes: {
+        savedObjectId: '1',
+        title: 'MyRenamedOps',
+        description: '',
+        visualizationType: 'lnsXY',
+        state: {
+          visualization: {
+            layers: [
+              { layerType: 'data' },
+              {
+                layerType: 'annotations',
+                annotations: [{ id: 'annotation-id' }],
+              },
+            ],
+          },
+        },
+      },
+    } as unknown as SavedObjectUnsanitizedDoc<LensDocShape850<XYVisStatePre850>>;
+
+    it('migrates existing annotation events as manual type', () => {
+      const result = migrations['8.5.0'](example, context) as ReturnType<
+        SavedObjectMigrationFn<LensDocShape, LensDocShape>
+      >;
+      const visState = result.attributes.state.visualization as VisState850;
+      const [dataLayer, annotationLayer] = visState.layers;
+      expect(dataLayer).toEqual({ layerType: 'data' });
+      expect(annotationLayer).toEqual({
+        layerType: 'annotations',
+        annotations: [{ id: 'annotation-id', type: 'manual' }],
+      });
+    });
+  });
+
   describe('8.5.0 migrates metric IDs', () => {
     const context = { log: { warn: () => {} } } as unknown as SavedObjectMigrationContext;
     const example = {

x-pack/plugins/lens/server/migrations/saved_object_migrations.ts

@@ -34,7 +34,10 @@ import {
   XYVisualizationState830,
   VisState810,
   VisState820,
+  XYVisStatePre850,
+  LensDocShape850,
   LensDocShape840,
+  VisState850,
 } from './types';
 import {
   commonRenameOperationsForFormula,
@@ -52,6 +55,7 @@ import {
   commonFixValueLabelsInXY,
   commonLockOldMetricVisSettings,
   commonPreserveOldLegendSizeDefault,
+  commonExplicitAnnotationType,
   getLensDataViewMigrations,
   commonMigrateMetricIds,
   commonMigratePartitionChartGroups,
@@ -516,7 +520,15 @@ const preserveOldLegendSizeDefault: SavedObjectMigrationFn<LensDocShape810, Lens
   doc
 ) => ({ ...doc, attributes: commonPreserveOldLegendSizeDefault(doc.attributes) });
 
-const migrateMetricIds: SavedObjectMigrationFn<LensDocShape840, LensDocShape840> = (doc) => ({
+const addEventAnnotationType: SavedObjectMigrationFn<
+  LensDocShape850<XYVisStatePre850>,
+  LensDocShape850<VisState850>
+> = (doc) => {
+  const newDoc = cloneDeep(doc);
+  return { ...newDoc, attributes: commonExplicitAnnotationType(newDoc.attributes) };
+};
+
+const migrateMetricIds: SavedObjectMigrationFn<LensDocShape850, LensDocShape850> = (doc) => ({
   ...doc,
   attributes: commonMigrateMetricIds(doc.attributes),
 });
@@ -553,7 +565,7 @@ const lensMigrations: SavedObjectMigrationMap = {
     enhanceTableRowHeight
   ),
   '8.3.0': flow(lockOldMetricVisSettings, preserveOldLegendSizeDefault, fixValueLabelsInXY),
-  '8.5.0': flow(migrateMetricIds, migratePartitionChartGroups),
+  '8.5.0': flow(migrateMetricIds, addEventAnnotationType, migratePartitionChartGroups),
 };
 
 export const getAllMigrations = (

x-pack/plugins/lens/server/migrations/types.ts

@@ -270,6 +270,34 @@ export interface XYVisualizationState830 extends VisState820 {
 export type VisStatePre830 = XYVisualizationStatePre830;
 export type VisState830 = XYVisualizationState830;
 
+export interface XYVisStatePre850 {
+  layers: Array<
+    | {
+        layerId: string;
+        layerType: Exclude<LayerType, 'annotations'>;
+      }
+    | {
+        layerId: string;
+        layerType: Extract<LayerType, 'annotations'>;
+        annotations: Array<{ id: string }>;
+      }
+  >;
+}
+
+export interface XYVisState850 {
+  layers: Array<
+    | {
+        layerId: string;
+        layerType: Exclude<LayerType, 'annotations'>;
+      }
+    | {
+        layerId: string;
+        layerType: Extract<LayerType, 'annotations'>;
+        annotations: Array<{ id: string; type: 'manual' | 'query' }>;
+      }
+  >;
+}
+export type VisState850 = XYVisState850;
 export type VisState840 = VisState830;
 export type LensDocShape840<VisualizationState = unknown> = LensDocShape830<VisualizationState>;