Logs Sample Data with index in Time Series mode (#152686)

src/plugins/home/server/plugin.ts

@@ -31,9 +31,12 @@ export class HomeServerPlugin implements Plugin<HomeServerPluginSetup, HomeServe
   private readonly sampleDataRegistry: SampleDataRegistry;
   private customIntegrations?: CustomIntegrationsPluginSetup;
 
+  private readonly isDevMode: boolean;
+
   constructor(private readonly initContext: PluginInitializerContext) {
     this.sampleDataRegistry = new SampleDataRegistry(this.initContext);
     this.tutorialsRegistry = new TutorialsRegistry(this.initContext);
+    this.isDevMode = this.initContext.env.mode.dev;
   }
 
   public setup(core: CoreSetup, plugins: HomeServerPluginSetupDependencies): HomeServerPluginSetup {
@@ -48,7 +51,12 @@ export class HomeServerPlugin implements Plugin<HomeServerPluginSetup, HomeServe
     return {
       tutorials: { ...this.tutorialsRegistry.setup(core, plugins.customIntegrations) },
       sampleData: {
-        ...this.sampleDataRegistry.setup(core, plugins.usageCollection, plugins.customIntegrations),
+        ...this.sampleDataRegistry.setup(
+          core,
+          plugins.usageCollection,
+          plugins.customIntegrations,
+          this.isDevMode
+        ),
       },
     };
   }

src/plugins/home/server/services/sample_data/data_sets/index.ts

@@ -9,3 +9,4 @@
 export { flightsSpecProvider } from './flights';
 export { logsSpecProvider } from './logs';
 export { ecommerceSpecProvider } from './ecommerce';
+export { logsTSDBSpecProvider } from './logs_tsdb';

src/plugins/home/server/services/sample_data/data_sets/logs_tsdb/field_mappings.ts

@@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const fieldMappings = {
+  request: {
+    type: 'keyword',
+    time_series_dimension: true,
+  },
+  geo: {
+    properties: {
+      srcdest: {
+        type: 'keyword',
+      },
+      src: {
+        type: 'keyword',
+      },
+      dest: {
+        type: 'keyword',
+      },
+      coordinates: {
+        type: 'geo_point',
+      },
+    },
+  },
+  utc_time: {
+    type: 'date',
+  },
+  url: {
+    type: 'keyword',
+    time_series_dimension: true,
+  },
+  message: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  host: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  clientip: {
+    type: 'ip',
+  },
+  response: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  machine: {
+    properties: {
+      ram: {
+        type: 'long',
+      },
+      os: {
+        type: 'text',
+        fields: {
+          keyword: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+        },
+      },
+    },
+  },
+  agent: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  bytes: {
+    type: 'long',
+  },
+  tags: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  referer: {
+    type: 'keyword',
+  },
+  ip: {
+    type: 'ip',
+  },
+  '@timestamp': {
+    type: 'date',
+  },
+  timestamp: {
+    type: 'alias',
+    path: '@timestamp',
+  },
+  phpmemory: {
+    type: 'long',
+  },
+  bytes_counter: {
+    type: 'long',
+    time_series_metric: 'counter',
+  },
+  bytes_gauge: {
+    type: 'long',
+    time_series_metric: 'gauge',
+  },
+  memory: {
+    type: 'double',
+  },
+  extension: {
+    type: 'text',
+    fields: {
+      keyword: {
+        type: 'keyword',
+        ignore_above: 256,
+      },
+    },
+  },
+  event: {
+    properties: {
+      dataset: {
+        type: 'keyword',
+      },
+    },
+  },
+};

src/plugins/home/server/services/sample_data/data_sets/logs_tsdb/index.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import path from 'path';
+import { i18n } from '@kbn/i18n';
+import { getSavedObjects } from './saved_objects';
+import { fieldMappings } from './field_mappings';
+import { SampleDatasetSchema } from '../../lib/sample_dataset_registry_types';
+
+const logsName = i18n.translate('home.sampleData.logsTsdbSpecTitle', {
+  defaultMessage: 'Sample web logs (TSDB)',
+});
+const logsDescription = i18n.translate('home.sampleData.logsTsdbSpecDescription', {
+  defaultMessage: 'Sample data, visualizations, and dashboards for monitoring web logs.',
+});
+
+export const GLOBE_ICON_PATH = '/plugins/home/assets/sample_data_resources/logs/icon.svg';
+export const logsTSDBSpecProvider = function (): SampleDatasetSchema {
+  const startDate = new Date();
+  const endDate = new Date();
+  startDate.setMonth(startDate.getMonth() - 1);
+  endDate.setMonth(endDate.getMonth() + 2);
+  return {
+    id: 'logstsdb',
+    name: logsName,
+    description: logsDescription,
+    previewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard.webp',
+    darkPreviewImagePath: '/plugins/home/assets/sample_data_resources/logs/dashboard_dark.webp',
+    overviewDashboard: 'edf84fe0-e1a0-11e7-b6d5-4dc382ef8f5b',
+    defaultIndex: '90943e30-9a47-11e8-b64d-95841ca0c247',
+    savedObjects: getSavedObjects(),
+    dataIndices: [
+      {
+        id: 'logstsdb',
+        dataPath: path.join(__dirname, './logs.json.gz'),
+        fields: fieldMappings,
+        timeFields: ['@timestamp', 'utc_time'],
+        currentTimeMarker: '2018-08-01T00:00:00',
+        preserveDayOfWeekTimeOfDay: true,
+        indexSettings: {
+          number_of_shards: 1,
+          auto_expand_replicas: '0-1',
+          mode: 'time_series',
+          routing_path: 'request',
+          'time_series.start_time': startDate.toISOString(),
+          'time_series.end_time': endDate.toISOString(),
+        },
+      },
+    ],
+    status: 'not_installed',
+    iconPath: GLOBE_ICON_PATH,
+  };
+};

src/plugins/home/server/services/sample_data/lib/sample_dataset_schema.ts

@@ -30,6 +30,9 @@ const dataIndexSchema = schema.object({
   // should index be created as data stream
   isDataStream: schema.maybe(schema.boolean({ defaultValue: false })),
 
+  // additional index settings
+  indexSettings: schema.maybe(schema.recordOf(schema.string(), schema.any())),
+
   // Reference to now in your test data set.
   // When data is installed, timestamps are converted to the present time.
   // The distance between a timestamp and currentTimeMarker is preserved but the date and time will change.

src/plugins/home/server/services/sample_data/sample_data_installer.ts

@@ -177,7 +177,13 @@ export class SampleDataInstaller {
         await this.esClient.asCurrentUser.indices.create({
           index,
           body: {
-            settings: { index: { number_of_shards: 1, auto_expand_replicas: '0-1' } },
+            settings: {
+              index: {
+                ...dataIndex.indexSettings,
+                number_of_shards: 1,
+                auto_expand_replicas: '0-1',
+              },
+            },
             mappings: { properties: dataIndex.fields },
           },
         });

src/plugins/home/server/services/sample_data/sample_data_registry.ts

@@ -18,7 +18,12 @@ import {
 } from './lib/sample_dataset_registry_types';
 import { sampleDataSchema } from './lib/sample_dataset_schema';
 
-import { flightsSpecProvider, logsSpecProvider, ecommerceSpecProvider } from './data_sets';
+import {
+  flightsSpecProvider,
+  logsSpecProvider,
+  ecommerceSpecProvider,
+  logsTSDBSpecProvider,
+} from './data_sets';
 import { createListRoute, createInstallRoute } from './routes';
 import { makeSampleDataUsageCollector, usage } from './usage';
 import { createUninstallRoute } from './routes/uninstall';
@@ -59,7 +64,8 @@ export class SampleDataRegistry {
   public setup(
     core: CoreSetup,
     usageCollections: UsageCollectionSetup | undefined,
-    customIntegrations?: CustomIntegrationsPluginSetup
+    customIntegrations?: CustomIntegrationsPluginSetup,
+    isDevMode?: boolean
   ) {
     if (usageCollections) {
       const kibanaIndex = core.savedObjects.getKibanaIndex();
@@ -78,6 +84,9 @@ export class SampleDataRegistry {
     this.registerSampleDataSet(flightsSpecProvider);
     this.registerSampleDataSet(logsSpecProvider);
     this.registerSampleDataSet(ecommerceSpecProvider);
+    if (isDevMode) {
+      this.registerSampleDataSet(logsTSDBSpecProvider);
+    }
     if (customIntegrations && core) {
       registerSampleDatasetWithIntegration(customIntegrations, core);
     }