[Dataset quality] Use synthtrace for api integration tests (#171621)

x-pack/test/dataset_quality_api_integration/common/config.ts

@@ -13,6 +13,7 @@ import { createDatasetQualityUsers } from '@kbn/dataset-quality-plugin/server/te
 import { FtrConfigProviderContext } from '@kbn/test';
 import supertest from 'supertest';
 import { format, UrlObject } from 'url';
+import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
 import {
   FtrProviderContext,
   InheritedFtrProviderContext,
@@ -64,6 +65,9 @@ export interface CreateTest {
   services: InheritedServices & {
     datasetQualityFtrConfig: () => DatasetQualityFtrConfig;
     registry: ({ getService }: FtrProviderContext) => ReturnType<typeof RegistryProvider>;
+    logSynthtraceEsClient: (
+      context: InheritedFtrProviderContext
+    ) => Promise<LogsSynthtraceEsClient>;
     datasetQualityApiClient: (context: InheritedFtrProviderContext) => DatasetQualityApiClient;
   };
   junit: { reportName: string };
@@ -95,6 +99,12 @@ export function createTestConfig(
         ...services,
         datasetQualityFtrConfig: () => config,
         registry: RegistryProvider,
+        logSynthtraceEsClient: (context: InheritedFtrProviderContext) =>
+          new LogsSynthtraceEsClient({
+            client: context.getService('es'),
+            logger: createLogger(LogLevel.info),
+            refreshAfterIndex: true,
+          }),
         datasetQualityApiClient: async (_: InheritedFtrProviderContext) => {
           const { username, password } = servers.kibana;
           const esUrl = format(esServer);

x-pack/test/dataset_quality_api_integration/tests/data_streams.spec.ts

@@ -5,15 +5,19 @@
  * 2.0.
  */
 
+import { log, timerange } from '@kbn/apm-synthtrace-client';
 import expect from '@kbn/expect';
 import { DatasetQualityApiClientKey } from '../common/config';
 import { DatasetQualityApiError } from '../common/dataset_quality_api_supertest';
 import { FtrProviderContext } from '../common/ftr_provider_context';
 import { expectToReject } from '../utils';
+import { cleanLogIndexTemplate, addIntegrationToLogIndexTemplate } from './es_utils';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
   const registry = getService('registry');
+  const synthtrace = getService('logSynthtraceEsClient');
   const datasetQualityApiClient = getService('datasetQualityApiClient');
+  const es = getService('es');
 
   async function callApiAs(user: DatasetQualityApiClientKey) {
     return await datasetQualityApiClient[user]({
@@ -39,10 +43,67 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
 
     describe('when required privileges are set', () => {
-      it('returns true when user has logMonitoring privileges', async () => {
-        const privileges = await callApiAs('datasetQualityLogsUser');
+      describe('and uncategorized datastreams', () => {
+        const integration = 'my-custom-integration';
 
-        expect(privileges.body.items.length).to.be(0);
+        before(async () => {
+          await addIntegrationToLogIndexTemplate({ esClient: es, name: integration });
+
+          await synthtrace.index([
+            timerange('2023-11-20T15:00:00.000Z', '2023-11-20T15:01:00.000Z')
+              .interval('1m')
+              .rate(1)
+              .generator((timestamp) =>
+                log.create().message('This is a log message').timestamp(timestamp).defaults({
+                  'log.file.path': '/my-service.log',
+                })
+              ),
+          ]);
+        });
+
+        it('returns stats correctly', async () => {
+          const stats = await callApiAs('datasetQualityLogsUser');
+
+          expect(stats.body.items.length).to.be(1);
+          expect(stats.body.items[0].integration).to.be(integration);
+          expect(stats.body.items[0].size).not.empty();
+          expect(stats.body.items[0].size_bytes).greaterThan(0);
+          expect(stats.body.items[0].last_activity).greaterThan(0);
+        });
+
+        after(async () => {
+          await synthtrace.clean();
+          await cleanLogIndexTemplate({ esClient: es });
+        });
+      });
+
+      describe('and categorized datastreams', () => {
+        before(async () => {
+          await synthtrace.index([
+            timerange('2023-11-20T15:00:00.000Z', '2023-11-20T15:01:00.000Z')
+              .interval('1m')
+              .rate(1)
+              .generator((timestamp) =>
+                log.create().message('This is a log message').timestamp(timestamp).defaults({
+                  'log.file.path': '/my-service.log',
+                })
+              ),
+          ]);
+        });
+
+        it('returns stats correctly', async () => {
+          const stats = await callApiAs('datasetQualityLogsUser');
+
+          expect(stats.body.items.length).to.be(1);
+          expect(stats.body.items[0].integration).not.ok();
+          expect(stats.body.items[0].size).not.empty();
+          expect(stats.body.items[0].size_bytes).greaterThan(0);
+          expect(stats.body.items[0].last_activity).greaterThan(0);
+        });
+
+        after(async () => {
+          await synthtrace.clean();
+        });
       });
     });
   });

x-pack/test/dataset_quality_api_integration/tests/es_utils.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+export async function addIntegrationToLogIndexTemplate({
+  esClient,
+  name,
+  managedBy = 'fleet',
+}: {
+  esClient: Client;
+  name: string;
+  managedBy?: string;
+}) {
+  const { index_templates: indexTemplates } = await esClient.indices.getIndexTemplate({
+    name: 'logs',
+  });
+
+  await esClient.indices.putIndexTemplate({
+    name: 'logs',
+    body: {
+      ...indexTemplates[0].index_template,
+      _meta: {
+        ...indexTemplates[0].index_template._meta,
+        package: {
+          name,
+        },
+        managed_by: managedBy,
+      },
+    },
+  });
+}
+
+export async function cleanLogIndexTemplate({ esClient }: { esClient: Client }) {
+  const { index_templates: indexTemplates } = await esClient.indices.getIndexTemplate({
+    name: 'logs',
+  });
+
+  await esClient.indices.putIndexTemplate({
+    name: 'logs',
+    body: {
+      ...indexTemplates[0].index_template,
+      _meta: {
+        ...indexTemplates[0].index_template._meta,
+        package: undefined,
+        managed_by: undefined,
+      },
+    },
+  });
+}