[Synthetics] Fix settings data retention update (#148437)

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Fixes https://github.com/elastic/kibana/issues/146343

.gitignore

@@ -113,3 +113,4 @@ fleet-server.yml
 /packages/kbn-package-map/package-map.json
 /packages/kbn-synthetic-package-map/
 **/.synthetics/
+**/.journeys/

x-pack/plugins/observability/e2e/synthetics_runner.ts

@@ -12,6 +12,7 @@ import { run as syntheticsRun } from '@elastic/synthetics';
 import { PromiseType } from 'utility-types';
 import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
 
+import { EsArchiver } from '@kbn/es-archiver';
 import { esArchiverUnload } from './tasks/es_archiver';
 import { TestReporter } from './test_reporter';
 
@@ -59,9 +60,17 @@ export class SyntheticsRunner {
     try {
       console.log('Loading esArchiver...');
 
-      const esArchiver = this.getService('esArchiver');
+      const esArchiver: EsArchiver = this.getService('esArchiver');
 
-      const promises = dataArchives.map((archive) => esArchiver.loadIfNeeded(e2eDir + archive));
+      const promises = dataArchives.map((archive) => {
+        if (archive === 'synthetics_data') {
+          return esArchiver.load(e2eDir + archive, {
+            docsOnly: true,
+            skipExisting: true,
+          });
+        }
+        return esArchiver.load(e2eDir + archive, { skipExisting: true });
+      });
 
       await Promise.all([...promises]);
     } catch (e) {

x-pack/plugins/observability/e2e/test_reporter.ts

@@ -135,7 +135,12 @@ export class TestReporter implements Reporter {
     );
 
     successfulJourneys.forEach(([journeyName, steps]) => {
-      // fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
+      try {
+        fs.unlinkSync('.journeys/videos/' + journeyName + '.webm');
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.log(e);
+      }
     });
 
     const { failed, succeeded, skipped } = this.metrics;

x-pack/plugins/observability/e2e/tsconfig.json

@@ -9,5 +9,6 @@
   "kbn_references": [
     "@kbn/apm-plugin",
     "@kbn/test",
+    "@kbn/es-archiver",
   ]
 }

x-pack/plugins/synthetics/e2e/helpers/record_video.ts

@@ -14,12 +14,15 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
 // @ts-ignore
 export const runner: Runner = global[SYNTHETICS_RUNNER];
 
-export const recordVideo = (page: Page) => {
+export const recordVideo = (page: Page, postfix = '') => {
   after(async () => {
     try {
       const videoFilePath = await page.video()?.path();
       const pathToVideo = videoFilePath?.replace('.journeys/videos/', '').replace('.webm', '');
-      const newVideoPath = videoFilePath?.replace(pathToVideo!, runner.currentJourney!.name);
+      const newVideoPath = videoFilePath?.replace(
+        pathToVideo!,
+        runner.currentJourney!.name + `-${postfix}`
+      );
       fs.renameSync(videoFilePath!, newVideoPath!);
     } catch (e) {
       // eslint-disable-next-line no-console

x-pack/plugins/synthetics/e2e/journeys/synthetics/alert_rules/default_status_alert.journey.ts

@@ -111,9 +111,13 @@ journey(`DefaultStatusAlert`, async ({ page, params }) => {
     });
     await retry.tryForTime(2 * 60 * 1000, async () => {
       await page.click(byTestId('querySubmitButton'));
-      const text = await page.textContent(`${byTestId('dataGridRowCell')} .euiLink`, {
-        timeout: 5 * 1000,
-      });
+      expect(
+        await page.isVisible(`text=1 Alert`, {
+          timeout: 10 * 1000,
+        })
+      ).toBe(true);
+
+      const text = await page.textContent(`${byTestId('dataGridRowCell')} .euiLink`);
 
       expect(text).toBe(reasonMessage);
       expect(await page.isVisible(`text=1 Alert`)).toBe(true);

x-pack/plugins/synthetics/e2e/journeys/synthetics/data_retention.journey.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { journey, step, expect, Page } from '@elastic/synthetics';
+import { assertText, byTestId } from '@kbn/observability-plugin/e2e/utils';
+import { RetryService } from '@kbn/ftr-common-functional-services';
+import { recordVideo } from '../../helpers/record_video';
+import { syntheticsAppPageProvider } from '../../page_objects/synthetics/synthetics_app';
+
+let page1: Page;
+journey(`DataRetentionPage`, async ({ page, params }) => {
+  page.setDefaultTimeout(60 * 1000);
+  recordVideo(page);
+  const syntheticsApp = syntheticsAppPageProvider({ page, kibanaUrl: params.kibanaUrl });
+
+  const getService = params.getService;
+  const retry: RetryService = getService('retry');
+
+  step('Go to monitor-management', async () => {
+    await syntheticsApp.navigateToMonitorManagement(true);
+  });
+
+  step('validate data retention tab', async () => {
+    await page.click('text=Settings');
+    expect(page.url()).toBe('http://localhost:5620/app/synthetics/settings/alerting');
+    await page.click('text=Data retention');
+    expect(page.url()).toBe('http://localhost:5620/app/synthetics/settings/data-retention');
+    await page.click('text=Synthetics data is configured by managed index lifecycle policies');
+
+    await page.click('text=0 Bytes');
+    await page.click('text=365 days + rollover');
+    await page.click('text=14 days + rollover');
+    await page.click(':nth-match(:text("14 days + rollover"), 2)');
+    await page.click(':nth-match(:text("365 days + rollover"), 2)');
+    await page.click(':nth-match(:text("365 days + rollover"), 3)');
+    await page.click(':nth-match(:text("365 days + rollover"), 4)');
+    await page.click('tbody div:has-text("synthetics(opens in a new tab or window)")');
+  });
+
+  step('validate data sizes', async () => {
+    const allChecks = await page.textContent(`tr:has-text("All Checks") span:has-text("KB")`);
+    const browserChecks = await page.textContent(
+      `tr:has-text("Browser Checks") span:has-text("KB")`
+    );
+    const networkChecks = await page.textContent(
+      `tr:has-text("Browser Network Requests") span:has-text("KB")`
+    );
+    const screenshotChecks = await page.textContent(
+      `tr:has-text("Browser Screenshots") span:has-text("KB")`
+    );
+    expect(Number(allChecks?.split('KB')[0])).toBeGreaterThan(450);
+    expect(Number(browserChecks?.split('KB')[0])).toBeGreaterThan(50);
+    expect(Number(networkChecks?.split('KB')[0])).toBeGreaterThan(300);
+    expect(Number(screenshotChecks?.split('KB')[0])).toBeGreaterThan(25);
+  });
+
+  step('it can click through for policy', async () => {
+    [page1] = await Promise.all([
+      page.waitForEvent('popup'),
+      page.click(
+        'tbody div:has-text("synthetics-synthetics.browser-default_policy(opens in a new tab or window)")'
+      ),
+    ]);
+    recordVideo(page1, 'data_retention_policy_change');
+    await page1.setDefaultTimeout(60 * 1000);
+    await page1.click('text=Edit policy synthetics-synthetics.browser-default_policy');
+    await page1.click('text=Save as new policy');
+  });
+
+  step('newly created policy is reflected in settings', async () => {
+    await page1.fill(
+      byTestId('policyNameField'),
+      'synthetics-synthetics.browser-default_policy-copy'
+    );
+    await page1.fill(byTestId('delete-selectedMinimumAge'), '10000');
+
+    await page1.click(byTestId('savePolicyButton'));
+
+    await page1.click('text=Include managed system policies');
+
+    await page1.fill(byTestId('ilmSearchBar'), 'copy');
+
+    await retry.tryForTime(30 * 1000, async () => {
+      await page1.click(byTestId('addPolicyToTemplate'), { clickCount: 2, timeout: 5000 });
+      expect(await page1.isVisible(byTestId('confirmModalConfirmButton'), { timeout: 5000 })).toBe(
+        true
+      );
+    });
+
+    await page1.click(byTestId('comboBoxToggleListButton'));
+    await page1.fill(byTestId('comboBoxSearchInput'), 'synthetics-browser');
+
+    await page1.click('button[title="synthetics-browser"]');
+
+    await page1.click(byTestId('confirmModalConfirmButton'));
+
+    await page.reload();
+
+    await page.click('tbody div:has-text("synthetics(opens in a new tab or window)")');
+    await page1.close();
+
+    await assertText({ page, text: '10000 days + rollover' });
+  });
+});

x-pack/plugins/synthetics/e2e/journeys/synthetics/index.ts

@@ -18,3 +18,4 @@ export * from './alerting_default.journey';
 export * from './global_parameters.journey';
 export * from './detail_flyout';
 export * from './alert_rules/default_status_alert.journey';
+export * from './data_retention.journey';

x-pack/plugins/synthetics/e2e/journeys/uptime/alerts/status_alert_flyouts_in_alerting_app.ts

@@ -62,8 +62,10 @@ journey('StatusFlyoutInAlertingApp', async ({ page, params }) => {
     await assertText({ page, text: 'browser' });
     await assertText({ page, text: 'http' });
 
-    const suggestionItem = await page.$(byTestId('autoCompleteSuggestionText'));
-    expect(await suggestionItem?.textContent()).toBe('"browser" ');
+    await retry.tryForTime(30 * 1000, async () => {
+      const suggestionItem = await page.$(byTestId('autoCompleteSuggestionText'));
+      expect(await suggestionItem?.textContent()).toBe('"browser" ');
+    });
 
     await page.click(byTestId('euiFlyoutCloseButton'));
     await page.click(byTestId('confirmModalConfirmButton'));

x-pack/plugins/synthetics/e2e/synthetics_run.ts

@@ -25,7 +25,11 @@ async function runE2ETests({ readConfigFile }: FtrConfigProviderContext) {
       await syntheticsRunner.setup();
       const fixturesDir = path.join(__dirname, '../e2e/fixtures/es_archiver/');
 
-      await syntheticsRunner.loadTestData(fixturesDir, ['full_heartbeat', 'browser']);
+      await syntheticsRunner.loadTestData(fixturesDir, [
+        'synthetics_data',
+        'full_heartbeat',
+        'browser',
+      ]);
 
       await syntheticsRunner.loadTestFiles(async () => {
         require('./journeys');

x-pack/plugins/synthetics/public/apps/synthetics/components/settings/data_retention.tsx

@@ -35,7 +35,7 @@ export const DataRetentionTab = () => {
       }),
     },
     {
-      field: 'name',
+      field: 'policy.name',
       name: i18n.translate('xpack.synthetics.settingsRoute.table.policy', {
         defaultMessage: 'Policy',
       }),

x-pack/plugins/synthetics/public/apps/synthetics/components/settings/hooks/use_get_ilm_policies.ts

@@ -16,23 +16,42 @@ const policyLabels = [
     label: i18n.translate('xpack.synthetics.settingsRoute.allChecks', {
       defaultMessage: 'All Checks',
     }),
+    indexTemplate: 'synthetics',
   },
   {
     name: 'synthetics-synthetics.browser-default_policy',
     label: i18n.translate('xpack.synthetics.settingsRoute.browserChecks', {
       defaultMessage: 'Browser Checks',
     }),
+    indexTemplate: 'synthetics-browser',
   },
   {
     name: 'synthetics-synthetics.browser_network-default_policy',
     label: i18n.translate('xpack.synthetics.settingsRoute.browserNetworkRequests', {
       defaultMessage: 'Browser Network Requests',
     }),
+    indexTemplate: 'synthetics-browser.network',
+  },
+  {
+    name: 'synthetics-synthetics.browser_screenshot-default_policy',
+    label: 'Browser Screenshots',
+    indexTemplate: 'synthetics-browser.screenshot',
+  },
+  {
+    name: 'synthetics-synthetics.http-default_policy',
+    label: 'HTTP Pings',
+    indexTemplate: 'synthetics-http',
+  },
+  {
+    name: 'synthetics-synthetics.icmp-default_policy',
+    label: 'ICMP Pings',
+    indexTemplate: 'synthetics-icmp',
+  },
+  {
+    name: 'synthetics-synthetics.tcp-default_policy',
+    label: 'TCP Pings',
+    indexTemplate: 'synthetics-tcp',
   },
-  { name: 'synthetics-synthetics.browser_screenshot-default_policy', label: 'Browser Screenshots' },
-  { name: 'synthetics-synthetics.http-default_policy', label: 'HTTP Pings' },
-  { name: 'synthetics-synthetics.icmp-default_policy', label: 'ICMP Pings' },
-  { name: 'synthetics-synthetics.tcp-default_policy', label: 'TCP Pings' },
 ];
 
 export const useGetIlmPolicies = () => {
@@ -44,11 +63,16 @@ export const useGetIlmPolicies = () => {
     return getIndicesData();
   }, []);
 
-  const syntheticsILMPolicies = data?.filter(({ name }) => name.includes('synthetics')) ?? [];
+  const syntheticsILMPolicies =
+    data?.filter(({ indexTemplates }) =>
+      indexTemplates?.some((indTemp) => indTemp.includes('synthetics'))
+    ) ?? [];
 
   return {
-    data: policyLabels.map(({ name, label }) => {
-      const policy = syntheticsILMPolicies.find((p) => p.name === name);
+    data: policyLabels.map(({ name, label, indexTemplate }) => {
+      const policy = syntheticsILMPolicies.find((p) =>
+        p.indexTemplates?.some((indTemp) => indTemp.includes(indexTemplate))
+      );
       const policyIndices = sizeData?.data?.filter((d) => policy?.indices?.includes(d.index!));
 
       let totalSize =

x-pack/plugins/synthetics/public/apps/synthetics/components/settings/policy_link.tsx

@@ -22,14 +22,18 @@ export const PolicyLink = ({ name }: { name: string }) => {
 
   const { data } = useFetcher(async () => {
     return ilmLocator?.getLocation({ page: 'policy_edit', policyName: name });
-  }, []);
+  }, [name]);
 
   if (!data) {
     return <EuiLoadingContent lines={1} />;
   }
 
   return (
-    <EuiLink href={`${basePath}/app/${data.app}${data.path}`} target="_blank">
+    <EuiLink
+      href={`${basePath}/app/${data.app}${data.path}`}
+      target="_blank"
+      data-test-subj={name + 'PolicyLink'}
+    >
       {name}
     </EuiLink>
   );