[Infra] Create new formulas for rx and tx metrics (#189281)

Closes #188641

## Summary

This PR adds new formulas for rx and tx metrics for hosts. In inventory
we show the old metrics as legacy and the new ones with the old metrics
labels (this affects only hosts):

<img width="1788" alt="image"
src="https://github.com/user-attachments/assets/d3e5bf26-e521-4ff8-b00b-1d78eebd56f9">

All old alerts should work - The only difference is that it will show
the metric as "Legacy" and it still can be used in the rules. The hosts
view and the lens charts are using a new formula

## Testing
- Check the network metrics in the inventory / alert flyout (both the
new ones and the old ones)
- Check the network metrics and charts in the hosts view (only the new
ones should be available)


https://github.com/user-attachments/assets/886fd5a0-858c-458b-9025-eb55913b1932



https://github.com/user-attachments/assets/7752939f-f693-4021-bf23-89e264ef0c2d

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Carlos Crespo <crespocarlos@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/observability_solution/infra/common/formatters/snapshot_metric_formats.ts

@@ -35,6 +35,8 @@ export const METRIC_FORMATTERS: MetricFormatters = {
   },
   ['rx']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
   ['tx']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
+  ['rxV2']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
+  ['txV2']: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
   ['logRate']: {
     formatter: InfraFormatterType.abbreviatedNumber,
     template: '{{value}}/s',

x-pack/plugins/observability_solution/infra/common/http_api/infra/get_infra_metrics.ts

@@ -16,6 +16,8 @@ export const InfraMetricTypeRT = rt.keyof({
   memoryFree: null,
   rx: null,
   tx: null,
+  rxV2: null,
+  txV2: null,
 });
 
 export const RangeRT = rt.type({

x-pack/plugins/observability_solution/infra/common/inventory_models/intl_strings.ts

@@ -6,7 +6,11 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { SnapshotMetricType, SnapshotMetricTypeKeys } from '@kbn/metrics-data-access-plugin/common';
+import {
+  type InventoryItemType,
+  type SnapshotMetricType,
+  SnapshotMetricTypeKeys,
+} from '@kbn/metrics-data-access-plugin/common';
 import { toMetricOpt } from '../snapshot_metric_i18n';
 
 interface Lookup {
@@ -44,10 +48,11 @@ export const fieldToName = (field: string) => {
 };
 
 const snapshotTypeKeys = Object.keys(SnapshotMetricTypeKeys) as SnapshotMetricType[];
-export const SNAPSHOT_METRIC_TRANSLATIONS = snapshotTypeKeys.reduce((result, metric) => {
-  const text = toMetricOpt(metric)?.text;
-  if (text) {
-    result[metric] = text;
-  }
-  return result;
-}, {} as Record<SnapshotMetricType, string>);
+export const getSnapshotMetricTranslations = (nodeType: InventoryItemType) =>
+  snapshotTypeKeys.reduce((result, metric) => {
+    const text = toMetricOpt(metric, nodeType)?.text;
+    if (text) {
+      result[metric] = text;
+    }
+    return result;
+  }, {} as Record<SnapshotMetricType, string>);

x-pack/plugins/observability_solution/infra/common/snapshot_metric_i18n.ts

@@ -7,7 +7,7 @@
 
 import { i18n } from '@kbn/i18n';
 import { mapValues } from 'lodash';
-import { SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
+import type { InventoryItemType, SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
 
 // Lowercase versions of all metrics, for when they need to be used in the middle of a sentence;
 // these may need to be translated differently depending on language, e.g. still capitalizing "CPU"
@@ -28,6 +28,14 @@ const TranslationsLowercase = {
     defaultMessage: 'outbound traffic',
   }),
 
+  InboundTrafficLegacy: i18n.translate('xpack.infra.waffle.metricOptions.inboundTrafficText', {
+    defaultMessage: 'inbound traffic (Legacy)',
+  }),
+
+  OutboundTrafficLegacy: i18n.translate('xpack.infra.waffle.metricOptions.outboundTrafficText', {
+    defaultMessage: 'outbound traffic (Legacy)',
+  }),
+
   LogRate: i18n.translate('xpack.infra.waffle.metricOptions.hostLogRateText', {
     defaultMessage: 'log rate',
   }),
@@ -94,8 +102,11 @@ const Translations = mapValues(
   (translation) => `${translation[0].toUpperCase()}${translation.slice(1)}`
 );
 
+const showLegacyLabel = (nodeType?: InventoryItemType) => nodeType === 'host';
+
 export const toMetricOpt = (
-  metric: SnapshotMetricType
+  metric: SnapshotMetricType,
+  nodeType?: InventoryItemType
 ): { text: string; textLC: string; value: SnapshotMetricType } | undefined => {
   switch (metric) {
     case 'cpu':
@@ -112,15 +123,35 @@ export const toMetricOpt = (
       };
     case 'rx':
       return {
-        text: Translations.InboundTraffic,
-        textLC: TranslationsLowercase.InboundTraffic,
+        text: showLegacyLabel(nodeType)
+          ? Translations.InboundTrafficLegacy
+          : Translations.InboundTraffic,
+        textLC: showLegacyLabel(nodeType)
+          ? TranslationsLowercase.InboundTrafficLegacy
+          : TranslationsLowercase.InboundTraffic,
         value: 'rx',
       };
     case 'tx':
+      return {
+        text: showLegacyLabel(nodeType)
+          ? Translations.OutboundTrafficLegacy
+          : Translations.OutboundTraffic,
+        textLC: showLegacyLabel(nodeType)
+          ? TranslationsLowercase.OutboundTrafficLegacy
+          : TranslationsLowercase.OutboundTraffic,
+        value: 'tx',
+      };
+    case 'rxV2':
+      return {
+        text: Translations.InboundTraffic,
+        textLC: TranslationsLowercase.InboundTraffic,
+        value: 'rxV2',
+      };
+    case 'txV2':
       return {
         text: Translations.OutboundTraffic,
         textLC: TranslationsLowercase.OutboundTraffic,
-        value: 'tx',
+        value: 'txV2',
       };
     case 'logRate':
       return {

x-pack/plugins/observability_solution/infra/public/alerting/inventory/components/expression.tsx

@@ -577,7 +577,7 @@ export const ExpressionRow: FC<PropsWithChildren<ExpressionRowProps>> = (props)
         myMetrics = containerSnapshotMetricTypes;
         break;
     }
-    return myMetrics.map(toMetricOpt);
+    return myMetrics.map((myMetric) => toMetricOpt(myMetric, props.nodeType));
   }, [props.nodeType]);
 
   return (
@@ -775,6 +775,8 @@ const metricUnit: Record<string, { label: string }> = {
   memory: { label: '%' },
   rx: { label: 'bits/s' },
   tx: { label: 'bits/s' },
+  rxV2: { label: 'bits/s' },
+  txV2: { label: 'bits/s' },
   logRate: { label: '/s' },
   diskIOReadBytes: { label: 'bytes/s' },
   diskIOWriteBytes: { label: 'bytes/s' },

x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/timeline/timeline.tsx

@@ -113,8 +113,8 @@ export const Timeline: React.FC<Props> = ({ interval, yAxisFormatter, isVisible
     }
   }, [nodeType, metricsHostsAnomalies, metricsK8sAnomalies]);
 
-  const metricLabel = toMetricOpt(metric.type)?.textLC;
-  const metricPopoverLabel = toMetricOpt(metric.type)?.text;
+  const metricLabel = toMetricOpt(metric.type, nodeType)?.textLC;
+  const metricPopoverLabel = toMetricOpt(metric.type, nodeType)?.text;
 
   const chartMetric = {
     color: Color.color0,

x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/toolbars/metrics_and_groupby_toolbar_items.tsx

@@ -23,8 +23,10 @@ interface Props extends ToolbarProps {
 export const MetricsAndGroupByToolbarItems = (props: Props) => {
   const metricOptions = useMemo(
     () =>
-      props.metricTypes.map(toMetricOpt).filter((v) => v) as Array<{ text: string; value: string }>,
-    [props.metricTypes]
+      props.metricTypes
+        .map((metric) => toMetricOpt(metric, props.nodeType))
+        .filter((v) => v) as Array<{ text: string; value: string }>,
+    [props.metricTypes, props.nodeType]
   );
 
   const groupByOptions = useMemo(

x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.test.tsx

@@ -47,8 +47,8 @@ describe('ConditionalToolTip', () => {
           metrics: [
             { name: 'cpu', value: 0.1, avg: 0.4, max: 0.7 },
             { name: 'memory', value: 0.8, avg: 0.8, max: 1 },
-            { name: 'tx', value: 1000000, avg: 1000000, max: 1000000 },
-            { name: 'rx', value: 1000000, avg: 1000000, max: 1000000 },
+            { name: 'txV2', value: 1000000, avg: 1000000, max: 1000000 },
+            { name: 'rxV2', value: 1000000, avg: 1000000, max: 1000000 },
             {
               name: 'cedd6ca0-5775-11eb-a86f-adb714b6c486',
               max: 0.34164999922116596,
@@ -80,8 +80,8 @@ describe('ConditionalToolTip', () => {
     const expectedMetrics = [
       { type: 'cpu' },
       { type: 'memory' },
-      { type: 'tx' },
-      { type: 'rx' },
+      { type: 'txV2' },
+      { type: 'rxV2' },
       {
         aggregation: 'avg',
         field: 'host.cpu.pct',

x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/components/waffle/conditional_tooltip.tsx

@@ -20,7 +20,7 @@ import { useSourceContext } from '../../../../../containers/metrics_source';
 import { InfraWaffleMapNode } from '../../../../../common/inventory/types';
 import { useSnapshot } from '../../hooks/use_snaphot';
 import { createInventoryMetricFormatter } from '../../lib/create_inventory_metric_formatter';
-import { SNAPSHOT_METRIC_TRANSLATIONS } from '../../../../../../common/inventory_models/intl_strings';
+import { getSnapshotMetricTranslations } from '../../../../../../common/inventory_models/intl_strings';
 import { useWaffleOptionsContext } from '../../hooks/use_waffle_options';
 import { createFormatterForMetric } from '../../../metrics_explorer/components/helpers/create_formatter_for_metric';
 
@@ -86,7 +86,7 @@ export const ConditionalToolTip = ({ node, nodeType, currentTime }: Props) => {
       ) : (
         metrics.map((metric) => {
           const metricName = SnapshotMetricTypeRT.is(metric.name) ? metric.name : 'custom';
-          const name = SNAPSHOT_METRIC_TRANSLATIONS[metricName] || metricName;
+          const name = getSnapshotMetricTranslations(nodeType)[metricName] || metricName;
           // if custom metric, find field and label from waffleOptionsContext result
           // because useSnapshot does not return it
           const customMetric =

x-pack/plugins/observability_solution/infra/public/pages/metrics/inventory_view/lib/create_inventory_metric_formatter.ts

@@ -65,6 +65,8 @@ const METRIC_FORMATTERS: MetricFormatters = {
   },
   rx: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
   tx: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
+  rxV2: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
+  txV2: { formatter: InfraFormatterType.bits, template: '{{value}}/s' },
   logRate: {
     formatter: InfraFormatterType.abbreviatedNumber,
     template: '{{value}}/s',

x-pack/plugins/observability_solution/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts

@@ -22,7 +22,7 @@ import {
 } from '@kbn/alerting-plugin/common';
 import { AlertsClientError, RuleExecutorOptions, RuleTypeState } from '@kbn/alerting-plugin/server';
 import { convertToBuiltInComparators, getAlertUrl } from '@kbn/observability-plugin/common';
-import { SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
+import type { InventoryItemType, SnapshotMetricType } from '@kbn/metrics-data-access-plugin/common';
 import { ObservabilityMetricsAlert } from '@kbn/alerts-as-data-utils';
 import { getOriginalActionGroup } from '../../../utils/get_original_action_group';
 import {
@@ -226,12 +226,13 @@ export const createInventoryMetricThresholdExecutor =
       if (nextState === AlertStates.ALERT || nextState === AlertStates.WARNING) {
         reason = results
           .map((result) =>
-            buildReasonWithVerboseMetricName(
+            buildReasonWithVerboseMetricName({
               group,
-              result[group],
-              buildFiredAlertReason,
-              nextState === AlertStates.WARNING
-            )
+              resultItem: result[group],
+              buildReason: buildFiredAlertReason,
+              useWarningThreshold: nextState === AlertStates.WARNING,
+              nodeType,
+            })
           )
           .join('\n');
       }
@@ -240,14 +241,24 @@ export const createInventoryMetricThresholdExecutor =
           reason = results
             .filter((result) => result[group].isNoData)
             .map((result) =>
-              buildReasonWithVerboseMetricName(group, result[group], buildNoDataAlertReason)
+              buildReasonWithVerboseMetricName({
+                group,
+                resultItem: result[group],
+                buildReason: buildNoDataAlertReason,
+                nodeType,
+              })
             )
             .join('\n');
         } else if (nextState === AlertStates.ERROR) {
           reason = results
             .filter((result) => result[group].isError)
             .map((result) =>
-              buildReasonWithVerboseMetricName(group, result[group], buildErrorAlertReason)
+              buildReasonWithVerboseMetricName({
+                group,
+                resultItem: result[group],
+                buildReason: buildErrorAlertReason,
+                nodeType,
+              })
             )
             .join('\n');
         }
@@ -384,12 +395,19 @@ const formatThreshold = (metric: SnapshotMetricType, value: number | number[]) =
   return threshold;
 };
 
-const buildReasonWithVerboseMetricName = (
-  group: string,
-  resultItem: ConditionResult,
-  buildReason: (r: any) => string,
-  useWarningThreshold?: boolean
-) => {
+const buildReasonWithVerboseMetricName = ({
+  group,
+  resultItem,
+  buildReason,
+  useWarningThreshold,
+  nodeType,
+}: {
+  group: string;
+  resultItem: ConditionResult;
+  buildReason: (r: any) => string;
+  useWarningThreshold?: boolean;
+  nodeType?: InventoryItemType;
+}) => {
   if (!resultItem) return '';
 
   const thresholdToFormat = useWarningThreshold
@@ -399,7 +417,7 @@ const buildReasonWithVerboseMetricName = (
     ...resultItem,
     group,
     metric:
-      toMetricOpt(resultItem.metric)?.text ||
+      toMetricOpt(resultItem.metric, nodeType)?.text ||
       (resultItem.metric === 'custom' && resultItem.customMetric
         ? getCustomMetricLabel(resultItem.customMetric)
         : resultItem.metric),

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/host/index.ts

@@ -54,5 +54,5 @@ export const host: InventoryModel<typeof metrics> = {
     ...awsRequiredMetrics,
     ...nginxRequireMetrics,
   ],
-  tooltipMetrics: ['cpu', 'memory', 'tx', 'rx'],
+  tooltipMetrics: ['cpu', 'memory', 'txV2', 'rxV2'],
 };

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/host/metrics/formulas/network.ts

@@ -12,20 +12,16 @@ export const rx: LensBaseLayer = {
   label: i18n.translate('xpack.metricsData.assetDetails.formulas.rx', {
     defaultMessage: 'Network Inbound (RX)',
   }),
-  value:
-    "average(host.network.ingress.bytes) * 8 / (max(metricset.period, kql='host.network.ingress.bytes: *') / 1000)",
+  value: 'sum(host.network.ingress.bytes) * 8',
   format: 'bits',
   decimals: 1,
-  normalizeByUnit: 's',
 };
 
 export const tx: LensBaseLayer = {
   label: i18n.translate('xpack.metricsData.assetDetails.formulas.tx', {
     defaultMessage: 'Network Outbound (TX)',
   }),
-  value:
-    "average(host.network.egress.bytes) * 8 / (max(metricset.period, kql='host.network.egress.bytes: *') / 1000)",
+  value: 'sum(host.network.egress.bytes) * 8',
   format: 'bits',
   decimals: 1,
-  normalizeByUnit: 's',
 };

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/host/metrics/snapshot/index.ts

@@ -17,6 +17,8 @@ import { memoryTotal } from './memory_total';
 import { normalizedLoad1m } from './normalized_load_1m';
 import { rx } from './rx';
 import { tx } from './tx';
+import { txV2 } from './tx_v2';
+import { rxV2 } from './rx_v2';
 
 export const snapshot = {
   cpu,
@@ -31,4 +33,6 @@ export const snapshot = {
   normalizedLoad1m,
   rx,
   tx,
+  rxV2,
+  txV2,
 };

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/host/metrics/snapshot/rx_v2.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetricsUIAggregation } from '../../../types';
+export const rxV2: MetricsUIAggregation = {
+  rx_sum: {
+    sum: {
+      field: 'host.network.ingress.bytes',
+    },
+  },
+  min_timestamp: {
+    min: {
+      field: '@timestamp',
+    },
+  },
+  max_timestamp: {
+    max: {
+      field: '@timestamp',
+    },
+  },
+  rxV2: {
+    bucket_script: {
+      buckets_path: {
+        value: 'rx_sum',
+        minTime: 'min_timestamp',
+        maxTime: 'max_timestamp',
+      },
+      script: {
+        source: 'params.value / ((params.maxTime - params.minTime) / 1000)',
+        lang: 'painless',
+      },
+      gap_policy: 'skip',
+    },
+  },
+};

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/host/metrics/snapshot/tx_v2.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MetricsUIAggregation } from '../../../types';
+export const txV2: MetricsUIAggregation = {
+  tx_sum: {
+    sum: {
+      field: 'host.network.egress.bytes',
+    },
+  },
+  min_timestamp: {
+    min: {
+      field: '@timestamp',
+    },
+  },
+  max_timestamp: {
+    max: {
+      field: '@timestamp',
+    },
+  },
+  txV2: {
+    bucket_script: {
+      buckets_path: {
+        value: 'tx_sum',
+        minTime: 'min_timestamp',
+        maxTime: 'max_timestamp',
+      },
+      script: {
+        source: 'params.value / ((params.maxTime - params.minTime) / 1000)',
+        lang: 'painless',
+      },
+      gap_policy: 'skip',
+    },
+  },
+};

x-pack/plugins/observability_solution/metrics_data_access/common/inventory_models/types.ts

@@ -358,6 +358,8 @@ export const SnapshotMetricTypeKeys = {
   normalizedLoad1m: null,
   tx: null,
   rx: null,
+  txV2: null,
+  rxV2: null,
   logRate: null,
   diskIOReadBytes: null,
   diskIOWriteBytes: null,
@@ -385,7 +387,7 @@ export interface InventoryMetrics {
   tsvb: { [name: string]: TSVBMetricModelCreator };
   snapshot: { [name: string]: MetricsUIAggregation | undefined };
   defaultSnapshot: SnapshotMetricType;
-  /** This is used by the inventory view to calculate the appropriate amount of time for the metrics detail page. Some metris like awsS3 require multiple days where others like host only need an hour.*/
+  /** This is used by the inventory view to calculate the appropriate amount of time for the metrics detail page. Some metrics like awsS3 require multiple days where others like host only need an hour.*/
   defaultTimeRangeInSeconds: number;
 }
 

x-pack/test/api_integration/apis/metrics_ui/infra.ts

@@ -246,7 +246,7 @@ export default function ({ getService }: FtrProviderContext) {
         const response = await makeRequest({ invalidBody, expectedHTTPCode: 400 });
 
         expect(normalizeNewLine(response.body.message)).to.be(
-          '[request body]: Failed to validate: in metrics/0/type: "any" does not match expected type "cpu" | "normalizedLoad1m" | "diskSpaceUsage" | "memory" | "memoryFree" | "rx" | "tx"'
+          '[request body]: Failed to validate: in metrics/0/type: "any" does not match expected type "cpu" | "normalizedLoad1m" | "diskSpaceUsage" | "memory" | "memoryFree" | "rx" | "tx" | "rxV2" | "txV2"'
         );
       });
 

x-pack/test/functional/apps/infra/home_page.ts

@@ -573,8 +573,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
           'system.core.steal.pct',
           'system.cpu.nice.pct',
           'system.cpu.idle.pct',
-          'system.cpu.iowait.pct',
-          'system.cpu.irq.pct',
         ];
 
         for (const field of fields) {

x-pack/test/functional/apps/infra/hosts_view.ts

@@ -560,6 +560,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
         it('should have an option to open the chart in lens', async () => {
           await retry.try(async () => {
             await pageObjects.infraHostsView.clickAndValidateMetricChartActionOptions();
+            await browser.pressKeys(browser.keys.ESCAPE);
           });
         });
       });

x-pack/test/functional/page_objects/infra_hosts_view.ts

@@ -129,8 +129,6 @@ export function InfraHostsViewProvider({ getService }: FtrProviderContext) {
       const button = await element.findByTestSubject('embeddablePanelToggleMenuIcon');
       await button.click();
       await testSubjects.existOrFail('embeddablePanelAction-openInLens');
-      // forces the modal to close
-      await element.click();
     },
 
     // KPIs