[8.6] [Fleet] Bugfix: count agent policies not agent policy revisions in telemetry (#149858) (#150036)

# Backport

This will backport the following commits from `main` to `8.6`:
- [[Fleet] Bugfix: count agent policies not agent policy revisions in
telemetry (#149858)](https://github.com/elastic/kibana/pull/149858)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mark
Hopkin","email":"mark.hopkin@elastic.co"},"sourceCommit":{"committedDate":"2023-01-31T09:51:23Z","message":"[Fleet]
Bugfix: count agent policies not agent policy revisions in telemetry
(#149858)\n\n## Summary\r\n\r\nPreviously we were counting all agent
policy revisions + even if the\r\nagent policy had been deleted. I have
moved to using the saved object\r\nclient to get the agent
policies.","sha":"7006c7a02d7ec213b1e4cb16c2711d05159a864a","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:Fleet","backport:prev-minor","v8.7.0"],"number":149858,"url":"https://github.com/elastic/kibana/pull/149858","mergeCommit":{"message":"[Fleet]
Bugfix: count agent policies not agent policy revisions in telemetry
(#149858)\n\n## Summary\r\n\r\nPreviously we were counting all agent
policy revisions + even if the\r\nagent policy had been deleted. I have
moved to using the saved object\r\nclient to get the agent
policies.","sha":"7006c7a02d7ec213b1e4cb16c2711d05159a864a"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/149858","number":149858,"mergeCommit":{"message":"[Fleet]
Bugfix: count agent policies not agent policy revisions in telemetry
(#149858)\n\n## Summary\r\n\r\nPreviously we were counting all agent
policy revisions + even if the\r\nagent policy had been deleted. I have
moved to using the saved object\r\nclient to get the agent
policies.","sha":"7006c7a02d7ec213b1e4cb16c2711d05159a864a"}}]}]
BACKPORT-->

x-pack/plugins/fleet/server/collectors/agent_policies.ts

@@ -5,57 +5,55 @@
  * 2.0.
  */
 
-import type { ElasticsearchClient } from '@kbn/core/server';
+import type { SavedObjectsClientContract } from '@kbn/core/server';
+import _ from 'lodash';
 
-import { AGENT_POLICY_INDEX } from '../../common';
+import {
-import { ES_SEARCH_LIMIT } from '../../common/constants';
+  AGENT_POLICY_SAVED_OBJECT_TYPE,
-import { appContextService } from '../services';
+  OUTPUT_SAVED_OBJECT_TYPE,
+  SO_SEARCH_LIMIT,
+} from '../../common';
+import type { OutputSOAttributes, AgentPolicy } from '../types';
 
 export interface AgentPoliciesUsage {
   count: number;
   output_types: string[];
 }
 
-const DEFAULT_AGENT_POLICIES_USAGE = {
-  count: 0,
-  output_types: [],
-};
-
 export const getAgentPoliciesUsage = async (
-  esClient: ElasticsearchClient,
+  soClient: SavedObjectsClientContract
-  abortController: AbortController
 ): Promise<AgentPoliciesUsage> => {
-  try {
+  const { saved_objects: outputs } = await soClient.find<OutputSOAttributes>({
-    const res = await esClient.search(
+    type: OUTPUT_SAVED_OBJECT_TYPE,
-      {
+    page: 1,
-        index: AGENT_POLICY_INDEX,
+    perPage: SO_SEARCH_LIMIT,
-        size: ES_SEARCH_LIMIT,
+  });
-        track_total_hits: true,
-        rest_total_hits_as_int: true,
-      },
-      { signal: abortController.signal }
-    );
 
-    const agentPolicies = res.hits.hits;
+  const defaultOutputId = outputs.find((output) => output.attributes.is_default)?.id || '';
 
-    const outputTypes = new Set<string>();
+  const outputsById = _.keyBy(outputs, 'id');
-    agentPolicies.forEach((item) => {
+
-      const source = (item._source as any) ?? {};
+  const { saved_objects: agentPolicies, total: totalAgentPolicies } =
-      Object.keys(source.data.outputs).forEach((output) => {
+    await soClient.find<AgentPolicy>({
-        outputTypes.add(source.data.outputs[output].type);
+      type: AGENT_POLICY_SAVED_OBJECT_TYPE,
-      });
+      page: 1,
+      perPage: SO_SEARCH_LIMIT,
     });
 
-    return {
+  const uniqueOutputIds = new Set<string>();
-      count: res.hits.total as number,
+  agentPolicies.forEach((agentPolicy) => {
-      output_types: Array.from(outputTypes),
+    uniqueOutputIds.add(agentPolicy.attributes.monitoring_output_id || defaultOutputId);
-    };
+    uniqueOutputIds.add(agentPolicy.attributes.data_output_id || defaultOutputId);
-  } catch (error) {
+  });
-    if (error.statusCode === 404) {
+
-      appContextService.getLogger().debug('Index .fleet-policies does not exist yet.');
+  const uniqueOutputTypes = new Set(
-    } else {
+    Array.from(uniqueOutputIds).map((outputId) => {
-      throw error;
+      return outputsById[outputId]?.attributes.type;
-    }
+    })
-    return DEFAULT_AGENT_POLICIES_USAGE;
+  );
-  }
+
+  return {
+    count: totalAgentPolicies,
+    output_types: Array.from(uniqueOutputTypes),
+  };
 };

x-pack/plugins/fleet/server/collectors/register.ts

@@ -62,7 +62,7 @@ export const fetchFleetUsage = async (
     packages: await getPackageUsage(soClient),
     ...(await getAgentData(esClient, abortController)),
     fleet_server_config: await getFleetServerConfig(soClient),
-    agent_policies: await getAgentPoliciesUsage(esClient, abortController),
+    agent_policies: await getAgentPoliciesUsage(soClient),
     ...(await getPanicLogsLastHour(esClient)),
     // TODO removed top errors telemetry as it causes this issue: https://github.com/elastic/kibana/issues/148976
     // ...(await getAgentLogsTopErrors(esClient)),

x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts

@@ -265,6 +265,49 @@ describe('fleet usage telemetry', () => {
         },
       ],
     });
+
+    await soClient.create(
+      'ingest-outputs',
+      {
+        name: 'output2',
+        type: 'third_type',
+        hosts: ['http://localhost:9300'],
+        is_default: false,
+        is_default_monitoring: false,
+        config_yaml: '',
+        ca_trusted_fingerprint: '',
+        proxy_id: null,
+      },
+      { id: 'output2' }
+    );
+    await soClient.create(
+      'ingest-outputs',
+      {
+        name: 'output3',
+        type: 'logstash',
+        hosts: ['http://localhost:9400'],
+        is_default: false,
+        is_default_monitoring: false,
+        config_yaml: '',
+        ca_trusted_fingerprint: '',
+        proxy_id: null,
+      },
+      { id: 'output3' }
+    );
+
+    await soClient.create('ingest-agent-policies', {
+      namespace: 'default',
+      monitoring_enabled: ['logs', 'metrics'],
+      name: 'Another policy',
+      description: 'Policy 2',
+      status: 'active',
+      is_managed: false,
+      revision: 2,
+      updated_by: 'system',
+      schema_version: '1.0.0',
+      data_output_id: 'output2',
+      monitoring_output_id: 'output3',
+    });
   });
 
   afterAll(async () => {
@@ -317,7 +360,10 @@ describe('fleet usage telemetry', () => {
             },
           ],
         },
-        agent_policies: { count: 3, output_types: ['elasticsearch'] },
+        agent_policies: {
+          count: 2,
+          output_types: expect.arrayContaining(['elasticsearch', 'logstash', 'third_type']),
+        },
         agent_logs_panics_last_hour: [
           {
             timestamp: expect.any(String),