github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] added agents per output telemetry (#166432)

Browse source 
## Summary

Resolves https://github.com/elastic/ingest-dev/issues/1729

Added new telemetry about which output types agents use. Since agents
can have different data and monitoring outputs, added 2 different
counts. If an agent uses the same output as data and monitoring, it
shows up in both counts.

```
[
      { output_type: 'elasticsearch', count_as_data: 3, count_as_monitoring: 3 },
      { output_type: 'logstash', count_as_data: 1, count_as_monitoring: 0 },
      { output_type: 'kafka', count_as_data: 0, count_as_monitoring: 1 },
    ]
```

To verify: start kibana locally and wait for the FleetUsageSender task
to fire (every hour), it should show up in the debug logs. To speed it
up, change the interval locally to a few minutes
[here](https://github.com/elastic/kibana/pull/166432/files#diff-fca0b4eb6c08f4b21ad3c69bd1b9376d4665083a49095f1b621f6d86cd091674).

```
[2023-09-14T11:37:49.358+02:00][DEBUG][plugins.fleet] Agents per output type telemetry: [{"output_type":"elasticsearch","count_as_data":32,"count_as_monitoring":32}]
```

Telemetry mappings merged to staging
[here](https://github.com/elastic/telemetry/pull/2602/), should update
prod version when verified.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Julia Bardi 2023-09-18 12:44:06 +02:00 • committed by GitHub
parent 5e7984650b
commit b2057ac148
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 228 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

51
x-pack/plugins/fleet/server/collectors/agents_per_output.test.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
import { getAgentsPerOutput } from './agents_per_output';
jest.mock('../services', () => {
return {
agentPolicyService: {
list: jest.fn().mockResolvedValue({
items: [
{ agents: 0, data_output_id: 'logstash1', monitoring_output_id: 'kafka1' },
{ agents: 1 },
{ agents: 1, data_output_id: 'logstash1' },
{ agents: 1, monitoring_output_id: 'kafka1' },
{ agents: 1, data_output_id: 'elasticsearch2', monitoring_output_id: 'elasticsearch2' },
],
}),
},
};
});
describe('agents_per_output', () => {
const soClientMock = {
find: jest.fn().mockResolvedValue({
saved_objects: [
{
id: 'default-output',
attributes: { is_default: true, is_default_monitoring: true, type: 'elasticsearch' },
},
{ id: 'logstash1', attributes: { type: 'logstash' } },
{ id: 'kafka1', attributes: { type: 'kafka' } },
{ id: 'elasticsearch2', attributes: { type: 'elasticsearch' } },
],
}),
} as unknown as SavedObjectsClientContract;
it('should return agent count by output type', async () => {
const res = await getAgentsPerOutput(soClientMock, {} as unknown as ElasticsearchClient);
expect(res).toEqual([
{ output_type: 'elasticsearch', count_as_data: 3, count_as_monitoring: 3 },
{ output_type: 'logstash', count_as_data: 1, count_as_monitoring: 0 },
{ output_type: 'kafka', count_as_data: 0, count_as_monitoring: 1 },
]);
});
});

70
x-pack/plugins/fleet/server/collectors/agents_per_output.ts Normal file
View file

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
import _ from 'lodash';
import { OUTPUT_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../common';
import type { OutputSOAttributes } from '../types';
import { agentPolicyService } from '../services';
export interface AgentsPerOutputType {
output_type: string;
count_as_data: number;
count_as_monitoring: number;
}
export async function getAgentsPerOutput(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient
): Promise<AgentsPerOutputType[]> {
const { saved_objects: outputs } = await soClient.find<OutputSOAttributes>({
type: OUTPUT_SAVED_OBJECT_TYPE,
page: 1,
perPage: SO_SEARCH_LIMIT,
});
const defaultOutputId = outputs.find((output) => output.attributes.is_default)?.id || '';
const defaultMonitoringOutputId =
outputs.find((output) => output.attributes.is_default_monitoring)?.id || '';
const outputsById = _.keyBy(outputs, 'id');
const getOutputTypeById = (outputId: string): string => outputsById[outputId]?.attributes.type;
const { items } = await agentPolicyService.list(soClient, {
esClient,
withAgentCount: true,
page: 1,
perPage: SO_SEARCH_LIMIT,
});
const outputTypes: { [key: string]: AgentsPerOutputType } = {};
items
.filter((item) => (item.agents ?? 0) > 0)
.forEach((item) => {
const dataOutputType = getOutputTypeById(item.data_output_id || defaultOutputId);
if (!outputTypes[dataOutputType]) {
outputTypes[dataOutputType] = {
output_type: dataOutputType,
count_as_data: 0,
count_as_monitoring: 0,
};
}
outputTypes[dataOutputType].count_as_data += item.agents ?? 0;
const monitoringOutputType = getOutputTypeById(
item.monitoring_output_id || defaultMonitoringOutputId
);
if (!outputTypes[monitoringOutputType]) {
outputTypes[monitoringOutputType] = {
output_type: monitoringOutputType,
count_as_data: 0,
count_as_monitoring: 0,
};
}
outputTypes[monitoringOutputType].count_as_monitoring += item.agents ?? 0;
});
return Object.values(outputTypes);
}

4
x-pack/plugins/fleet/server/collectors/register.ts
View file

@ -22,6 +22,8 @@ import { getAgentPoliciesUsage } from './agent_policies';
import type { AgentPanicLogsData } from './agent_logs_panics';
import { getPanicLogsLastHour } from './agent_logs_panics';
import { getAgentLogsTopErrors } from './agent_logs_top_errors';
import type { AgentsPerOutputType } from './agents_per_output';
import { getAgentsPerOutput } from './agents_per_output';
export interface Usage {
agents_enabled: boolean;
@ -36,6 +38,7 @@ export interface FleetUsage extends Usage, AgentData {
agent_logs_panics_last_hour: AgentPanicLogsData['agent_logs_panics_last_hour'];
agent_logs_top_errors?: string[];
fleet_server_logs_top_errors?: string[];
agents_per_output_type: AgentsPerOutputType[];
}
export const fetchFleetUsage = async (
@ -57,6 +60,7 @@ export const fetchFleetUsage = async (
agent_policies: await getAgentPoliciesUsage(soClient),
...(await getPanicLogsLastHour(esClient)),
...(await getAgentLogsTopErrors(esClient)),
agents_per_output_type: await getAgentsPerOutput(soClient, esClient),
};
return usage;
};

86
x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts
View file

@ -207,6 +207,20 @@ describe('fleet usage telemetry', () => {
},
],
},
{
create: {
_id: 'agent3',
},
},
{
agent: {
version: '8.6.0',
},
last_checkin_status: 'online',
last_checkin: '2023-09-13T12:26:24Z',
active: true,
policy_id: 'policy2',
},
],
refresh: 'wait_for',
});
@ -348,20 +362,24 @@ describe('fleet usage telemetry', () => {
{ id: 'output3' }
);
await soClient.create('ingest-agent-policies', {
namespace: 'default',
monitoring_enabled: ['logs', 'metrics'],
name: 'Another policy',
description: 'Policy 2',
inactivity_timeout: 1209600,
status: 'active',
is_managed: false,
revision: 2,
updated_by: 'system',
schema_version: '1.0.0',
data_output_id: 'output2',
monitoring_output_id: 'output3',
});
await soClient.create(
'ingest-agent-policies',
{
namespace: 'default',
monitoring_enabled: ['logs', 'metrics'],
name: 'Another policy',
description: 'Policy 2',
inactivity_timeout: 1209600,
status: 'active',
is_managed: false,
revision: 2,
updated_by: 'system',
schema_version: '1.0.0',
data_output_id: 'output2',
monitoring_output_id: 'output3',
},
{ id: 'policy2' }
);
});
afterAll(async () => {
@ -379,13 +397,13 @@ describe('fleet usage telemetry', () => {
expect.objectContaining({
agents_enabled: true,
agents: {
total_enrolled: 2,
total_enrolled: 3,
healthy: 0,
unhealthy: 0,
inactive: 0,
unenrolled: 1,
offline: 2,
total_all_statuses: 3,
offline: 3,
total_all_statuses: 4,
updating: 0,
},
fleet_server: {
@ -399,6 +417,16 @@ describe('fleet usage telemetry', () => {
},
packages: [],
agents_per_version: [
{
version: '8.6.0',
count: 2,
healthy: 0,
inactive: 0,
offline: 2,
unenrolled: 0,
unhealthy: 0,
updating: 0,
},
{
version: '8.5.1',
count: 1,
@ -409,19 +437,9 @@ describe('fleet usage telemetry', () => {
unhealthy: 0,
updating: 0,
},
{
version: '8.6.0',
count: 1,
healthy: 0,
inactive: 0,
offline: 1,
unenrolled: 0,
unhealthy: 0,
updating: 0,
},
],
agent_checkin_status: { error: 1, degraded: 1 },
agents_per_policy: [2],
agents_per_policy: [2, 1],
agents_per_os: [
{
name: 'Ubuntu',
@ -434,6 +452,18 @@ describe('fleet usage telemetry', () => {
count: 1,
},
],
agents_per_output_type: [
{
count_as_data: 1,
count_as_monitoring: 0,
output_type: 'third_type',
},
{
count_as_data: 0,
count_as_monitoring: 1,
output_type: 'logstash',
},
],
fleet_server_config: {
policies: [
{

17
x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts
View file

@ -24,7 +24,7 @@ const FLEET_AGENTS_EVENT_TYPE = 'fleet_agents';
export class FleetUsageSender {
private taskManager?: TaskManagerStartContract;
private taskVersion = '1.1.1';
private taskVersion = '1.1.2';
private taskType = 'Fleet-Usage-Sender';
private wasStarted: boolean = false;
private interval = '1h';
@ -80,7 +80,11 @@ export class FleetUsageSender {
if (!usageData) {
return;
}
const { agents_per_version: agentsPerVersion, ...fleetUsageData } = usageData;
const {
agents_per_version: agentsPerVersion,
agents_per_output_type: agentsPerOutputType,
...fleetUsageData
} = usageData;
appContextService
.getLogger()
.debug('Fleet usage telemetry: ' + JSON.stringify(fleetUsageData));
@ -93,6 +97,15 @@ export class FleetUsageSender {
agentsPerVersion.forEach((byVersion) => {
core.analytics.reportEvent(FLEET_AGENTS_EVENT_TYPE, { agents_per_version: byVersion });
});
appContextService
.getLogger()
.debug('Agents per output type telemetry: ' + JSON.stringify(agentsPerOutputType));
agentsPerOutputType.forEach((byOutputType) => {
core.analytics.reportEvent(FLEET_AGENTS_EVENT_TYPE, {
agents_per_output_type: byOutputType,
});
});
} catch (error) {
appContextService
.getLogger()

30
x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts
View file

@ -9,6 +9,10 @@ import type { RootSchema } from '@kbn/analytics-client';
export const fleetAgentsSchema: RootSchema<any> = {
agents_per_version: {
_meta: {
description: 'Agents per version telemetry',
optional: true,
},
properties: {
version: {
type: 'keyword',
@ -60,6 +64,32 @@ export const fleetAgentsSchema: RootSchema<any> = {
},
},
},
agents_per_output_type: {
_meta: {
description: 'Agents per output type telemetry',
optional: true,
},
properties: {
output_type: {
type: 'keyword',
_meta: {
description: 'Output type used by agent',
},
},
count_as_data: {
type: 'long',
_meta: {
description: 'Number of agents enrolled that use this output type as data output',
},
},
count_as_monitoring: {
type: 'long',
_meta: {
description: 'Number of agents enrolled that use this output type as monitoring output',
},
},
},
},
};
export const fleetUsagesSchema: RootSchema<any> = {