mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
Deprecate Cloud Defend Telemetry
This commit is contained in:
Ido Cohen
GitHub
parent
6efa46a1ab
commit
58e25b564e
10 changed files with 0 additions and 1241 deletions
|
|
@ -1,292 +1,5 @@
|
|||
{
|
||||
"properties": {
|
||||
"cloud_defend": {
|
||||
"properties": {
|
||||
"indices": {
|
||||
"properties": {
|
||||
"alerts": {
|
||||
"properties": {
|
||||
"doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"deleted": {
|
||||
"type": "long"
|
||||
},
|
||||
"size_in_bytes": {
|
||||
"type": "long"
|
||||
},
|
||||
"last_doc_timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"file": {
|
||||
"properties": {
|
||||
"doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"deleted": {
|
||||
"type": "long"
|
||||
},
|
||||
"size_in_bytes": {
|
||||
"type": "long"
|
||||
},
|
||||
"last_doc_timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"process": {
|
||||
"properties": {
|
||||
"doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"deleted": {
|
||||
"type": "long"
|
||||
},
|
||||
"size_in_bytes": {
|
||||
"type": "long"
|
||||
},
|
||||
"last_doc_timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"latestPackageVersion": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"packageStatus": {
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"installedPackagePolicies": {
|
||||
"type": "long"
|
||||
},
|
||||
"healthyAgents": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"pods_stats": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"properties": {
|
||||
"account_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"container_image_name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"container_image_tag": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"pod_name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"total_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"process_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"file_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"alert_doc_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"accounts_stats": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"properties": {
|
||||
"account_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"cloud_provider": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"kubernetes_version": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"total_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"file_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"process_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"alert_doc_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"agents_count": {
|
||||
"type": "short"
|
||||
},
|
||||
"nodes_count": {
|
||||
"type": "short"
|
||||
},
|
||||
"pods_count": {
|
||||
"type": "short"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"installation_stats": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"properties": {
|
||||
"package_policy_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"package_version": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"agent_policy_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "date"
|
||||
},
|
||||
"agent_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"policy_yaml": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"selectors": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"operation": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"containerImageFullName": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"containerImageName": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"containerImageTag": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"kubernetesClusterId": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"kubernetesClusterName": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"kubernetesNamespace": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"kubernetesPodLabel": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"kubernetesPodName": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"targetFilePath": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"ignoreVolumeFiles": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"ignoreVolumeMounts": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"processExecutable": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"processName": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"sessionLeaderInteractive": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"match": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"exclude": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
},
|
||||
"actions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"cloud_security_posture": {
|
||||
"properties": {
|
||||
"indices": {
|
||||
|
|
|
|||
|
|
@ -1,218 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { Logger } from '@kbn/core/server';
|
||||
import type {
|
||||
AggregationsMultiBucketBase,
|
||||
SearchRequest,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { CloudDefendAccountsStats } from './types';
|
||||
import { LOGS_CLOUD_DEFEND_PATTERN } from '../../../../common/constants';
|
||||
|
||||
interface Value {
|
||||
value: number;
|
||||
}
|
||||
|
||||
interface KubernetesVersion {
|
||||
metrics: { 'orchestrator.version': string };
|
||||
}
|
||||
|
||||
interface CloudProvider {
|
||||
metrics: { 'cloud.provider': string };
|
||||
}
|
||||
|
||||
interface AccountsStats {
|
||||
accounts: {
|
||||
buckets: AccountEntity[];
|
||||
};
|
||||
}
|
||||
interface AccountEntity {
|
||||
key: string; // aggregation bucket key (currently: orchestrator.cluster.id)
|
||||
doc_count: number; // total doc count (process + file + alerts)
|
||||
process_doc_count: AggregationsMultiBucketBase;
|
||||
file_doc_count: AggregationsMultiBucketBase;
|
||||
alert_doc_count: AggregationsMultiBucketBase;
|
||||
cloud_provider: { top: CloudProvider[] };
|
||||
kubernetes_version: { top: KubernetesVersion[] };
|
||||
agents_count: Value;
|
||||
nodes_count: Value;
|
||||
pods_count: Value;
|
||||
resources: {
|
||||
pods_count: Value;
|
||||
};
|
||||
}
|
||||
|
||||
const getAccountsStatsQuery = (): SearchRequest => ({
|
||||
index: LOGS_CLOUD_DEFEND_PATTERN,
|
||||
query: {
|
||||
match_all: {},
|
||||
},
|
||||
aggs: {
|
||||
accounts: {
|
||||
terms: {
|
||||
field: 'orchestrator.cluster.id',
|
||||
order: {
|
||||
_count: 'desc',
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
aggs: {
|
||||
nodes_count: {
|
||||
cardinality: {
|
||||
field: 'cloud.instance.name',
|
||||
},
|
||||
},
|
||||
agents_count: {
|
||||
cardinality: {
|
||||
field: 'agent.id',
|
||||
},
|
||||
},
|
||||
kubernetes_version: {
|
||||
top_metrics: {
|
||||
metrics: {
|
||||
field: 'orchestrator.version',
|
||||
},
|
||||
size: 1,
|
||||
sort: {
|
||||
'@timestamp': 'desc',
|
||||
},
|
||||
},
|
||||
},
|
||||
cloud_provider: {
|
||||
top_metrics: {
|
||||
metrics: {
|
||||
field: 'cloud.provider',
|
||||
},
|
||||
size: 1,
|
||||
sort: {
|
||||
'@timestamp': 'desc',
|
||||
},
|
||||
},
|
||||
},
|
||||
file_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.category': 'file',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
process_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.category': 'process',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
alert_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.kind': 'alert',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
pods_count: {
|
||||
cardinality: {
|
||||
field: 'orchestrator.resource.name',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
size: 0,
|
||||
_source: false,
|
||||
});
|
||||
|
||||
const getCloudDefendAccountsStats = (
|
||||
aggregatedResourcesStats: AccountsStats,
|
||||
logger: Logger
|
||||
): CloudDefendAccountsStats[] => {
|
||||
const accounts = aggregatedResourcesStats.accounts.buckets;
|
||||
|
||||
const cloudDefendAccountsStats = accounts.map((account) => ({
|
||||
account_id: account.key,
|
||||
total_doc_count: account.doc_count,
|
||||
file_doc_count: account.file_doc_count.doc_count,
|
||||
process_doc_count: account.process_doc_count.doc_count,
|
||||
alert_doc_count: account.alert_doc_count.doc_count,
|
||||
kubernetes_version: account.kubernetes_version?.top?.[0]?.metrics['orchestrator.version'],
|
||||
cloud_provider: account.cloud_provider?.top?.[0]?.metrics['cloud.provider'],
|
||||
agents_count: account.agents_count.value,
|
||||
nodes_count: account.nodes_count.value,
|
||||
pods_count: account.pods_count.value,
|
||||
}));
|
||||
logger.info('CloudDefend telemetry: accounts stats was sent');
|
||||
|
||||
return cloudDefendAccountsStats;
|
||||
};
|
||||
|
||||
export const getAccountsStats = async (
|
||||
esClient: ElasticsearchClient,
|
||||
logger: Logger
|
||||
): Promise<CloudDefendAccountsStats[]> => {
|
||||
try {
|
||||
const isIndexExists = await esClient.indices.exists({
|
||||
index: LOGS_CLOUD_DEFEND_PATTERN,
|
||||
});
|
||||
|
||||
if (isIndexExists) {
|
||||
const accountsStatsResponse = await esClient.search<unknown, AccountsStats>(
|
||||
getAccountsStatsQuery()
|
||||
);
|
||||
|
||||
const cloudDefendAccountsStats = accountsStatsResponse.aggregations
|
||||
? getCloudDefendAccountsStats(accountsStatsResponse.aggregations, logger)
|
||||
: [];
|
||||
|
||||
return cloudDefendAccountsStats;
|
||||
}
|
||||
|
||||
return [];
|
||||
} catch (e) {
|
||||
logger.error(`Failed to get account stats ${e}`);
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { CoreStart, Logger, SavedObjectsClientContract } from '@kbn/core/server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { getCloudDefendStatus } from '../../../routes/status/status';
|
||||
import type { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types';
|
||||
|
||||
import type { CloudDefendIndicesStats, IndexStats } from './types';
|
||||
import {
|
||||
ALERTS_INDEX_PATTERN,
|
||||
FILE_INDEX_PATTERN,
|
||||
PROCESS_INDEX_PATTERN,
|
||||
} from '../../../../common/constants';
|
||||
|
||||
const getIndexDocCount = (esClient: ElasticsearchClient, index: string) =>
|
||||
esClient.indices.stats({ index });
|
||||
|
||||
const getLatestDocTimestamp = async (
|
||||
esClient: ElasticsearchClient,
|
||||
index: string
|
||||
): Promise<string | null> => {
|
||||
const latestTimestamp = await esClient.search({
|
||||
index,
|
||||
query: {
|
||||
match_all: {},
|
||||
},
|
||||
sort: '@timestamp:desc',
|
||||
size: 1,
|
||||
fields: ['@timestamp'],
|
||||
_source: false,
|
||||
});
|
||||
|
||||
const latestEventTimestamp = latestTimestamp.hits?.hits[0]?.fields;
|
||||
|
||||
return latestEventTimestamp ? latestEventTimestamp['@timestamp'][0] : null;
|
||||
};
|
||||
|
||||
const getIndexStats = async (
|
||||
esClient: ElasticsearchClient,
|
||||
index: string,
|
||||
logger: Logger
|
||||
): Promise<IndexStats | {}> => {
|
||||
try {
|
||||
const lastDocTimestamp = await getLatestDocTimestamp(esClient, index);
|
||||
|
||||
if (lastDocTimestamp) {
|
||||
const indexStats = await getIndexDocCount(esClient, index);
|
||||
return {
|
||||
doc_count: indexStats._all.primaries?.docs ? indexStats._all.primaries?.docs?.count : 0,
|
||||
deleted: indexStats._all.primaries?.docs?.deleted
|
||||
? indexStats._all.primaries?.docs?.deleted
|
||||
: 0,
|
||||
size_in_bytes: indexStats._all.primaries?.store
|
||||
? indexStats._all.primaries?.store.size_in_bytes
|
||||
: 0,
|
||||
last_doc_timestamp: lastDocTimestamp,
|
||||
};
|
||||
}
|
||||
|
||||
return {};
|
||||
} catch (e) {
|
||||
logger.error(`Failed to get index stats for ${index}`);
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
export const getIndicesStats = async (
|
||||
esClient: ElasticsearchClient,
|
||||
soClient: SavedObjectsClientContract,
|
||||
coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>,
|
||||
logger: Logger
|
||||
): Promise<CloudDefendIndicesStats> => {
|
||||
const [alerts, file, process] = await Promise.all([
|
||||
getIndexStats(esClient, ALERTS_INDEX_PATTERN, logger),
|
||||
getIndexStats(esClient, FILE_INDEX_PATTERN, logger),
|
||||
getIndexStats(esClient, PROCESS_INDEX_PATTERN, logger),
|
||||
]);
|
||||
|
||||
const [, cloudDefendPluginStartDeps] = await coreServices;
|
||||
|
||||
const { status, latestPackageVersion, installedPackagePolicies, healthyAgents } =
|
||||
await getCloudDefendStatus({
|
||||
logger,
|
||||
esClient,
|
||||
soClient,
|
||||
agentPolicyService: cloudDefendPluginStartDeps.fleet.agentPolicyService,
|
||||
agentService: cloudDefendPluginStartDeps.fleet.agentService,
|
||||
packagePolicyService: cloudDefendPluginStartDeps.fleet.packagePolicyService,
|
||||
packageService: cloudDefendPluginStartDeps.fleet.packageService,
|
||||
});
|
||||
|
||||
return {
|
||||
alerts,
|
||||
file,
|
||||
process,
|
||||
latestPackageVersion,
|
||||
packageStatus: {
|
||||
status,
|
||||
installedPackagePolicies,
|
||||
healthyAgents,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { CoreStart, Logger, SavedObjectsClientContract } from '@kbn/core/server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import {
|
||||
AgentPolicy,
|
||||
PackagePolicy,
|
||||
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
|
||||
SO_SEARCH_LIMIT,
|
||||
} from '@kbn/fleet-plugin/common';
|
||||
import { agentPolicyService } from '@kbn/fleet-plugin/server/services';
|
||||
import type { CloudDefendInstallationStats } from './types';
|
||||
import type { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types';
|
||||
import { INTEGRATION_PACKAGE_NAME, INPUT_CONTROL } from '../../../../common/constants';
|
||||
import {
|
||||
getInputFromPolicy,
|
||||
getSelectorsAndResponsesFromYaml,
|
||||
} from '../../../../common/utils/helpers';
|
||||
|
||||
export const getInstallationStats = async (
|
||||
esClient: ElasticsearchClient,
|
||||
soClient: SavedObjectsClientContract,
|
||||
coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>,
|
||||
logger: Logger
|
||||
): Promise<CloudDefendInstallationStats[]> => {
|
||||
const [, cloudDefendServerPluginStartDeps] = await coreServices;
|
||||
|
||||
const cloudDefendContext = {
|
||||
logger,
|
||||
esClient,
|
||||
soClient,
|
||||
agentPolicyService: cloudDefendServerPluginStartDeps.fleet.agentPolicyService,
|
||||
packagePolicyService: cloudDefendServerPluginStartDeps.fleet.packagePolicyService,
|
||||
};
|
||||
|
||||
const getInstalledPackagePolicies = async (
|
||||
packagePolicies: PackagePolicy[],
|
||||
agentPolicies: AgentPolicy[]
|
||||
) => {
|
||||
const installationStats = packagePolicies.map(
|
||||
(packagePolicy: PackagePolicy): CloudDefendInstallationStats => {
|
||||
const agentCounts =
|
||||
agentPolicies?.find((agentPolicy) => agentPolicy?.id === packagePolicy.policy_id)
|
||||
?.agents ?? 0;
|
||||
|
||||
const input = getInputFromPolicy(packagePolicy, INPUT_CONTROL);
|
||||
const policyYaml = input?.vars?.configuration?.value;
|
||||
const { selectors, responses } = getSelectorsAndResponsesFromYaml(policyYaml);
|
||||
|
||||
return {
|
||||
package_policy_id: packagePolicy.id,
|
||||
package_version: packagePolicy.package?.version as string,
|
||||
created_at: packagePolicy.created_at,
|
||||
agent_policy_id: packagePolicy.policy_ids[0],
|
||||
agent_count: agentCounts,
|
||||
policy_yaml: policyYaml,
|
||||
selectors,
|
||||
responses,
|
||||
};
|
||||
}
|
||||
);
|
||||
return installationStats;
|
||||
};
|
||||
|
||||
const packagePolicies = await cloudDefendContext.packagePolicyService.list(soClient, {
|
||||
perPage: SO_SEARCH_LIMIT,
|
||||
kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:"${INTEGRATION_PACKAGE_NAME}"`,
|
||||
});
|
||||
|
||||
const agentPolicies = await agentPolicyService.list(soClient, {
|
||||
perPage: SO_SEARCH_LIMIT,
|
||||
kuery: '',
|
||||
esClient,
|
||||
withAgentCount: true,
|
||||
});
|
||||
|
||||
if (!packagePolicies) return [];
|
||||
|
||||
const installationStats: CloudDefendInstallationStats[] = await getInstalledPackagePolicies(
|
||||
packagePolicies.items,
|
||||
agentPolicies?.items || []
|
||||
);
|
||||
|
||||
return installationStats;
|
||||
};
|
||||
|
|
@ -1,210 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { Logger } from '@kbn/core/server';
|
||||
import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { CloudDefendPodsStats } from './types';
|
||||
import { LOGS_CLOUD_DEFEND_PATTERN } from '../../../../common/constants';
|
||||
|
||||
interface PodsStats {
|
||||
accounts: {
|
||||
buckets: AccountEntity[];
|
||||
};
|
||||
}
|
||||
|
||||
interface Bucket {
|
||||
key: string;
|
||||
}
|
||||
|
||||
export interface AccountEntity {
|
||||
key: string; // orchestrator.cluster.id
|
||||
doc_count: number;
|
||||
pods: {
|
||||
buckets: Pod[];
|
||||
};
|
||||
}
|
||||
|
||||
interface Pod {
|
||||
key: string; // orchestrator.resource.name
|
||||
container_image_name: {
|
||||
buckets: Bucket[];
|
||||
};
|
||||
container_image_tag: {
|
||||
buckets: Bucket[];
|
||||
};
|
||||
doc_count: number;
|
||||
file_doc_count: {
|
||||
doc_count: number;
|
||||
};
|
||||
process_doc_count: {
|
||||
doc_count: number;
|
||||
};
|
||||
alert_doc_count: {
|
||||
doc_count: number;
|
||||
};
|
||||
}
|
||||
|
||||
const getPodsStatsQuery = (index: string): SearchRequest => ({
|
||||
index,
|
||||
query: {
|
||||
match_all: {},
|
||||
},
|
||||
aggs: {
|
||||
accounts: {
|
||||
terms: {
|
||||
field: 'orchestrator.cluster.id',
|
||||
order: {
|
||||
_count: 'desc',
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
aggs: {
|
||||
// all cloud-defend logs are from the viewpoint of an orchestrator.resource.type = "pod"
|
||||
// so no need to filter by orchestrator.resource.type.
|
||||
pods: {
|
||||
terms: {
|
||||
field: 'orchestrator.resource.name',
|
||||
order: {
|
||||
_count: 'desc',
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
aggs: {
|
||||
container_image_name: {
|
||||
terms: {
|
||||
field: 'container.image.name',
|
||||
size: 1,
|
||||
},
|
||||
},
|
||||
container_image_tag: {
|
||||
terms: {
|
||||
field: 'container.image.tag',
|
||||
size: 1,
|
||||
},
|
||||
},
|
||||
file_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.category': 'file',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
process_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.category': 'process',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
alert_doc_count: {
|
||||
filter: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
term: {
|
||||
'event.kind': 'alert',
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
size: 0,
|
||||
_source: false,
|
||||
});
|
||||
|
||||
const getCloudDefendPodsStats = (
|
||||
aggregatedPodsStats: PodsStats,
|
||||
logger: Logger
|
||||
): CloudDefendPodsStats[] => {
|
||||
const accounts = aggregatedPodsStats.accounts.buckets;
|
||||
const podsStats = accounts.map((account) => {
|
||||
const accountId = account.key;
|
||||
return account.pods.buckets.map((pod) => {
|
||||
return {
|
||||
account_id: accountId,
|
||||
pod_name: pod.key,
|
||||
container_image_name: pod.container_image_name?.buckets?.[0]?.key,
|
||||
container_image_tag: pod.container_image_tag?.buckets?.[0]?.key,
|
||||
total_doc_count: pod.doc_count,
|
||||
file_doc_count: pod.file_doc_count.doc_count,
|
||||
process_doc_count: pod.process_doc_count.doc_count,
|
||||
alert_doc_count: pod.alert_doc_count.doc_count,
|
||||
};
|
||||
});
|
||||
});
|
||||
logger.info('Cloud defend telemetry: pods stats was sent');
|
||||
|
||||
return podsStats.flat(2);
|
||||
};
|
||||
|
||||
export const getPodsStats = async (
|
||||
esClient: ElasticsearchClient,
|
||||
logger: Logger
|
||||
): Promise<CloudDefendPodsStats[]> => {
|
||||
try {
|
||||
const isIndexExists = await esClient.indices.exists({
|
||||
index: LOGS_CLOUD_DEFEND_PATTERN,
|
||||
});
|
||||
|
||||
if (isIndexExists) {
|
||||
const podsStatsResponse = await esClient.search<unknown, PodsStats>(
|
||||
getPodsStatsQuery(LOGS_CLOUD_DEFEND_PATTERN)
|
||||
);
|
||||
|
||||
const cloudDefendPodsStats = podsStatsResponse.aggregations
|
||||
? getCloudDefendPodsStats(podsStatsResponse.aggregations, logger)
|
||||
: [];
|
||||
|
||||
return cloudDefendPodsStats;
|
||||
}
|
||||
|
||||
return [];
|
||||
} catch (e) {
|
||||
logger.error(`Failed to get pods stats ${e}`);
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { CollectorFetchContext, UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
|
||||
import type { CoreStart, Logger } from '@kbn/core/server';
|
||||
import { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types';
|
||||
import { getIndicesStats } from './indices_stats_collector';
|
||||
import { getPodsStats } from './pods_stats_collector';
|
||||
import { cloudDefendUsageSchema } from './schema';
|
||||
import { CloudDefendUsage } from './types';
|
||||
import { getAccountsStats } from './accounts_stats_collector';
|
||||
import { getInstallationStats } from './installation_stats_collector';
|
||||
|
||||
export function registerCloudDefendUsageCollector(
|
||||
logger: Logger,
|
||||
coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>,
|
||||
usageCollection?: UsageCollectionSetup
|
||||
): void {
|
||||
// usageCollection is an optional dependency, so make sure to return if it is not registered
|
||||
if (!usageCollection) {
|
||||
logger.debug('Usage collection disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create usage collector
|
||||
const cloudDefendUsageCollector = usageCollection.makeUsageCollector<CloudDefendUsage>({
|
||||
type: 'cloud_defend',
|
||||
isReady: async () => {
|
||||
await coreServices;
|
||||
return true;
|
||||
},
|
||||
fetch: async (collectorFetchContext: CollectorFetchContext) => {
|
||||
logger.debug('Starting cloud_defend usage collection');
|
||||
|
||||
const [indicesStats, accountsStats, podsStats, installationStats] = await Promise.all([
|
||||
getIndicesStats(
|
||||
collectorFetchContext.esClient,
|
||||
collectorFetchContext.soClient,
|
||||
coreServices,
|
||||
logger
|
||||
),
|
||||
getAccountsStats(collectorFetchContext.esClient, logger),
|
||||
getPodsStats(collectorFetchContext.esClient, logger),
|
||||
getInstallationStats(
|
||||
collectorFetchContext.esClient,
|
||||
collectorFetchContext.soClient,
|
||||
coreServices,
|
||||
logger
|
||||
),
|
||||
]).catch((err) => {
|
||||
logger.error(err);
|
||||
|
||||
return err;
|
||||
});
|
||||
|
||||
return {
|
||||
indices: indicesStats,
|
||||
accounts_stats: accountsStats,
|
||||
pods_stats: podsStats,
|
||||
installation_stats: installationStats,
|
||||
};
|
||||
},
|
||||
schema: cloudDefendUsageSchema,
|
||||
});
|
||||
|
||||
// Register usage collector
|
||||
usageCollection.registerCollector(cloudDefendUsageCollector);
|
||||
}
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server';
|
||||
import type { CloudDefendUsage } from './types';
|
||||
|
||||
export const cloudDefendUsageSchema: MakeSchemaFrom<CloudDefendUsage> = {
|
||||
indices: {
|
||||
alerts: {
|
||||
doc_count: {
|
||||
type: 'long',
|
||||
},
|
||||
deleted: {
|
||||
type: 'long',
|
||||
},
|
||||
size_in_bytes: {
|
||||
type: 'long',
|
||||
},
|
||||
last_doc_timestamp: {
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
file: {
|
||||
doc_count: {
|
||||
type: 'long',
|
||||
},
|
||||
deleted: {
|
||||
type: 'long',
|
||||
},
|
||||
size_in_bytes: {
|
||||
type: 'long',
|
||||
},
|
||||
last_doc_timestamp: {
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
process: {
|
||||
doc_count: {
|
||||
type: 'long',
|
||||
},
|
||||
deleted: {
|
||||
type: 'long',
|
||||
},
|
||||
size_in_bytes: {
|
||||
type: 'long',
|
||||
},
|
||||
last_doc_timestamp: {
|
||||
type: 'date',
|
||||
},
|
||||
},
|
||||
latestPackageVersion: { type: 'keyword' },
|
||||
packageStatus: {
|
||||
status: { type: 'keyword' },
|
||||
installedPackagePolicies: { type: 'long' },
|
||||
healthyAgents: { type: 'long' },
|
||||
},
|
||||
},
|
||||
pods_stats: {
|
||||
type: 'array',
|
||||
items: {
|
||||
account_id: { type: 'keyword' },
|
||||
container_image_name: { type: 'keyword' },
|
||||
container_image_tag: { type: 'keyword' },
|
||||
pod_name: { type: 'keyword' },
|
||||
total_doc_count: { type: 'long' },
|
||||
process_doc_count: { type: 'long' },
|
||||
file_doc_count: { type: 'long' },
|
||||
alert_doc_count: { type: 'long' },
|
||||
},
|
||||
},
|
||||
accounts_stats: {
|
||||
type: 'array',
|
||||
items: {
|
||||
account_id: { type: 'keyword' },
|
||||
cloud_provider: { type: 'keyword' },
|
||||
kubernetes_version: { type: 'keyword' },
|
||||
total_doc_count: { type: 'long' },
|
||||
file_doc_count: { type: 'long' },
|
||||
process_doc_count: { type: 'long' },
|
||||
alert_doc_count: { type: 'long' },
|
||||
agents_count: { type: 'short' },
|
||||
nodes_count: { type: 'short' },
|
||||
pods_count: { type: 'short' },
|
||||
},
|
||||
},
|
||||
installation_stats: {
|
||||
type: 'array',
|
||||
items: {
|
||||
package_policy_id: { type: 'keyword' },
|
||||
package_version: { type: 'keyword' },
|
||||
agent_policy_id: { type: 'keyword' },
|
||||
created_at: { type: 'date' },
|
||||
agent_count: { type: 'long' },
|
||||
policy_yaml: { type: 'keyword' },
|
||||
selectors: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: { type: 'keyword' },
|
||||
name: { type: 'keyword' },
|
||||
operation: { type: 'array', items: { type: 'keyword' } },
|
||||
containerImageFullName: { type: 'array', items: { type: 'keyword' } },
|
||||
containerImageName: { type: 'array', items: { type: 'keyword' } },
|
||||
containerImageTag: { type: 'array', items: { type: 'keyword' } },
|
||||
kubernetesClusterId: { type: 'array', items: { type: 'keyword' } },
|
||||
kubernetesClusterName: { type: 'array', items: { type: 'keyword' } },
|
||||
kubernetesNamespace: { type: 'array', items: { type: 'keyword' } },
|
||||
kubernetesPodLabel: { type: 'array', items: { type: 'keyword' } },
|
||||
kubernetesPodName: { type: 'array', items: { type: 'keyword' } },
|
||||
targetFilePath: { type: 'array', items: { type: 'keyword' } },
|
||||
ignoreVolumeFiles: { type: 'boolean' },
|
||||
ignoreVolumeMounts: { type: 'boolean' },
|
||||
processExecutable: { type: 'array', items: { type: 'keyword' } },
|
||||
processName: { type: 'array', items: { type: 'keyword' } },
|
||||
sessionLeaderInteractive: { type: 'boolean' },
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: { type: 'keyword' },
|
||||
match: { type: 'array', items: { type: 'keyword' } },
|
||||
exclude: { type: 'array', items: { type: 'keyword' } },
|
||||
actions: { type: 'array', items: { type: 'keyword' } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
@ -1,121 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
// for some reason we can't reference common/index.ts because
|
||||
// the `node scripts/check_telemetry.js --fix` command fails with the error
|
||||
// ERROR Error: Error extracting collector in x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/register.ts
|
||||
// Error: Unable to find identifier in source Selector
|
||||
// at createFailError (dev_cli_errors.ts:27:24)
|
||||
// at parseUsageCollection (ts_parser.ts:226:32)
|
||||
// at parseUsageCollection.next (<anonymous>)
|
||||
// at extractCollectors (extract_collectors.ts:58:32)
|
||||
// at extractCollectors.next (<anonymous>)
|
||||
// at Task.task (extract_collectors_task.ts:43:53)
|
||||
// at runMicrotasks (<anonymous>)
|
||||
// at processTicksAndRejections (node:internal/process/task_queues:96:5)
|
||||
//
|
||||
// I guess the intermediate import/export is causing problems
|
||||
// for now we will just point to the current version (v1)
|
||||
import type {
|
||||
Selector,
|
||||
Response,
|
||||
SelectorType,
|
||||
SelectorCondition,
|
||||
ResponseAction,
|
||||
} from '../../../../common/v1';
|
||||
|
||||
export interface CloudDefendUsage {
|
||||
indices: CloudDefendIndicesStats;
|
||||
pods_stats: CloudDefendPodsStats[];
|
||||
accounts_stats: CloudDefendAccountsStats[];
|
||||
installation_stats: CloudDefendInstallationStats[];
|
||||
}
|
||||
|
||||
export interface PackageSetupStatus {
|
||||
status: string;
|
||||
installedPackagePolicies: number;
|
||||
healthyAgents: number;
|
||||
}
|
||||
|
||||
export interface CloudDefendIndicesStats {
|
||||
alerts: IndexStats | {};
|
||||
file: IndexStats | {};
|
||||
process: IndexStats | {};
|
||||
latestPackageVersion: string;
|
||||
packageStatus: PackageSetupStatus;
|
||||
}
|
||||
|
||||
export interface IndexStats {
|
||||
doc_count: number;
|
||||
deleted: number;
|
||||
size_in_bytes: number;
|
||||
last_doc_timestamp: string | null;
|
||||
}
|
||||
|
||||
export interface CloudDefendPodsStats {
|
||||
account_id: string;
|
||||
pod_name: string;
|
||||
container_image_name: string;
|
||||
container_image_tag: string;
|
||||
total_doc_count: number;
|
||||
file_doc_count: number;
|
||||
process_doc_count: number;
|
||||
alert_doc_count: number;
|
||||
}
|
||||
|
||||
export interface CloudDefendAccountsStats {
|
||||
account_id: string;
|
||||
total_doc_count: number;
|
||||
cloud_provider: string;
|
||||
kubernetes_version: string | null;
|
||||
file_doc_count: number;
|
||||
process_doc_count: number;
|
||||
alert_doc_count: number;
|
||||
agents_count: number;
|
||||
nodes_count: number;
|
||||
pods_count: number;
|
||||
}
|
||||
|
||||
export type CloudDefendSelectorTypeCounts = {
|
||||
[key in SelectorType]: number;
|
||||
};
|
||||
|
||||
export type CloudDefendResponseTypeCounts = {
|
||||
[key in SelectorType]: number;
|
||||
};
|
||||
|
||||
export type CloudDefendConditionsCounts = {
|
||||
[key in SelectorCondition]?: number;
|
||||
};
|
||||
|
||||
export type CloudDefendActionCounts = {
|
||||
[key in ResponseAction]?: number;
|
||||
};
|
||||
|
||||
export interface CloudDefendPolicyYamlStats {
|
||||
policy_yaml: string;
|
||||
policy_json: string; // to be used for further digging in BigQuery
|
||||
selector_counts: CloudDefendSelectorTypeCounts;
|
||||
response_counts: CloudDefendResponseTypeCounts;
|
||||
selector_conditions_counts: CloudDefendConditionsCounts;
|
||||
response_actions_counts: CloudDefendActionCounts;
|
||||
response_match_names: string[];
|
||||
response_exclude_names: string[];
|
||||
}
|
||||
|
||||
type CloudDefendSelector = Omit<Selector, 'hasErrors'>;
|
||||
type CloudDefendResponse = Omit<Response, 'hasErrors'>;
|
||||
|
||||
export interface CloudDefendInstallationStats {
|
||||
package_policy_id: string;
|
||||
package_version: string;
|
||||
agent_policy_id: string;
|
||||
created_at: string;
|
||||
agent_count: number;
|
||||
policy_yaml: string;
|
||||
selectors: CloudDefendSelector[];
|
||||
responses: CloudDefendResponse[];
|
||||
}
|
||||
|
|
@ -23,7 +23,6 @@ import { setupRoutes } from './routes/setup_routes';
|
|||
import { isCloudDefendPackage } from '../common/utils/helpers';
|
||||
import { isSubscriptionAllowed } from '../common/utils/subscription';
|
||||
import { onPackagePolicyPostCreateCallback } from './lib/fleet_util';
|
||||
import { registerCloudDefendUsageCollector } from './lib/telemetry/collectors/register';
|
||||
|
||||
export class CloudDefendPlugin implements Plugin<CloudDefendPluginSetup, CloudDefendPluginStart> {
|
||||
private readonly logger: Logger;
|
||||
|
|
@ -44,9 +43,6 @@ export class CloudDefendPlugin implements Plugin<CloudDefendPluginSetup, CloudDe
|
|||
logger: this.logger,
|
||||
});
|
||||
|
||||
const coreStartServices = core.getStartServices();
|
||||
registerCloudDefendUsageCollector(this.logger, coreStartServices, plugins.usageCollection);
|
||||
|
||||
this.isCloudEnabled = plugins.cloud.isCloudEnabled;
|
||||
|
||||
return {};
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@
|
|||
"@kbn/data-views-plugin",
|
||||
"@kbn/utility-types",
|
||||
"@kbn/core-http-router-server-mocks",
|
||||
"@kbn/core-elasticsearch-server",
|
||||
"@kbn/code-editor",
|
||||
"@kbn/code-editor-mock",
|
||||
"@kbn/core-security-common"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue