github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-28 11:05:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][Endpoint] script to load data into an environment for testing or development (#175132)

Browse source 
## Summary

- New script to load data to an environment for testing
- currently supports loading Endpoint policies and endpoint artifacts
(Trusted Apps, Blocklists, Event filters, Host Isolation Exceptions and
Endpoint Exceptions
This commit is contained in:
Paul Tavares 2024-01-30 09:14:09 -05:00 • committed by GitHub
parent 851df6906b
commit 9666b19644
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 2387 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

9
packages/kbn-securitysolution-io-ts-list-types/src/common/list_operator/index.ts
View file

@ -15,6 +15,15 @@ export enum ListOperatorEnum {
EXCLUDED = 'excluded', EXCLUDED = 'excluded',
} }
export const listOperatorType = t.keyof({
nested: null,
match: null,
match_any: null,
wildcard: null,
exists: null,
list: null,
});
export type ListOperatorType = t.TypeOf<typeof listOperatorType>;
export enum ListOperatorTypeEnum { export enum ListOperatorTypeEnum {
NESTED = 'nested', NESTED = 'nested',
MATCH = 'match', MATCH = 'match',

926
x-pack/plugins/security_solution/common/endpoint/data_generators/common/alerts_ecs_fields.ts Normal file
View file

@ -0,0 +1,926 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/**
* A list of ECS fields that can appear on the `logs-endpoint.events.*` index.
* Used to defined Event Filters and Endpoint Exceptions
*/
export const ENDPOINT_EVENTS_LOG_INDEX_FIELDS: readonly string[] = [
'@timestamp',
'Effective_process.entity_id',
'Effective_process.executable',
'Effective_process.name',
'Effective_process.pid',
'Persistence.args',
'Persistence.executable',
'Persistence.keepalive',
'Persistence.name',
'Persistence.path',
'Persistence.runatload',
'Target.process.Ext.memory_region.allocation_base',
'Target.process.Ext.memory_region.allocation_protection',
'Target.process.Ext.memory_region.allocation_size',
'Target.process.Ext.memory_region.allocation_type',
'Target.process.Ext.memory_region.bytes_address',
'Target.process.Ext.memory_region.bytes_allocation_offset',
'Target.process.Ext.memory_region.bytes_compressed',
'Target.process.Ext.memory_region.bytes_compressed_present',
'Target.process.Ext.memory_region.hash.sha256',
'Target.process.Ext.memory_region.malware_signature.all_names',
'Target.process.Ext.memory_region.malware_signature.identifier',
'Target.process.Ext.memory_region.malware_signature.primary.matches',
'Target.process.Ext.memory_region.malware_signature.primary.signature.hash.sha256',
'Target.process.Ext.memory_region.malware_signature.primary.signature.id',
'Target.process.Ext.memory_region.malware_signature.primary.signature.name',
'Target.process.Ext.memory_region.malware_signature.version',
'Target.process.Ext.memory_region.mapped_path',
'Target.process.Ext.memory_region.mapped_pe.Ext.dotnet',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.md5',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha1',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha256',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha384',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha512',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.ssdeep',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.hash.tlsh',
'Target.process.Ext.memory_region.mapped_pe.Ext.sections.name',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.md5',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha1',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha256',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha384',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha512',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.ssdeep',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.hash.tlsh',
'Target.process.Ext.memory_region.mapped_pe.Ext.streams.name',
'Target.process.Ext.memory_region.mapped_pe.architecture',
'Target.process.Ext.memory_region.mapped_pe.company',
'Target.process.Ext.memory_region.mapped_pe.description',
'Target.process.Ext.memory_region.mapped_pe.file_version',
'Target.process.Ext.memory_region.mapped_pe.go_import_hash',
'Target.process.Ext.memory_region.mapped_pe.go_imports',
'Target.process.Ext.memory_region.mapped_pe.go_imports_names_entropy',
'Target.process.Ext.memory_region.mapped_pe.go_imports_names_var_entropy',
'Target.process.Ext.memory_region.mapped_pe.go_stripped',
'Target.process.Ext.memory_region.mapped_pe.imphash',
'Target.process.Ext.memory_region.mapped_pe.import_hash',
'Target.process.Ext.memory_region.mapped_pe.imports',
'Target.process.Ext.memory_region.mapped_pe.imports_names_entropy',
'Target.process.Ext.memory_region.mapped_pe.imports_names_var_entropy',
'Target.process.Ext.memory_region.mapped_pe.original_file_name',
'Target.process.Ext.memory_region.mapped_pe.pehash',
'Target.process.Ext.memory_region.mapped_pe.product',
'Target.process.Ext.memory_region.mapped_pe.sections.entropy',
'Target.process.Ext.memory_region.mapped_pe.sections.name',
'Target.process.Ext.memory_region.mapped_pe.sections.physical_size',
'Target.process.Ext.memory_region.mapped_pe.sections.var_entropy',
'Target.process.Ext.memory_region.mapped_pe.sections.virtual_size',
'Target.process.Ext.memory_region.mapped_pe_detected',
'Target.process.Ext.memory_region.memory_pe.Ext.dotnet',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.md5',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.sha1',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.sha256',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.sha384',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.sha512',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.ssdeep',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.hash.tlsh',
'Target.process.Ext.memory_region.memory_pe.Ext.sections.name',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.md5',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.sha1',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.sha256',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.sha384',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.sha512',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.ssdeep',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.hash.tlsh',
'Target.process.Ext.memory_region.memory_pe.Ext.streams.name',
'Target.process.Ext.memory_region.memory_pe.architecture',
'Target.process.Ext.memory_region.memory_pe.company',
'Target.process.Ext.memory_region.memory_pe.description',
'Target.process.Ext.memory_region.memory_pe.file_version',
'Target.process.Ext.memory_region.memory_pe.go_import_hash',
'Target.process.Ext.memory_region.memory_pe.go_imports',
'Target.process.Ext.memory_region.memory_pe.go_imports_names_entropy',
'Target.process.Ext.memory_region.memory_pe.go_imports_names_var_entropy',
'Target.process.Ext.memory_region.memory_pe.go_stripped',
'Target.process.Ext.memory_region.memory_pe.imphash',
'Target.process.Ext.memory_region.memory_pe.import_hash',
'Target.process.Ext.memory_region.memory_pe.imports',
'Target.process.Ext.memory_region.memory_pe.imports_names_entropy',
'Target.process.Ext.memory_region.memory_pe.imports_names_var_entropy',
'Target.process.Ext.memory_region.memory_pe.original_file_name',
'Target.process.Ext.memory_region.memory_pe.pehash',
'Target.process.Ext.memory_region.memory_pe.product',
'Target.process.Ext.memory_region.memory_pe.sections.entropy',
'Target.process.Ext.memory_region.memory_pe.sections.name',
'Target.process.Ext.memory_region.memory_pe.sections.physical_size',
'Target.process.Ext.memory_region.memory_pe.sections.var_entropy',
'Target.process.Ext.memory_region.memory_pe.sections.virtual_size',
'Target.process.Ext.memory_region.memory_pe_detected',
'Target.process.Ext.memory_region.region_base',
'Target.process.Ext.memory_region.region_protection',
'Target.process.Ext.memory_region.region_size',
'Target.process.Ext.memory_region.region_state',
'Target.process.Ext.memory_region.strings',
'Target.process.Ext.token.integrity_level_name',
'Target.process.entity_id',
'Target.process.executable',
'Target.process.executable.caseless',
'Target.process.executable.text',
'Target.process.name',
'Target.process.name.caseless',
'Target.process.name.text',
'Target.process.pid',
'_id',
'_index',
'_score',
'_source',
'agent.ephemeral_id',
'agent.id',
'agent.name',
'agent.type',
'agent.version',
'cloud.account.id',
'cloud.instance.name',
'cloud.project.id',
'cloud.provider',
'cloud.region',
'container.id',
'container.image.hash.all',
'container.image.name',
'container.image.tag',
'container.name',
'data_stream.dataset',
'data_stream.namespace',
'data_stream.type',
'destination.address',
'destination.as.number',
'destination.as.organization.name',
'destination.as.organization.name.text',
'destination.bytes',
'destination.domain',
'destination.geo.city_name',
'destination.geo.continent_code',
'destination.geo.continent_name',
'destination.geo.country_iso_code',
'destination.geo.country_name',
'destination.geo.location',
'destination.geo.name',
'destination.geo.postal_code',
'destination.geo.region_iso_code',
'destination.geo.region_name',
'destination.geo.timezone',
'destination.ip',
'destination.packets',
'destination.port',
'destination.registered_domain',
'destination.top_level_domain',
'dll.Ext.code_signature.exists',
'dll.Ext.code_signature.status',
'dll.Ext.code_signature.subject_name',
'dll.Ext.code_signature.trusted',
'dll.Ext.code_signature.valid',
'dll.Ext.defense_evasions',
'dll.Ext.device.bus_type',
'dll.Ext.device.dos_name',
'dll.Ext.device.file_system_type',
'dll.Ext.device.nt_name',
'dll.Ext.device.product_id',
'dll.Ext.device.serial_number',
'dll.Ext.device.vendor_id',
'dll.Ext.device.volume_device_type',
'dll.Ext.load_index',
'dll.Ext.relative_file_creation_time',
'dll.Ext.relative_file_name_modify_time',
'dll.Ext.size',
'dll.code_signature.exists',
'dll.code_signature.signing_id',
'dll.code_signature.status',
'dll.code_signature.subject_name',
'dll.code_signature.team_id',
'dll.code_signature.trusted',
'dll.code_signature.valid',
'dll.hash.md5',
'dll.hash.sha1',
'dll.hash.sha256',
'dll.hash.sha512',
'dll.name',
'dll.path',
'dll.pe.company',
'dll.pe.description',
'dll.pe.file_version',
'dll.pe.imphash',
'dll.pe.original_file_name',
'dll.pe.product',
'dns.Ext.options',
'dns.Ext.status',
'dns.question.name',
'dns.question.registered_domain',
'dns.question.subdomain',
'dns.question.top_level_domain',
'dns.question.type',
'dns.resolved_ip',
'ecs.version',
'event.Ext.correlation.id',
'event.action',
'event.agent_id_status',
'event.category',
'event.code',
'event.created',
'event.dataset',
'event.end',
'event.hash',
'event.id',
'event.ingested',
'event.kind',
'event.module',
'event.outcome',
'event.provider',
'event.sequence',
'event.severity',
'event.start',
'event.type',
'file.Ext.code_signature.exists',
'file.Ext.code_signature.status',
'file.Ext.code_signature.subject_name',
'file.Ext.code_signature.trusted',
'file.Ext.code_signature.valid',
'file.Ext.device.bus_type',
'file.Ext.device.dos_name',
'file.Ext.device.file_system_type',
'file.Ext.device.nt_name',
'file.Ext.device.product_id',
'file.Ext.device.serial_number',
'file.Ext.device.vendor_id',
'file.Ext.device.volume_device_type',
'file.Ext.entropy',
'file.Ext.header_bytes',
'file.Ext.header_data',
'file.Ext.malware_signature.all_names',
'file.Ext.malware_signature.identifier',
'file.Ext.malware_signature.primary.signature.hash.sha256',
'file.Ext.malware_signature.primary.signature.id',
'file.Ext.malware_signature.primary.signature.name',
'file.Ext.malware_signature.version',
'file.Ext.monotonic_id',
'file.Ext.original.extension',
'file.Ext.original.gid',
'file.Ext.original.group',
'file.Ext.original.mode',
'file.Ext.original.name',
'file.Ext.original.owner',
'file.Ext.original.path',
'file.Ext.original.uid',
'file.Ext.windows.zone_identifier',
'file.accessed',
'file.attributes',
'file.code_signature.exists',
'file.code_signature.signing_id',
'file.code_signature.status',
'file.code_signature.subject_name',
'file.code_signature.team_id',
'file.code_signature.trusted',
'file.code_signature.valid',
'file.created',
'file.ctime',
'file.device',
'file.directory',
'file.drive_letter',
'file.extension',
'file.gid',
'file.group',
'file.hash.md5',
'file.hash.sha1',
'file.hash.sha256',
'file.hash.sha512',
'file.inode',
'file.mime_type',
'file.mode',
'file.mtime',
'file.name',
'file.owner',
'file.path',
'file.path.caseless',
'file.path.text',
'file.pe.company',
'file.pe.description',
'file.pe.file_version',
'file.pe.imphash',
'file.pe.original_file_name',
'file.pe.product',
'file.size',
'file.target_path',
'file.target_path.caseless',
'file.target_path.text',
'file.type',
'file.uid',
'group.Ext.real.id',
'group.Ext.real.name',
'group.domain',
'group.id',
'group.name',
'host.architecture',
'host.boot.id',
'host.domain',
'host.hostname',
'host.id',
'host.ip',
'host.mac',
'host.name',
'host.os.Ext.variant',
'host.os.family',
'host.os.full',
'host.os.full.caseless',
'host.os.full.text',
'host.os.kernel',
'host.os.name',
'host.os.name.caseless',
'host.os.name.text',
'host.os.platform',
'host.os.type',
'host.os.version',
'host.pid_ns_ino',
'host.type',
'host.uptime',
'http.request.body.bytes',
'http.request.body.content',
'http.request.body.content.text',
'http.request.bytes',
'http.response.Ext.version',
'http.response.body.bytes',
'http.response.body.content',
'http.response.body.content.text',
'http.response.bytes',
'http.response.status_code',
'message',
'network.bytes',
'network.community_id',
'network.direction',
'network.iana_number',
'network.packets',
'network.protocol',
'network.transport',
'network.type',
'orchestrator.cluster.id',
'orchestrator.cluster.name',
'orchestrator.namespace',
'orchestrator.resource.ip',
'orchestrator.resource.name',
'orchestrator.resource.parent.type',
'orchestrator.resource.type',
'package.name',
'process.Ext.ancestry',
'process.Ext.api.behaviors',
'process.Ext.api.metadata.background_callcount',
'process.Ext.api.metadata.ms_since_last_keyevent',
'process.Ext.api.metadata.procedure_symbol',
'process.Ext.api.metadata.return_value',
'process.Ext.api.metadata.start_address_allocation_protection',
'process.Ext.api.metadata.start_address_module',
'process.Ext.api.metadata.target_address_name',
'process.Ext.api.metadata.target_address_path',
'process.Ext.api.metadata.thread_info_flags',
'process.Ext.api.metadata.visible_windows_count',
'process.Ext.api.metadata.windows_count',
'process.Ext.api.name',
'process.Ext.api.parameters.address',
'process.Ext.api.parameters.allocation_type',
'process.Ext.api.parameters.argument1',
'process.Ext.api.parameters.argument2',
'process.Ext.api.parameters.argument3',
'process.Ext.api.parameters.context_flags',
'process.Ext.api.parameters.desired_access',
'process.Ext.api.parameters.desired_access_numeric',
'process.Ext.api.parameters.device',
'process.Ext.api.parameters.driver',
'process.Ext.api.parameters.eax',
'process.Ext.api.parameters.ebp',
'process.Ext.api.parameters.ebx',
'process.Ext.api.parameters.ecx',
'process.Ext.api.parameters.edi',
'process.Ext.api.parameters.edx',
'process.Ext.api.parameters.eip',
'process.Ext.api.parameters.esi',
'process.Ext.api.parameters.esp',
'process.Ext.api.parameters.flags',
'process.Ext.api.parameters.handle_type',
'process.Ext.api.parameters.hook_module',
'process.Ext.api.parameters.hook_type',
'process.Ext.api.parameters.procedure',
'process.Ext.api.parameters.protection',
'process.Ext.api.parameters.protection_old',
'process.Ext.api.parameters.r8',
'process.Ext.api.parameters.r9',
'process.Ext.api.parameters.rax',
'process.Ext.api.parameters.rbp',
'process.Ext.api.parameters.rbx',
'process.Ext.api.parameters.rcx',
'process.Ext.api.parameters.rdi',
'process.Ext.api.parameters.rdx',
'process.Ext.api.parameters.rip',
'process.Ext.api.parameters.rsi',
'process.Ext.api.parameters.rsp',
'process.Ext.api.parameters.size',
'process.Ext.api.parameters.usage',
'process.Ext.api.parameters.usage_page',
'process.Ext.api.summary',
'process.Ext.architecture',
'process.Ext.authentication_id',
'process.Ext.code_signature.exists',
'process.Ext.code_signature.status',
'process.Ext.code_signature.subject_name',
'process.Ext.code_signature.trusted',
'process.Ext.code_signature.valid',
'process.Ext.defense_evasions',
'process.Ext.device.bus_type',
'process.Ext.device.dos_name',
'process.Ext.device.file_system_type',
'process.Ext.device.nt_name',
'process.Ext.device.product_id',
'process.Ext.device.serial_number',
'process.Ext.device.vendor_id',
'process.Ext.device.volume_device_type',
'process.Ext.dll.Ext.mapped_address',
'process.Ext.dll.Ext.mapped_size',
'process.Ext.dll.name',
'process.Ext.dll.path',
'process.Ext.effective_parent.entity_id',
'process.Ext.effective_parent.executable',
'process.Ext.effective_parent.name',
'process.Ext.effective_parent.pid',
'process.Ext.memory_region.allocation_base',
'process.Ext.memory_region.allocation_protection',
'process.Ext.memory_region.allocation_size',
'process.Ext.memory_region.allocation_type',
'process.Ext.memory_region.bytes_address',
'process.Ext.memory_region.bytes_allocation_offset',
'process.Ext.memory_region.bytes_compressed',
'process.Ext.memory_region.bytes_compressed_present',
'process.Ext.memory_region.hash.sha256',
'process.Ext.memory_region.malware_signature.all_names',
'process.Ext.memory_region.malware_signature.identifier',
'process.Ext.memory_region.malware_signature.primary.matches',
'process.Ext.memory_region.malware_signature.primary.signature.hash.sha256',
'process.Ext.memory_region.malware_signature.primary.signature.id',
'process.Ext.memory_region.malware_signature.primary.signature.name',
'process.Ext.memory_region.malware_signature.version',
'process.Ext.memory_region.mapped_path',
'process.Ext.memory_region.mapped_pe.Ext.dotnet',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.md5',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha1',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha256',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha384',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.sha512',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.ssdeep',
'process.Ext.memory_region.mapped_pe.Ext.sections.hash.tlsh',
'process.Ext.memory_region.mapped_pe.Ext.sections.name',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.md5',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha1',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha256',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha384',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.sha512',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.ssdeep',
'process.Ext.memory_region.mapped_pe.Ext.streams.hash.tlsh',
'process.Ext.memory_region.mapped_pe.Ext.streams.name',
'process.Ext.memory_region.mapped_pe.architecture',
'process.Ext.memory_region.mapped_pe.company',
'process.Ext.memory_region.mapped_pe.description',
'process.Ext.memory_region.mapped_pe.file_version',
'process.Ext.memory_region.mapped_pe.go_import_hash',
'process.Ext.memory_region.mapped_pe.go_imports',
'process.Ext.memory_region.mapped_pe.go_imports_names_entropy',
'process.Ext.memory_region.mapped_pe.go_imports_names_var_entropy',
'process.Ext.memory_region.mapped_pe.go_stripped',
'process.Ext.memory_region.mapped_pe.imphash',
'process.Ext.memory_region.mapped_pe.import_hash',
'process.Ext.memory_region.mapped_pe.imports',
'process.Ext.memory_region.mapped_pe.imports_names_entropy',
'process.Ext.memory_region.mapped_pe.imports_names_var_entropy',
'process.Ext.memory_region.mapped_pe.original_file_name',
'process.Ext.memory_region.mapped_pe.pehash',
'process.Ext.memory_region.mapped_pe.product',
'process.Ext.memory_region.mapped_pe.sections.entropy',
'process.Ext.memory_region.mapped_pe.sections.name',
'process.Ext.memory_region.mapped_pe.sections.physical_size',
'process.Ext.memory_region.mapped_pe.sections.var_entropy',
'process.Ext.memory_region.mapped_pe.sections.virtual_size',
'process.Ext.memory_region.mapped_pe_detected',
'process.Ext.memory_region.memory_pe.Ext.dotnet',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.md5',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.sha1',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.sha256',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.sha384',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.sha512',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.ssdeep',
'process.Ext.memory_region.memory_pe.Ext.sections.hash.tlsh',
'process.Ext.memory_region.memory_pe.Ext.sections.name',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.md5',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.sha1',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.sha256',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.sha384',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.sha512',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.ssdeep',
'process.Ext.memory_region.memory_pe.Ext.streams.hash.tlsh',
'process.Ext.memory_region.memory_pe.Ext.streams.name',
'process.Ext.memory_region.memory_pe.architecture',
'process.Ext.memory_region.memory_pe.company',
'process.Ext.memory_region.memory_pe.description',
'process.Ext.memory_region.memory_pe.file_version',
'process.Ext.memory_region.memory_pe.go_import_hash',
'process.Ext.memory_region.memory_pe.go_imports',
'process.Ext.memory_region.memory_pe.go_imports_names_entropy',
'process.Ext.memory_region.memory_pe.go_imports_names_var_entropy',
'process.Ext.memory_region.memory_pe.go_stripped',
'process.Ext.memory_region.memory_pe.imphash',
'process.Ext.memory_region.memory_pe.import_hash',
'process.Ext.memory_region.memory_pe.imports',
'process.Ext.memory_region.memory_pe.imports_names_entropy',
'process.Ext.memory_region.memory_pe.imports_names_var_entropy',
'process.Ext.memory_region.memory_pe.original_file_name',
'process.Ext.memory_region.memory_pe.pehash',
'process.Ext.memory_region.memory_pe.product',
'process.Ext.memory_region.memory_pe.sections.entropy',
'process.Ext.memory_region.memory_pe.sections.name',
'process.Ext.memory_region.memory_pe.sections.physical_size',
'process.Ext.memory_region.memory_pe.sections.var_entropy',
'process.Ext.memory_region.memory_pe.sections.virtual_size',
'process.Ext.memory_region.memory_pe_detected',
'process.Ext.memory_region.region_base',
'process.Ext.memory_region.region_protection',
'process.Ext.memory_region.region_size',
'process.Ext.memory_region.region_state',
'process.Ext.memory_region.strings',
'process.Ext.mitigation_policies',
'process.Ext.protection',
'process.Ext.relative_file_creation_time',
'process.Ext.relative_file_name_modify_time',
'process.Ext.session',
'process.Ext.session_info.authentication_package',
'process.Ext.session_info.client_address',
'process.Ext.session_info.id',
'process.Ext.session_info.logon_type',
'process.Ext.session_info.relative_logon_time',
'process.Ext.session_info.relative_password_age',
'process.Ext.session_info.user_flags',
'process.Ext.token.elevation',
'process.Ext.token.elevation_level',
'process.Ext.token.elevation_type',
'process.Ext.token.integrity_level_name',
'process.Ext.token.security_attributes',
'process.Ext.trusted',
'process.Ext.trusted_descendant',
'process.args',
'process.args_count',
'process.code_signature.exists',
'process.code_signature.signing_id',
'process.code_signature.status',
'process.code_signature.subject_name',
'process.code_signature.team_id',
'process.code_signature.trusted',
'process.code_signature.valid',
'process.command_line',
'process.command_line.caseless',
'process.command_line.text',
'process.end',
'process.entity_id',
'process.entry_leader.args',
'process.entry_leader.args_count',
'process.entry_leader.attested_groups.name',
'process.entry_leader.attested_user.id',
'process.entry_leader.attested_user.name',
'process.entry_leader.attested_user.name.text',
'process.entry_leader.command_line',
'process.entry_leader.command_line.caseless',
'process.entry_leader.command_line.text',
'process.entry_leader.entity_id',
'process.entry_leader.entry_meta.source.ip',
'process.entry_leader.entry_meta.type',
'process.entry_leader.executable',
'process.entry_leader.executable.caseless',
'process.entry_leader.executable.text',
'process.entry_leader.group.id',
'process.entry_leader.group.name',
'process.entry_leader.interactive',
'process.entry_leader.name',
'process.entry_leader.name.caseless',
'process.entry_leader.name.text',
'process.entry_leader.parent.entity_id',
'process.entry_leader.parent.pid',
'process.entry_leader.parent.session_leader.entity_id',
'process.entry_leader.parent.session_leader.pid',
'process.entry_leader.parent.session_leader.start',
'process.entry_leader.parent.start',
'process.entry_leader.pid',
'process.entry_leader.real_group.id',
'process.entry_leader.real_group.name',
'process.entry_leader.real_user.id',
'process.entry_leader.real_user.name',
'process.entry_leader.real_user.name.text',
'process.entry_leader.same_as_process',
'process.entry_leader.saved_group.id',
'process.entry_leader.saved_group.name',
'process.entry_leader.saved_user.id',
'process.entry_leader.saved_user.name',
'process.entry_leader.saved_user.name.text',
'process.entry_leader.start',
'process.entry_leader.supplemental_groups.id',
'process.entry_leader.supplemental_groups.name',
'process.entry_leader.tty.char_device.major',
'process.entry_leader.tty.char_device.minor',
'process.entry_leader.user.id',
'process.entry_leader.user.name',
'process.entry_leader.user.name.text',
'process.entry_leader.working_directory',
'process.entry_leader.working_directory.caseless',
'process.entry_leader.working_directory.text',
'process.env_vars',
'process.executable',
'process.executable.caseless',
'process.executable.text',
'process.exit_code',
'process.group.id',
'process.group.name',
'process.group_leader.args',
'process.group_leader.args_count',
'process.group_leader.command_line',
'process.group_leader.command_line.caseless',
'process.group_leader.command_line.text',
'process.group_leader.entity_id',
'process.group_leader.executable',
'process.group_leader.executable.caseless',
'process.group_leader.executable.text',
'process.group_leader.group.id',
'process.group_leader.group.name',
'process.group_leader.interactive',
'process.group_leader.name',
'process.group_leader.name.caseless',
'process.group_leader.name.text',
'process.group_leader.pid',
'process.group_leader.real_group.id',
'process.group_leader.real_group.name',
'process.group_leader.real_user.id',
'process.group_leader.real_user.name',
'process.group_leader.real_user.name.text',
'process.group_leader.same_as_process',
'process.group_leader.saved_group.id',
'process.group_leader.saved_group.name',
'process.group_leader.saved_user.id',
'process.group_leader.saved_user.name',
'process.group_leader.saved_user.name.text',
'process.group_leader.start',
'process.group_leader.supplemental_groups.id',
'process.group_leader.supplemental_groups.name',
'process.group_leader.tty.char_device.major',
'process.group_leader.tty.char_device.minor',
'process.group_leader.user.id',
'process.group_leader.user.name',
'process.group_leader.user.name.text',
'process.group_leader.working_directory',
'process.group_leader.working_directory.caseless',
'process.group_leader.working_directory.text',
'process.hash.md5',
'process.hash.sha1',
'process.hash.sha256',
'process.hash.sha512',
'process.interactive',
'process.io.max_bytes_per_process_exceeded',
'process.io.text',
'process.io.total_bytes_captured',
'process.io.total_bytes_skipped',
'process.name',
'process.name.caseless',
'process.name.text',
'process.parent.Ext.architecture',
'process.parent.Ext.code_signature.exists',
'process.parent.Ext.code_signature.status',
'process.parent.Ext.code_signature.subject_name',
'process.parent.Ext.code_signature.trusted',
'process.parent.Ext.code_signature.valid',
'process.parent.Ext.protection',
'process.parent.Ext.real.pid',
'process.parent.Ext.user',
'process.parent.args',
'process.parent.args_count',
'process.parent.code_signature.exists',
'process.parent.code_signature.signing_id',
'process.parent.code_signature.status',
'process.parent.code_signature.subject_name',
'process.parent.code_signature.team_id',
'process.parent.code_signature.trusted',
'process.parent.code_signature.valid',
'process.parent.command_line',
'process.parent.command_line.caseless',
'process.parent.command_line.text',
'process.parent.entity_id',
'process.parent.executable',
'process.parent.executable.caseless',
'process.parent.executable.text',
'process.parent.exit_code',
'process.parent.group.id',
'process.parent.group.name',
'process.parent.group_leader.entity_id',
'process.parent.group_leader.pid',
'process.parent.group_leader.start',
'process.parent.hash.md5',
'process.parent.hash.sha1',
'process.parent.hash.sha256',
'process.parent.hash.sha512',
'process.parent.interactive',
'process.parent.name',
'process.parent.name.caseless',
'process.parent.name.text',
'process.parent.pe.company',
'process.parent.pe.description',
'process.parent.pe.file_version',
'process.parent.pe.imphash',
'process.parent.pe.original_file_name',
'process.parent.pe.product',
'process.parent.pgid',
'process.parent.pid',
'process.parent.ppid',
'process.parent.real_group.id',
'process.parent.real_group.name',
'process.parent.real_user.id',
'process.parent.real_user.name',
'process.parent.real_user.name.text',
'process.parent.saved_group.id',
'process.parent.saved_group.name',
'process.parent.saved_user.id',
'process.parent.saved_user.name',
'process.parent.saved_user.name.text',
'process.parent.start',
'process.parent.supplemental_groups.id',
'process.parent.supplemental_groups.name',
'process.parent.thread.Ext.call_stack.allocation_private_bytes',
'process.parent.thread.Ext.call_stack.callsite_leading_bytes',
'process.parent.thread.Ext.call_stack.callsite_trailing_bytes',
'process.parent.thread.Ext.call_stack.protection',
'process.parent.thread.Ext.call_stack.symbol_info',
'process.parent.thread.Ext.call_stack_contains_unbacked',
'process.parent.thread.Ext.call_stack_summary',
'process.parent.thread.id',
'process.parent.thread.name',
'process.parent.title',
'process.parent.title.text',
'process.parent.tty.char_device.major',
'process.parent.tty.char_device.minor',
'process.parent.uptime',
'process.parent.user.id',
'process.parent.user.name',
'process.parent.user.name.text',
'process.parent.working_directory',
'process.parent.working_directory.caseless',
'process.parent.working_directory.text',
'process.pe.company',
'process.pe.description',
'process.pe.file_version',
'process.pe.imphash',
'process.pe.original_file_name',
'process.pe.product',
'process.pgid',
'process.pid',
'process.ppid',
'process.previous.args',
'process.previous.args_count',
'process.previous.executable',
'process.previous.executable.caseless',
'process.previous.executable.text',
'process.real_group.id',
'process.real_group.name',
'process.real_user.id',
'process.real_user.name',
'process.real_user.name.text',
'process.saved_group.id',
'process.saved_group.name',
'process.saved_user.id',
'process.saved_user.name',
'process.saved_user.name.text',
'process.session_leader.args',
'process.session_leader.args_count',
'process.session_leader.command_line',
'process.session_leader.command_line.caseless',
'process.session_leader.command_line.text',
'process.session_leader.entity_id',
'process.session_leader.executable',
'process.session_leader.executable.caseless',
'process.session_leader.executable.text',
'process.session_leader.group.id',
'process.session_leader.group.name',
'process.session_leader.interactive',
'process.session_leader.name',
'process.session_leader.name.caseless',
'process.session_leader.name.text',
'process.session_leader.parent.entity_id',
'process.session_leader.parent.pid',
'process.session_leader.parent.session_leader.entity_id',
'process.session_leader.parent.session_leader.pid',
'process.session_leader.parent.session_leader.start',
'process.session_leader.parent.start',
'process.session_leader.pid',
'process.session_leader.real_group.id',
'process.session_leader.real_group.name',
'process.session_leader.real_user.id',
'process.session_leader.real_user.name',
'process.session_leader.real_user.name.text',
'process.session_leader.same_as_process',
'process.session_leader.saved_group.id',
'process.session_leader.saved_group.name',
'process.session_leader.saved_user.id',
'process.session_leader.saved_user.name',
'process.session_leader.saved_user.name.text',
'process.session_leader.start',
'process.session_leader.supplemental_groups.id',
'process.session_leader.supplemental_groups.name',
'process.session_leader.tty.char_device.major',
'process.session_leader.tty.char_device.minor',
'process.session_leader.user.id',
'process.session_leader.user.name',
'process.session_leader.user.name.text',
'process.session_leader.working_directory',
'process.session_leader.working_directory.caseless',
'process.session_leader.working_directory.text',
'process.start',
'process.supplemental_groups.id',
'process.supplemental_groups.name',
'process.thread.Ext.call_stack.allocation_private_bytes',
'process.thread.Ext.call_stack.callsite_leading_bytes',
'process.thread.Ext.call_stack.callsite_trailing_bytes',
'process.thread.Ext.call_stack.instruction_pointer',
'process.thread.Ext.call_stack.module_path',
'process.thread.Ext.call_stack.protection',
'process.thread.Ext.call_stack.protection_provenance',
'process.thread.Ext.call_stack.symbol_info',
'process.thread.Ext.call_stack_contains_unbacked',
'process.thread.Ext.call_stack_final_user_module.code_signature.exists',
'process.thread.Ext.call_stack_final_user_module.code_signature.status',
'process.thread.Ext.call_stack_final_user_module.code_signature.subject_name',
'process.thread.Ext.call_stack_final_user_module.code_signature.trusted',
'process.thread.Ext.call_stack_final_user_module.code_signature.valid',
'process.thread.Ext.call_stack_final_user_module.hash.sha256',
'process.thread.Ext.call_stack_final_user_module.name',
'process.thread.Ext.call_stack_final_user_module.path',
'process.thread.Ext.call_stack_final_user_module.protection_provenance',
'process.thread.Ext.call_stack_summary',
'process.thread.capabilities.effective',
'process.thread.capabilities.permitted',
'process.thread.id',
'process.thread.name',
'process.title',
'process.title.text',
'process.tty.char_device.major',
'process.tty.char_device.minor',
'process.tty.columns',
'process.tty.rows',
'process.uptime',
'process.user.id',
'process.user.name',
'process.user.name.text',
'process.working_directory',
'process.working_directory.caseless',
'process.working_directory.text',
'registry.data.bytes',
'registry.data.strings',
'registry.data.type',
'registry.hive',
'registry.key',
'registry.path',
'registry.value',
'source.address',
'source.as.number',
'source.as.organization.name',
'source.as.organization.name.text',
'source.bytes',
'source.domain',
'source.geo.city_name',
'source.geo.continent_code',
'source.geo.continent_name',
'source.geo.country_iso_code',
'source.geo.country_name',
'source.geo.location',
'source.geo.name',
'source.geo.postal_code',
'source.geo.region_iso_code',
'source.geo.region_name',
'source.geo.timezone',
'source.ip',
'source.packets',
'source.port',
'source.registered_domain',
'source.top_level_domain',
'user.Ext.real.id',
'user.Ext.real.name',
'user.domain',
'user.email',
'user.full_name',
'user.full_name.text',
'user.group.Ext.real.id',
'user.group.Ext.real.name',
'user.group.domain',
'user.group.id',
'user.group.name',
'user.hash',
'user.id',
'user.name',
'user.name.text',
];

23
x-pack/plugins/security_solution/common/endpoint/data_generators/common/artifact_list_item_entry_values.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ListOperator, ListOperatorType } from '@kbn/securitysolution-io-ts-list-types';
import { ListOperatorEnum, ListOperatorTypeEnum } from '@kbn/securitysolution-io-ts-list-types';
export const LIST_ITEM_ENTRY_OPERATOR_TYPES: readonly ListOperatorType[] = Object.freeze([
ListOperatorTypeEnum.NESTED,
ListOperatorTypeEnum.MATCH,
ListOperatorTypeEnum.MATCH_ANY,
ListOperatorTypeEnum.WILDCARD,
ListOperatorTypeEnum.EXISTS,
ListOperatorTypeEnum.LIST,
]);
export const LIST_ITEM_ENTRY_OPERATOR: readonly ListOperator[] = Object.freeze([
ListOperatorEnum.INCLUDED,
ListOperatorEnum.EXCLUDED,
]);

85
x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_exceptions_generator.ts Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
CreateExceptionListItemSchema,
ExceptionListItemSchema,
ListOperatorType,
} from '@kbn/securitysolution-io-ts-list-types';
import { ListOperatorTypeEnum } from '@kbn/securitysolution-io-ts-list-types';
import { ENDPOINT_LIST_ID } from '@kbn/securitysolution-list-constants';
import { LIST_ITEM_ENTRY_OPERATOR_TYPES } from './common/artifact_list_item_entry_values';
import { exceptionItemToCreateExceptionItem } from './exceptions_list_item_generator';
import { BaseDataGenerator } from './base_data_generator';
import { GLOBAL_ARTIFACT_TAG } from '../service/artifacts';
import { ENDPOINT_EVENTS_LOG_INDEX_FIELDS } from './common/alerts_ecs_fields';
export class EndpointExceptionsGenerator extends BaseDataGenerator<ExceptionListItemSchema> {
generate(overrides: Partial<ExceptionListItemSchema> = {}): ExceptionListItemSchema {
return {
name: `Generated Exception (${this.randomString(5)})`,
comments: [],
description: 'created by EndpointExceptionsGenerator',
id: this.seededUUIDv4(),
item_id: this.seededUUIDv4(),
list_id: ENDPOINT_LIST_ID,
tags: [GLOBAL_ARTIFACT_TAG],
entries: this.randomEndpointExceptionEntries(1),
meta: undefined,
namespace_type: 'agnostic',
os_types: [this.randomOSFamily()] as ExceptionListItemSchema['os_types'],
created_at: this.randomPastDate(),
created_by: this.randomUser(),
updated_at: '2020-04-20T15:25:31.830Z',
expire_time: undefined,
updated_by: this.randomUser(),
_version: this.randomString(5),
type: 'simple',
tie_breaker_id: this.seededUUIDv4(),
...overrides,
};
}
generateEndpointExceptionForCreate(
overrides: Partial<CreateExceptionListItemSchema> = {}
): CreateExceptionListItemSchema {
return {
...exceptionItemToCreateExceptionItem(this.generate()),
...overrides,
};
}
protected randomEndpointExceptionEntries(
count: number = this.randomN(5)
): ExceptionListItemSchema['entries'] {
const operatorTypes = LIST_ITEM_ENTRY_OPERATOR_TYPES.filter(
(item) =>
!(
[
ListOperatorTypeEnum.LIST,
ListOperatorTypeEnum.NESTED,
ListOperatorTypeEnum.EXISTS,
] as ListOperatorType[]
).includes(item)
);
const fieldList = ENDPOINT_EVENTS_LOG_INDEX_FIELDS.filter((field) => field.endsWith('.text'));
return Array.from({ length: count || 1 }, () => {
const operatorType = this.randomChoice(operatorTypes);
return {
field: this.randomChoice(fieldList),
operator: 'included',
type: operatorType,
value:
operatorType === ListOperatorTypeEnum.MATCH_ANY
? [this.randomString(10), this.randomString(10)]
: this.randomString(10),
};
}) as ExceptionListItemSchema['entries'];
}
}

85
x-pack/plugins/security_solution/common/endpoint/data_generators/event_filters_generator.ts Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
ExceptionListItemSchema,
CreateExceptionListItemSchema,
ListOperator,
} from '@kbn/securitysolution-io-ts-list-types';
import { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
import { exceptionItemToCreateExceptionItem } from './exceptions_list_item_generator';
import { GLOBAL_ARTIFACT_TAG } from '../service/artifacts';
import { BaseDataGenerator } from './base_data_generator';
import { ENDPOINT_EVENTS_LOG_INDEX_FIELDS } from './common/alerts_ecs_fields';
const ENTRY_OPERATORS: readonly ListOperator[] = ['included', 'excluded'];
export class EventFiltersGenerator extends BaseDataGenerator<ExceptionListItemSchema> {
generate(overrides: Partial<ExceptionListItemSchema> = {}): ExceptionListItemSchema {
return {
id: this.seededUUIDv4(),
item_id: this.seededUUIDv4(),
list_id: ENDPOINT_ARTIFACT_LISTS.eventFilters.id,
meta: undefined,
name: `Event filter (${this.randomString(5)})`,
description: `created by ${this.constructor.name}`,
tags: [GLOBAL_ARTIFACT_TAG],
entries: this.randomEventFilterEntries(),
expire_time: undefined,
namespace_type: 'agnostic',
type: 'simple',
os_types: [this.randomOSFamily()] as ExceptionListItemSchema['os_types'],
tie_breaker_id: this.seededUUIDv4(),
_version: this.randomString(5),
comments: [],
created_at: this.randomPastDate(),
created_by: this.randomUser(),
updated_at: '2020-04-20T15:25:31.830Z',
updated_by: this.randomUser(),
...overrides,
};
}
generateEventFilterForCreate(
overrides: Partial<CreateExceptionListItemSchema> = {}
): CreateExceptionListItemSchema {
return {
...exceptionItemToCreateExceptionItem(this.generate()),
...overrides,
};
}
protected randomEventFilterEntries(
count: number = this.randomN(5)
): ExceptionListItemSchema['entries'] {
return Array.from({ length: count || 1 }, () => {
if (this.randomBoolean()) {
// single entry
return {
field: this.randomChoice(ENDPOINT_EVENTS_LOG_INDEX_FIELDS),
operator: this.randomChoice(ENTRY_OPERATORS),
type: 'match',
value: this.randomString(10),
};
} else {
// nested entry
return {
field: this.randomChoice(ENDPOINT_EVENTS_LOG_INDEX_FIELDS),
type: 'nested',
entries: [
{
field: this.randomChoice(ENDPOINT_EVENTS_LOG_INDEX_FIELDS),
operator: this.randomChoice(ENTRY_OPERATORS),
type: 'match',
value: this.randomString(10),
},
],
};
}
});
}
}

2
x-pack/plugins/security_solution/common/endpoint/data_generators/exceptions_list_item_generator.ts
View file

@ -39,7 +39,7 @@ type UpdateExceptionListItemSchemaWithNonNullProps = NonNullableTypeProperties<
> & > &
Pick<UpdateExceptionListItemSchema, 'meta' | 'expire_time'>; Pick<UpdateExceptionListItemSchema, 'meta' | 'expire_time'>;
const exceptionItemToCreateExceptionItem = ( export const exceptionItemToCreateExceptionItem = (
exceptionItem: ExceptionListItemSchema exceptionItem: ExceptionListItemSchema
): CreateExceptionListItemSchemaWithNonNullProps => { ): CreateExceptionListItemSchemaWithNonNullProps => {
const { const {

4
x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts
View file

@ -247,4 +247,8 @@ export class FleetAgentGenerator extends BaseDataGenerator<Agent> {
public randomAgentStatus() { public randomAgentStatus() {
return this.randomChoice(agentStatusList); return this.randomChoice(agentStatusList);
} }
public randomString(length: number = 5) {
return super.randomString(length);
}
} }

2
x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts
View file

@ -104,7 +104,7 @@ export const setupFleetForEndpoint = usageTracker.track(
export const installOrUpgradeEndpointFleetPackage = usageTracker.track( export const installOrUpgradeEndpointFleetPackage = usageTracker.track(
'installOrUpgradeEndpointFleetPackage', 'installOrUpgradeEndpointFleetPackage',
async (kbnClient: KbnClient, logger: ToolingLog): Promise<BulkInstallPackageInfo> => { async (kbnClient: KbnClient, logger: ToolingLog): Promise<BulkInstallPackageInfo> => {
logger.info(`installOrUpgradeEndpointFleetPackage(): starting`); logger.debug(`installOrUpgradeEndpointFleetPackage(): starting`);
const updatePackages = async () => { const updatePackages = async () => {
const installEndpointPackageResp = (await kbnClient const installEndpointPackageResp = (await kbnClient

30
x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts
View file

@ -9,6 +9,7 @@ import { mergeWith } from 'lodash';
import type { ToolingLogTextWriterConfig } from '@kbn/tooling-log'; import type { ToolingLogTextWriterConfig } from '@kbn/tooling-log';
import { ToolingLog } from '@kbn/tooling-log'; import { ToolingLog } from '@kbn/tooling-log';
import type { Flags } from '@kbn/dev-cli-runner'; import type { Flags } from '@kbn/dev-cli-runner';
import moment from 'moment/moment';
export const RETRYABLE_TRANSIENT_ERRORS: Readonly<Array<string | RegExp>> = [ export const RETRYABLE_TRANSIENT_ERRORS: Readonly<Array<string | RegExp>> = [
'no_shard_available_action_exception', 'no_shard_available_action_exception',
@ -76,18 +77,18 @@ export const retryOnError = async <T>(
const thisAttempt = attempt; const thisAttempt = attempt;
attempt++; attempt++;
log.info(msg(`attempt ${thisAttempt} started at: ${new Date().toISOString()}`)); log.debug(msg(`attempt ${thisAttempt} started at: ${new Date().toISOString()}`));
try { try {
responsePromise = callback(); // store promise so that if it fails and no more attempts, we return the last failure responsePromise = callback(); // store promise so that if it fails and no more attempts, we return the last failure
const result = await responsePromise; const result = await responsePromise;
log.info(msg(`attempt ${thisAttempt} was successful. Exiting retry`)); log.debug(msg(`attempt ${thisAttempt} was successful. Exiting retry`));
log.indent(-4); log.indent(-4);
return result; return result;
} catch (err) { } catch (err) {
log.info(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err); log.warning(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err);
// If not an error that is retryable, then end loop here and return that error; // If not an error that is retryable, then end loop here and return that error;
if (!isRetryableError(err)) { if (!isRetryableError(err)) {
@ -157,3 +158,26 @@ createToolingLogger.setDefaultLogLevelFromCliFlags = (flags) => {
? 'error' ? 'error'
: 'info'; : 'info';
}; };
/**
* Get human readable string of time elapsed between to dates. Return value will be in the format
* of `hh:mm:ss.ms`
* @param startDate
* @param endTime
*/
export const getElapsedTime = (
startDate: string | Date,
endTime: string | Date = new Date()
): string => {
const durationObj = moment.duration(moment(endTime).diff(startDate));
const pad = (num: number, max = 2): string => {
return String(num).padStart(max, '0');
};
const hours = pad(durationObj.hours());
const minutes = pad(durationObj.minutes());
const seconds = pad(durationObj.seconds());
const milliseconds = pad(durationObj.milliseconds(), 3);
return `${hours}:${minutes}:${seconds}.${milliseconds}`;
};

149
x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_artifact_services.ts Normal file
View file

@ -0,0 +1,149 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ENDPOINT_ARTIFACT_LISTS } from '@kbn/securitysolution-list-constants';
import {
ENDPOINT_LIST_DESCRIPTION,
ENDPOINT_LIST_ID,
ENDPOINT_LIST_NAME,
EXCEPTION_LIST_ITEM_URL,
INTERNAL_EXCEPTIONS_LIST_ENSURE_CREATED_URL,
} from '@kbn/securitysolution-list-constants';
import type { KbnClient } from '@kbn/test';
import type {
CreateExceptionListSchema,
ExceptionListItemSchema,
CreateExceptionListItemSchema,
} from '@kbn/securitysolution-io-ts-list-types';
import { memoize } from 'lodash';
import { ExceptionListTypeEnum } from '@kbn/securitysolution-io-ts-list-types';
import { catchAxiosErrorFormatAndThrow } from '../../../common/endpoint/format_axios_error';
import { TRUSTED_APPS_EXCEPTION_LIST_DEFINITION } from '../../../public/management/pages/trusted_apps/constants';
import { EVENT_FILTER_LIST_DEFINITION } from '../../../public/management/pages/event_filters/constants';
import { BLOCKLISTS_LIST_DEFINITION } from '../../../public/management/pages/blocklist/constants';
import { HOST_ISOLATION_EXCEPTIONS_LIST_DEFINITION } from '../../../public/management/pages/host_isolation_exceptions/constants';
import type { NewTrustedApp } from '../../../common/endpoint/types';
import { newTrustedAppToCreateExceptionListItem } from '../../../public/management/pages/trusted_apps/service/mappers';
const ensureArtifactListExists = memoize(
async (
kbnClient: KbnClient,
artifactType: keyof typeof ENDPOINT_ARTIFACT_LISTS | 'endpointExceptions'
) => {
let listDefinition: CreateExceptionListSchema;
switch (artifactType) {
case 'blocklists':
listDefinition = BLOCKLISTS_LIST_DEFINITION;
break;
case 'eventFilters':
listDefinition = EVENT_FILTER_LIST_DEFINITION;
break;
case 'hostIsolationExceptions':
listDefinition = HOST_ISOLATION_EXCEPTIONS_LIST_DEFINITION;
break;
case 'trustedApps':
listDefinition = TRUSTED_APPS_EXCEPTION_LIST_DEFINITION;
break;
case 'endpointExceptions':
listDefinition = {
name: ENDPOINT_LIST_NAME,
namespace_type: 'agnostic',
description: ENDPOINT_LIST_DESCRIPTION,
list_id: ENDPOINT_LIST_ID,
type: ExceptionListTypeEnum.ENDPOINT,
};
default:
throw new Error(`Unknown Artifact list: ${artifactType}`);
}
await kbnClient
.request({
method: 'POST',
path: INTERNAL_EXCEPTIONS_LIST_ENSURE_CREATED_URL,
body: listDefinition,
headers: {
'elastic-api-version': '1',
},
})
.catch(catchAxiosErrorFormatAndThrow);
},
(kbnClient: KbnClient, artifactType: string) => {
return `${artifactType}@[${kbnClient.resolveUrl('')}`;
}
);
/**
* Creates an exception list item.
* NOTE: this method does NOT create the list itself.
*
* @private
*
* @param kbnClient
* @param data
*/
const createExceptionListItem = async (
kbnClient: KbnClient,
data: CreateExceptionListItemSchema
): Promise<ExceptionListItemSchema> => {
return kbnClient
.request<ExceptionListItemSchema>({
method: 'POST',
path: EXCEPTION_LIST_ITEM_URL,
body: data,
headers: {
'elastic-api-version': '2023-10-31',
},
})
.catch(catchAxiosErrorFormatAndThrow)
.then((response) => response.data);
};
export const createTrustedApp = async (
kbnClient: KbnClient,
data: NewTrustedApp
): Promise<ExceptionListItemSchema> => {
await ensureArtifactListExists(kbnClient, 'trustedApps');
return createExceptionListItem(kbnClient, newTrustedAppToCreateExceptionListItem(data));
};
export const createEventFilter = async (
kbnClient: KbnClient,
data: CreateExceptionListItemSchema
): Promise<ExceptionListItemSchema> => {
await ensureArtifactListExists(kbnClient, 'eventFilters');
return createExceptionListItem(kbnClient, data);
};
export const createBlocklist = async (
kbnClient: KbnClient,
data: CreateExceptionListItemSchema
): Promise<ExceptionListItemSchema> => {
await ensureArtifactListExists(kbnClient, 'blocklists');
return createExceptionListItem(kbnClient, data);
};
export const createHostIsolationException = async (
kbnClient: KbnClient,
data: CreateExceptionListItemSchema
): Promise<ExceptionListItemSchema> => {
await ensureArtifactListExists(kbnClient, 'hostIsolationExceptions');
return createExceptionListItem(kbnClient, data);
};
export const createEndpointException = async (
kbnClient: KbnClient,
data: CreateExceptionListItemSchema
): Promise<ExceptionListItemSchema> => {
await ensureArtifactListExists(kbnClient, 'hostIsolationExceptions');
return createExceptionListItem(kbnClient, data);
};

79
x-pack/plugins/security_solution/scripts/endpoint/common/execution_throttler.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/* eslint-disable @typescript-eslint/no-explicit-any */
import type { ToolingLog } from '@kbn/tooling-log';
import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils';
export interface ExecutionThrottlerOptions {
concurrency?: number;
log?: ToolingLog;
}
/**
* Queue callback functions and execute them in parallel using the defined concurrency number. Execution of
* callbacks starts as soon as functions start to be added to the queue (ex. Does not wait for a "batch"
* size to be reached) and continues to process the queue until it is flushed out.
*/
export class ExecutionThrottler {
private readonly options: Required<ExecutionThrottlerOptions>;
private readonly queue: Array<() => Promise<any>> = [];
private readonly executing = new Set<Promise<any>>();
constructor({ concurrency = 10, log = createToolingLogger() }: ExecutionThrottlerOptions = {}) {
this.options = {
concurrency,
log,
};
}
private logCurrentState() {
this.options.log.debug(
`Executing count: [${this.executing.size}], Queue count: [${this.queue.length}]`
);
}
private async execute(): Promise<void> {
if (this.queue.length === 0) {
return;
}
while (this.executing.size < this.options.concurrency && this.queue.length) {
const callbackFn = this.queue.shift();
if (callbackFn) {
const callbackPromise = callbackFn();
this.executing.add(callbackPromise);
callbackPromise.finally(() => {
this.executing.delete(callbackPromise);
this.execute();
});
}
}
this.logCurrentState();
}
public addToQueue(fn: () => Promise<any>): void {
this.queue.push(fn);
this.execute();
}
public async complete(): Promise<void> {
while (this.executing.size > 0) {
this.logCurrentState();
await Promise.all(Array.from(this.executing)).catch(() => {});
}
}
public getStats(): { pending: number } {
return {
pending: this.queue.length,
};
}
}

139
x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts
View file

@ -24,6 +24,7 @@ import type {
GetPackagePoliciesResponse, GetPackagePoliciesResponse,
PackagePolicy, PackagePolicy,
PostFleetSetupResponse, PostFleetSetupResponse,
CopyAgentPolicyResponse,
} from '@kbn/fleet-plugin/common'; } from '@kbn/fleet-plugin/common';
import { import {
AGENT_API_ROUTES, AGENT_API_ROUTES,
@ -37,6 +38,7 @@ import {
epmRouteService, epmRouteService,
PACKAGE_POLICY_API_ROUTES, PACKAGE_POLICY_API_ROUTES,
SETUP_API_ROUTE, SETUP_API_ROUTE,
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
} from '@kbn/fleet-plugin/common'; } from '@kbn/fleet-plugin/common';
import type { ToolingLog } from '@kbn/tooling-log'; import type { ToolingLog } from '@kbn/tooling-log';
import type { KbnClient } from '@kbn/test'; import type { KbnClient } from '@kbn/test';
@ -54,6 +56,7 @@ import type {
GetEnrollmentAPIKeysResponse, GetEnrollmentAPIKeysResponse,
GetOutputsResponse, GetOutputsResponse,
PostAgentUnenrollResponse, PostAgentUnenrollResponse,
CopyAgentPolicyRequest,
} from '@kbn/fleet-plugin/common/types'; } from '@kbn/fleet-plugin/common/types';
import nodeFetch from 'node-fetch'; import nodeFetch from 'node-fetch';
import semver from 'semver'; import semver from 'semver';
@ -79,6 +82,14 @@ const DEFAULT_AGENT_POLICY_NAME = `${CURRENT_USERNAME} test policy`;
/** A Fleet agent policy that includes integrations that don't actually require an agent to run on a host. Example: SenttinelOne */ /** A Fleet agent policy that includes integrations that don't actually require an agent to run on a host. Example: SenttinelOne */
export const DEFAULT_AGENTLESS_INTEGRATIONS_AGENT_POLICY_NAME = `${CURRENT_USERNAME} - agentless integrations`; export const DEFAULT_AGENTLESS_INTEGRATIONS_AGENT_POLICY_NAME = `${CURRENT_USERNAME} - agentless integrations`;
const randomAgentPolicyName = (() => {
let counter = fleetGenerator.randomN(100);
return (): string => {
return `agent policy - ${fleetGenerator.randomString(10)}_${counter++}`;
};
})();
export const checkInFleetAgent = async ( export const checkInFleetAgent = async (
esClient: Client, esClient: Client,
agentId: string, agentId: string,
@ -753,6 +764,41 @@ export const enrollHostVmWithFleet = async ({
return waitForHostToEnroll(kbnClient, log, hostVm.name, timeoutMs); return waitForHostToEnroll(kbnClient, log, hostVm.name, timeoutMs);
}; };
interface CreateAgentPolicyOptions {
kbnClient: KbnClient;
policy?: CreateAgentPolicyRequest['body'];
}
/**
* Create a new Agent Policy in fleet
* @param kbnClient
* @param log
* @param policy
*/
export const createAgentPolicy = async ({
kbnClient,
policy,
}: CreateAgentPolicyOptions): Promise<AgentPolicy> => {
const body: CreateAgentPolicyRequest['body'] = policy ?? {
name: randomAgentPolicyName(),
description: `Policy created by security solution tooling: ${__filename}`,
namespace: 'default',
monitoring_enabled: ['logs', 'metrics'],
};
return kbnClient
.request<CreateAgentPolicyResponse>({
path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
headers: {
'elastic-api-version': API_VERSIONS.public.v1,
},
method: 'POST',
body,
})
.then((response) => response.data.item)
.catch(catchAxiosErrorFormatAndThrow);
};
interface GetOrCreateDefaultAgentPolicyOptions { interface GetOrCreateDefaultAgentPolicyOptions {
kbnClient: KbnClient; kbnClient: KbnClient;
log: ToolingLog; log: ToolingLog;
@ -784,24 +830,15 @@ export const getOrCreateDefaultAgentPolicy = async ({
log.info(`Creating default test/dev Fleet agent policy with name: [${policyName}]`); log.info(`Creating default test/dev Fleet agent policy with name: [${policyName}]`);
const newAgentPolicyData: CreateAgentPolicyRequest['body'] = { const newAgentPolicy = await createAgentPolicy({
kbnClient,
policy: {
name: policyName, name: policyName,
description: `Policy created by security solution tooling: ${__filename}`, description: `Policy created by security solution tooling: ${__filename}`,
namespace: 'default', namespace: 'default',
monitoring_enabled: ['logs', 'metrics'], monitoring_enabled: ['logs', 'metrics'],
};
const newAgentPolicy = await kbnClient
.request<CreateAgentPolicyResponse>({
path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
headers: {
'elastic-api-version': API_VERSIONS.public.v1,
}, },
method: 'POST', });
body: newAgentPolicyData,
})
.then((response) => response.data.item)
.catch(catchAxiosErrorFormatAndThrow);
log.verbose(newAgentPolicy); log.verbose(newAgentPolicy);
@ -1182,6 +1219,40 @@ export const addEndpointIntegrationToAgentPolicy = async ({
return newIntegrationPolicy; return newIntegrationPolicy;
}; };
type CopyAgentPolicyOptions = Partial<CopyAgentPolicyRequest['body']> & {
kbnClient: KbnClient;
agentPolicyId: string;
};
/**
* Copy (clone) a Fleet Agent Policy
* @param kbnClient
* @param agentPolicyId
* @param name
* @param description
*/
export const copyAgentPolicy = async ({
kbnClient,
agentPolicyId,
name = randomAgentPolicyName(),
description,
}: CopyAgentPolicyOptions) => {
return kbnClient
.request<CopyAgentPolicyResponse>({
path: agentPolicyRouteService.getCopyPath(agentPolicyId),
headers: {
'elastic-api-version': API_VERSIONS.public.v1,
},
method: 'POST',
body: {
name,
description,
},
})
.then((response) => response.data.item)
.catch(catchAxiosErrorFormatAndThrow);
};
/** /**
* Calls the fleet setup API to ensure fleet configured with default settings * Calls the fleet setup API to ensure fleet configured with default settings
* @param kbnClient * @param kbnClient
@ -1205,3 +1276,45 @@ export const ensureFleetSetup = memoize(
return setupResponse.data; return setupResponse.data;
} }
); );
/**
* Fetches a list of Endpoint Integration policies from fleet
* @param kbnClient
* @param kuery
* @param options
*/
export const fetchEndpointIntegrationPolicyList = async (
kbnClient: KbnClient,
{ kuery, ...options }: GetPackagePoliciesRequest['query'] = {}
) => {
const endpointPackageMatchValue = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name: endpoint`;
return fetchIntegrationPolicyList(kbnClient, {
...options,
kuery: kuery ? `${kuery} AND ${endpointPackageMatchValue}` : endpointPackageMatchValue,
});
};
/**
* Retrieves all Endpoint Integration policy IDs - but only up to 10k
* @param kbnClient
*/
export const fetchAllEndpointIntegrationPolicyListIds = async (
kbnClient: KbnClient
): Promise<string[]> => {
const perPage = 1000;
const policyIds = [];
let hasMoreData = true;
do {
const result = await fetchEndpointIntegrationPolicyList(kbnClient, { perPage });
policyIds.push(...result.items.map((policy) => policy.id));
// If no more results or the next page of content goes over 10k, then end loop here.
if (!result.items.length || policyIds.length + perPage < 10000) {
hasMoreData = false;
}
} while (hasMoreData);
return policyIds;
};

89
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/index.ts Normal file
View file

@ -0,0 +1,89 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { run } from '@kbn/dev-cli-runner';
import { HORIZONTAL_LINE } from '../common/constants';
import { createKbnClient } from '../common/stack_services';
import { load } from './src/load';
import { createToolingLogger } from '../../../common/endpoint/data_loaders/utils';
export const cli = () => {
run(
async (cliContext) => {
createToolingLogger.setDefaultLogLevelFromCliFlags(cliContext.flags);
const log = cliContext.log;
const kbnClient = createKbnClient({
log,
url: cliContext.flags.kibana as string,
username: cliContext.flags.username as string,
password: cliContext.flags.password as string,
noCertForSsl: true,
});
const options = {
policyCount: Number(cliContext.flags.policyCount),
trustedAppsCount: Number(cliContext.flags.trustedAppsCount),
eventFiltersCount: Number(cliContext.flags.eventFiltersCount),
blocklistsCount: Number(cliContext.flags.blocklistsCount),
hostIsolationExceptionsCount: Number(cliContext.flags.hostIsolationExceptionsCount),
endpointExceptionsCount: Number(cliContext.flags.endpointExceptionsCount),
globalArtifactRatio: Number(cliContext.flags.globalArtifactRatio),
concurrency: Number(cliContext.flags.concurrency),
};
log.info(`${HORIZONTAL_LINE}
Environment Data Loader
${HORIZONTAL_LINE}
`);
log.info(`Loading data to: ${kbnClient.resolveUrl('')}`);
await load({
kbnClient,
log,
...options,
});
},
// Options
{
description: `Loads data into a environment for testing/development`,
flags: {
string: ['kibana', 'username', 'password'],
default: {
kibana: 'http://127.0.0.1:5601',
username: 'elastic',
password: 'changeme',
policyCount: 10,
trustedAppsCount: 10,
eventFiltersCount: 10,
blocklistsCount: 10,
hostIsolationExceptionsCount: 10,
endpointExceptionsCount: 10,
globalArtifactRatio: 50,
concurrency: 10,
},
allowUnexpected: false,
help: `
--username User name to be used for auth against elasticsearch and
kibana (Default: elastic).
--password User name Password (Default: changeme)
--kibana The url to Kibana (Default: http://127.0.0.1:5601)
--policyCount How many policies to create (Default: 10)
--trustedAppsCount How many Trusted Applications to create (Default: 10)
--eventFiltersCount How many Event Filters to create (Default: 10)
--blocklistsCount How many Blocklists to create (Default: 10)
--hostIsolationExceptionsCount How many Host Isolation Exceptions to create (Default: 10)
--endpointExceptionsCount How many Endpoint Exceptions to create (Default: 10)
--globalArtifactRatio The percentage ratio of all artifacts that should be global
rather than per-policy. (Default: 50)
--concurrency The max number of request to process in parallel. (Default: 10)
`,
},
}
);
};

296
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/create_artifacts.ts Normal file
View file

@ -0,0 +1,296 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { KbnClient } from '@kbn/test';
import type { ToolingLog } from '@kbn/tooling-log';
import { BaseDataGenerator } from '../../../../common/endpoint/data_generators/base_data_generator';
import type { NewTrustedApp } from '../../../../common/endpoint/types';
import { stringify } from '../../../../server/endpoint/utils/stringify';
import { EndpointExceptionsGenerator } from '../../../../common/endpoint/data_generators/endpoint_exceptions_generator';
import {
BY_POLICY_ARTIFACT_TAG_PREFIX,
GLOBAL_ARTIFACT_TAG,
} from '../../../../common/endpoint/service/artifacts';
import { ExceptionsListItemGenerator } from '../../../../common/endpoint/data_generators/exceptions_list_item_generator';
import type { ExecutionThrottler } from '../../common/execution_throttler';
import { EventFiltersGenerator } from '../../../../common/endpoint/data_generators/event_filters_generator';
import { TrustedAppGenerator } from '../../../../common/endpoint/data_generators/trusted_app_generator';
import {
createBlocklist,
createEventFilter,
createHostIsolationException,
createTrustedApp,
} from '../../common/endpoint_artifact_services';
import type { ReportProgressCallback } from './types';
import { loop } from './utils';
interface ArtifactCreationOptions {
kbnClient: KbnClient;
log: ToolingLog;
count: number;
policyIds: string[];
reportProgress: ReportProgressCallback;
globalArtifactRatio: number;
throttler: ExecutionThrottler;
}
// Class instance that exposes protected methods for use locally in this module
const gen = new (class extends BaseDataGenerator {
public randomArray<T>(lengthLimit: number, generator: () => T): T[] {
return super.randomArray(lengthLimit, generator);
}
public randomChoice<T>(choices: T[] | readonly T[]): T {
return super.randomChoice(choices);
}
})();
const generatePerPolicyEffectiveScope = (policyIds: string[]): string[] => {
return gen.randomArray(
gen.randomN(Math.min(100, policyIds.length)),
() => `${BY_POLICY_ARTIFACT_TAG_PREFIX}${gen.randomChoice(policyIds)}`
);
};
const logError = (log: ToolingLog, artifactType: string, error: Error) => {
log.error(`[${artifactType}] error: ${error.message}`);
log.verbose(stringify(error));
};
const calculateGlobalAndPerPolicyCounts = (
total: number,
globalRatio: number
): { global: number; perPolicy: number } => {
const response = {
global: 0,
perPolicy: 0,
};
response.global = Math.floor(total * (globalRatio / 100));
// If a ratio was defined, then ensure at least one is created
if (globalRatio > 0 && response.global === 0) {
response.global = 1;
}
response.perPolicy = total - response.global;
return response;
};
export const createTrustedApps = async ({
kbnClient,
log,
count,
reportProgress,
throttler,
globalArtifactRatio,
policyIds,
}: ArtifactCreationOptions): Promise<void> => {
const generator = new TrustedAppGenerator();
const { global: globalCount, perPolicy } = calculateGlobalAndPerPolicyCounts(
count,
globalArtifactRatio
);
let globalDone = 0;
let doneCount = 0;
let errorCount = 0;
log.info(`Trusted Apps: Creating ${globalCount} global and ${perPolicy} per-policy artifacts`);
loop(count, () => {
throttler.addToQueue(async () => {
let effectScope: NewTrustedApp['effectScope'] = { type: 'global' };
if (globalDone < globalCount) {
globalDone++;
} else {
effectScope = {
type: 'policy',
policies: gen.randomArray(gen.randomN(Math.min(100, policyIds.length)), () =>
gen.randomChoice(policyIds)
),
};
}
await createTrustedApp(
kbnClient,
generator.generateTrustedAppForCreate({
effectScope,
})
)
.catch((e) => {
errorCount++;
logError(log, 'Trusted Application', e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount });
});
});
});
};
export const createEventFilters = async ({
kbnClient,
log,
count,
reportProgress,
throttler,
globalArtifactRatio,
policyIds,
}: ArtifactCreationOptions): Promise<void> => {
const eventGenerator = new EventFiltersGenerator();
const { global: globalCount, perPolicy } = calculateGlobalAndPerPolicyCounts(
count,
globalArtifactRatio
);
let globalDone = 0;
let doneCount = 0;
let errorCount = 0;
log.info(`Event Filters: Creating ${globalCount} global and ${perPolicy} per-policy artifacts`);
loop(count, () => {
throttler.addToQueue(async () => {
let tags = [GLOBAL_ARTIFACT_TAG];
if (globalDone < globalCount) {
globalDone++;
} else {
tags = generatePerPolicyEffectiveScope(policyIds);
}
await createEventFilter(kbnClient, eventGenerator.generateEventFilterForCreate({ tags }))
.catch((e) => {
errorCount++;
logError(log, 'Event Filter', e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount });
});
});
});
};
export const createBlocklists = async ({
kbnClient,
log,
count,
reportProgress,
throttler,
policyIds,
globalArtifactRatio,
}: ArtifactCreationOptions): Promise<void> => {
const generate = new ExceptionsListItemGenerator();
const { global: globalCount, perPolicy } = calculateGlobalAndPerPolicyCounts(
count,
globalArtifactRatio
);
let globalDone = 0;
let doneCount = 0;
let errorCount = 0;
log.info(`Blocklists: Creating ${globalCount} global and ${perPolicy} per-policy artifacts`);
loop(count, () => {
throttler.addToQueue(async () => {
let tags = [GLOBAL_ARTIFACT_TAG];
if (globalDone < globalCount) {
globalDone++;
} else {
tags = generatePerPolicyEffectiveScope(policyIds);
}
await createBlocklist(kbnClient, generate.generateBlocklistForCreate({ tags }))
.catch((e) => {
errorCount++;
logError(log, 'BLocklist', e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount });
});
});
});
};
export const createHostIsolationExceptions = async ({
kbnClient,
log,
count,
reportProgress,
throttler,
globalArtifactRatio,
policyIds,
}: ArtifactCreationOptions): Promise<void> => {
const generate = new ExceptionsListItemGenerator();
const { global: globalCount, perPolicy } = calculateGlobalAndPerPolicyCounts(
count,
globalArtifactRatio
);
let globalDone = 0;
let doneCount = 0;
let errorCount = 0;
log.info(
`Host Isolation Exceptions: Creating ${globalCount} global and ${perPolicy} per-policy artifacts`
);
loop(count, () => {
throttler.addToQueue(async () => {
let tags = [GLOBAL_ARTIFACT_TAG];
if (globalDone < globalCount) {
globalDone++;
} else {
tags = generatePerPolicyEffectiveScope(policyIds);
}
await createHostIsolationException(
kbnClient,
generate.generateHostIsolationExceptionForCreate({ tags })
)
.catch((e) => {
errorCount++;
logError(log, 'Host Isolation Exception', e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount });
});
});
});
};
export const createEndpointExceptions = async ({
kbnClient,
log,
count,
reportProgress,
throttler,
}: ArtifactCreationOptions): Promise<void> => {
const generate = new EndpointExceptionsGenerator();
let doneCount = 0;
let errorCount = 0;
loop(count, () => {
throttler.addToQueue(async () => {
await createHostIsolationException(kbnClient, generate.generateEndpointExceptionForCreate())
.catch((e) => {
errorCount++;
logError(log, 'Endpoint Exception', e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount });
});
});
});
};

89
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/create_policies.ts Normal file
View file

@ -0,0 +1,89 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { KbnClient } from '@kbn/test';
import type { ToolingLog } from '@kbn/tooling-log';
import { loop } from './utils';
import type { ExecutionThrottler } from '../../common/execution_throttler';
import {
addEndpointIntegrationToAgentPolicy,
copyAgentPolicy,
createAgentPolicy,
} from '../../common/fleet_services';
import type { ReportProgressCallback } from './types';
interface CreatePoliciesOptions {
kbnClient: KbnClient;
log: ToolingLog;
count: number;
reportProgress: ReportProgressCallback;
throttler: ExecutionThrottler;
}
export const createPolicies = async ({
kbnClient,
count,
reportProgress,
log,
throttler,
}: CreatePoliciesOptions): Promise<string[]> => {
const endpointIntegrationPolicyIds: string[] = [];
const errors: Error[] = [];
let doneCount = 0;
log.verbose(`creating [${count}] policies in fleet`);
// Create first policy with endpoint
const agentPolicyId = (await createAgentPolicy({ kbnClient })).id;
const endpointPolicy = await addEndpointIntegrationToAgentPolicy({
kbnClient,
log,
agentPolicyId,
name: `endpoint protect policy (${Math.random().toString(32).substring(2)})`,
});
endpointIntegrationPolicyIds.push(endpointPolicy.id);
doneCount++;
reportProgress({ doneCount, errorCount: errors.length });
// TODO:PT maybe use ES bulk create and bypass fleet so that we speed this up?
loop(count - 1, () => {
throttler.addToQueue(async () => {
await copyAgentPolicy({ kbnClient, agentPolicyId })
.then((response) => {
if (response.package_policies?.[0]) {
endpointIntegrationPolicyIds.push(response.package_policies[0].id);
} else {
errors.push(
new Error(
`Copy of agent policy [${agentPolicyId}] did not copy the Endpoint Integration!`
)
);
}
})
.catch((e) => {
errors.push(e);
})
.finally(() => {
doneCount++;
reportProgress({ doneCount, errorCount: errors.length });
});
});
});
await throttler.complete();
if (errors.length) {
log.error(
`${errors.length} errors encountered while trying to create policies. First error: ${errors[0].message}`
);
log.verbose(...errors);
}
return endpointIntegrationPolicyIds;
};

149
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/load.ts Normal file
View file

@ -0,0 +1,149 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { ToolingLog } from '@kbn/tooling-log';
import type { KbnClient } from '@kbn/test';
import { fetchAllEndpointIntegrationPolicyListIds } from '../../common/fleet_services';
import { ExecutionThrottler } from '../../common/execution_throttler';
import {
createBlocklists,
createEndpointExceptions,
createEventFilters,
createHostIsolationExceptions,
createTrustedApps,
} from './create_artifacts';
import { installOrUpgradeEndpointFleetPackage } from '../../../../common/endpoint/data_loaders/setup_fleet_for_endpoint';
import { ProgressReporter } from './progress_reporter';
import type { ProgressReporterInterface } from './types';
import { createPolicies } from './create_policies';
import { createToolingLogger } from '../../../../common/endpoint/data_loaders/utils';
interface LoadOptions {
kbnClient: KbnClient;
log?: ToolingLog;
policyCount: number;
trustedAppsCount: number;
eventFiltersCount: number;
blocklistsCount: number;
hostIsolationExceptionsCount: number;
endpointExceptionsCount: number;
globalArtifactRatio: number;
concurrency: number;
}
export const load = async ({
kbnClient,
log = createToolingLogger(),
policyCount,
trustedAppsCount,
eventFiltersCount,
blocklistsCount,
hostIsolationExceptionsCount,
endpointExceptionsCount,
globalArtifactRatio,
concurrency,
}: LoadOptions) => {
const throttler = new ExecutionThrottler({ log, concurrency });
const reportProgress: ProgressReporterInterface = new ProgressReporter({
reportStatus: (status) => {
const now = new Date();
log.info(`__
Status at: ${now.toString()}
${status}\nRequests pending: ${throttler.getStats().pending}
`);
},
});
const policyReporter = reportProgress.addCategory('policies', policyCount);
const trustedAppsReporter = reportProgress.addCategory('trusted apps', trustedAppsCount);
const eventFiltersReporter = reportProgress.addCategory('event filters', eventFiltersCount);
const blocklistsReporter = reportProgress.addCategory('blocklists', blocklistsCount);
const hostIsolationExceptionsReporter = reportProgress.addCategory(
'host isolation exceptions',
hostIsolationExceptionsCount
);
const endpointExceptionsReporter = reportProgress.addCategory(
'endpoint exceptions',
endpointExceptionsCount
);
// Ensure fleet is setup with endpoint (which also creates the DS/Transforms, etc)
await installOrUpgradeEndpointFleetPackage(kbnClient, log);
const endpointPolicyIds = policyCount
? await createPolicies({
kbnClient,
log,
count: policyCount,
reportProgress: policyReporter,
throttler,
})
: await fetchAllEndpointIntegrationPolicyListIds(kbnClient);
log?.verbose(`Policy IDs:\n${endpointPolicyIds.join('\n')}`);
await Promise.all([
trustedAppsCount &&
createTrustedApps({
kbnClient,
log,
reportProgress: trustedAppsReporter,
count: trustedAppsCount,
policyIds: endpointPolicyIds,
globalArtifactRatio,
throttler,
}),
eventFiltersCount &&
createEventFilters({
kbnClient,
log,
reportProgress: eventFiltersReporter,
count: eventFiltersCount,
policyIds: endpointPolicyIds,
globalArtifactRatio,
throttler,
}),
blocklistsCount &&
createBlocklists({
kbnClient,
log,
reportProgress: blocklistsReporter,
count: blocklistsCount,
policyIds: endpointPolicyIds,
globalArtifactRatio,
throttler,
}),
hostIsolationExceptionsCount &&
createHostIsolationExceptions({
kbnClient,
log,
reportProgress: hostIsolationExceptionsReporter,
count: hostIsolationExceptionsCount,
policyIds: endpointPolicyIds,
globalArtifactRatio,
throttler,
}),
endpointExceptionsCount &&
createEndpointExceptions({
kbnClient,
log,
reportProgress: endpointExceptionsReporter,
count: endpointExceptionsCount,
policyIds: endpointPolicyIds,
globalArtifactRatio,
throttler,
}),
]);
await throttler.complete();
reportProgress.stopReporting();
};

181
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/progress_reporter.ts Normal file
View file

@ -0,0 +1,181 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { once } from 'lodash';
import { HORIZONTAL_LINE } from '../../common/constants';
import { getElapsedTime } from '../../../../common/endpoint/data_loaders/utils';
import type {
ProgressReporterInterface,
ProgressReporterState,
ReportProgressCallback,
} from './types';
const NOOP = () => {};
interface ProgressReporterOptions {
/**
* If defined, this callback to be used in reporting status on an interval until
* the `doneCount` reaches the `totalCount` of all categories
* @param status
*/
reportStatus?: (status: string) => void;
}
export class ProgressReporter implements ProgressReporterInterface {
private readonly reportIntervalMs = 20000;
private readonly startedAt: Date = new Date();
private categories: Record<
string,
{ totalCount: number; doneCount: number; errorCount: number }
> = {};
private stopReportingLoop: () => void = NOOP;
constructor(private readonly options: ProgressReporterOptions = {}) {
if (options.reportStatus) {
this.startReporting();
}
}
public startReporting() {
this.stopReportingLoop();
if (!this.options.reportStatus) {
return;
}
const setIntId = setInterval(() => {
if (this.options.reportStatus) {
this.options.reportStatus(this.getStatus());
if (this.getState().prctDone === 100) {
this.stopReportingLoop();
}
}
}, this.reportIntervalMs);
const exitEvCallback = () => this.stopReportingLoop();
this.stopReportingLoop = once(() => {
clearInterval(setIntId);
process.off('exit', exitEvCallback);
});
process.on('exit', exitEvCallback);
}
stopReporting() {
this.stopReportingLoop();
if (this.options.reportStatus) {
this.options.reportStatus(this.getStatus());
}
}
addCategory(name: string, totalCount: number): ReportProgressCallback {
this.categories[name] = {
totalCount,
doneCount: 0,
errorCount: 0,
};
this.startReporting();
return this.getReporter(name);
}
getReporter(categoryName: string): ReportProgressCallback {
if (!this.categories[categoryName]) {
throw new Error(`category name [${categoryName}] has not known`);
}
return (options) => {
this.categories[categoryName].doneCount = options.doneCount;
};
}
getState(): ProgressReporterState {
const state: ProgressReporterState = {
prctDone: 0,
totalCount: 0,
doneCount: 0,
errorCount: 0,
categories: {},
};
Object.entries(this.categories).forEach(
([
categoryName,
{
totalCount: thisCategoryTotalCount,
doneCount: thisCategoryDoneCount,
errorCount: thisCategoryErrorCount,
},
]) => {
state.totalCount += thisCategoryTotalCount;
state.doneCount += thisCategoryDoneCount;
state.errorCount += thisCategoryErrorCount;
state.categories[categoryName] = {
totalCount: thisCategoryTotalCount,
doneCount: thisCategoryDoneCount,
errorCount: thisCategoryErrorCount,
prctDone: calculatePercentage(thisCategoryTotalCount, thisCategoryDoneCount),
};
}
);
state.prctDone = calculatePercentage(state.totalCount, state.doneCount);
return state;
}
getStatus(): string {
const state = this.getState();
const categoryNamesMaxChr =
Object.keys(state.categories).reduce((acc, categoryName) => {
return Math.max(acc, categoryName.length);
}, 10) + 4;
return `${HORIZONTAL_LINE}
${'Overall Progress: '.padEnd(categoryNamesMaxChr + 4)}${state.prctDone}%
${HORIZONTAL_LINE}
${
'Elapsed Time (hh:mm:ss.ms):'.padEnd(categoryNamesMaxChr + 4) +
getElapsedTime(this.getStartedTime())
}
${'Error Count:'.padEnd(categoryNamesMaxChr + 4) + state.errorCount}
Details:
${Object.entries(state.categories).reduce((acc, [categoryName, categoryState]) => {
let updatedOutput = acc;
if (updatedOutput.length) {
updatedOutput += `\n `;
}
updatedOutput += `${`${`${categoryName}:`
.concat(' '.repeat(categoryNamesMaxChr))
.substring(0, categoryNamesMaxChr)} ${categoryState.prctDone}%`.padEnd(
categoryNamesMaxChr + 10
)}(${categoryState.doneCount} / ${categoryState.totalCount}, ${
categoryState.errorCount
} errors)`;
return updatedOutput;
}, '')}`;
}
getStartedTime(): Date {
return new Date(this.startedAt);
}
}
const calculatePercentage = (totalCount: number, doneCount: number): number => {
if (totalCount <= 0 || doneCount <= 0) {
return 0;
}
return Math.min(100, Number(((doneCount / totalCount) * 100).toPrecision(3)));
};

35
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/types.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export type ReportProgressCallback = (options: { doneCount: number; errorCount: number }) => void;
interface ProgressReporterStateProperties {
prctDone: number;
totalCount: number;
doneCount: number;
errorCount: number;
}
export interface ProgressReporterState extends ProgressReporterStateProperties {
categories: Record<string, ProgressReporterStateProperties>;
}
export interface ProgressReporterInterface {
addCategory(name: string, totalCount: number): ReportProgressCallback;
getReporter(categoryName: string): ReportProgressCallback;
getState(): ProgressReporterState;
getStatus(): string;
getStartedTime(): Date;
startReporting(): void;
stopReporting(): void;
}

19
x-pack/plugins/security_solution/scripts/endpoint/env_data_loader/src/utils.ts Normal file
View file

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const loop = (count: number, callback: (instance: number) => any): void => {
let done = 1;
while (done <= count) {
try {
callback(done++);
} catch {
return;
}
}
};

9
x-pack/plugins/security_solution/scripts/endpoint/load_env_data.js Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
require('../../../../../src/setup_node_env');
require('./env_data_loader').cli();