mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[Security Solutions] (Phase 1) Adds an application cache called metrics entities and integrates it within Security Solutions behind a feature flag (#96446)
## Summary Phase 1 of a multi-phase cautious approach for adding an experimental application cache for Kibana solutions called `metric_entities` and integrates it within Security Solutions. Phase 1 is putting experimental support into the application without breaking existing features. Lots of TODO's, conversations and a possible RFC from phase 1 to phase 2 approach. Some features are missing, but for phase 1 the general idea and code is all there. To enable this first phase after checking out the branch add this to your `kibana.dev.yml` ```yml xpack.metricsEntities.enabled: true xpack.securitySolution.enableExperimental: ['metricsEntitiesEnabled'] ``` Then go into Stack Management -> Advanced Settings (Under Security Solutions) and set the enabled to true like so: <img width="1229" alt="Screen Shot 2021-04-08 at 2 21 02 PM" src="https://user-images.githubusercontent.com/1151048/114091276-b3cbb700-9875-11eb-9083-5c1d91dd20ed.png"> Next go to the security_solutions page and you will see it being activated and you will have these transforms running if you look under stack management: <img width="1710" alt="Screen Shot 2021-04-29 at 2 00 27 PM" src="https://user-images.githubusercontent.com/1151048/116611174-4a2e4e00-a8f3-11eb-9e15-55cb504dfb2a.png"> On the hosts page, network, page, etc... You can see them being activated when you have no query/filter and you click on request: <img width="1405" alt="Screen Shot 2021-04-29 at 2 01 28 PM" src="https://user-images.githubusercontent.com/1151048/116611274-6a5e0d00-a8f3-11eb-9998-9f5b3d1c5c63.png"> You will see in the request the index patterns all starting with `estc_xyz*` ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) We have lots of TODO's but no concrete docs with this just yet. - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials Behind a feature flag and this isn't there yet. - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Frank Hassanabad
GitHub
parent
4686f442ee
commit
af228f0f20
176 changed files with 6529 additions and 121 deletions
112
.eslintrc.js
112
.eslintrc.js
|
|
@ -1114,6 +1114,118 @@ module.exports = {
|
|||
'prefer-destructuring': 'error',
|
||||
},
|
||||
},
|
||||
/**
|
||||
* Metrics entities overrides
|
||||
*/
|
||||
{
|
||||
// front end and common typescript and javascript files only
|
||||
files: [
|
||||
'x-pack/plugins/metrics_entities/public/**/*.{js,mjs,ts,tsx}',
|
||||
'x-pack/plugins/metrics_entities/common/**/*.{js,mjs,ts,tsx}',
|
||||
],
|
||||
rules: {
|
||||
'import/no-nodejs-modules': 'error',
|
||||
'no-restricted-imports': [
|
||||
'error',
|
||||
{
|
||||
// prevents UI code from importing server side code and then webpack including it when doing builds
|
||||
patterns: ['**/server/*'],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
// typescript and javascript for front and back end
|
||||
files: ['x-pack/plugins/metrics_entities/**/*.{js,mjs,ts,tsx}'],
|
||||
plugins: ['eslint-plugin-node'],
|
||||
env: {
|
||||
jest: true,
|
||||
},
|
||||
rules: {
|
||||
'accessor-pairs': 'error',
|
||||
'array-callback-return': 'error',
|
||||
'no-array-constructor': 'error',
|
||||
complexity: 'error',
|
||||
'consistent-return': 'error',
|
||||
'func-style': ['error', 'expression'],
|
||||
'import/order': [
|
||||
'error',
|
||||
{
|
||||
groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
|
||||
'newlines-between': 'always',
|
||||
},
|
||||
],
|
||||
'sort-imports': [
|
||||
'error',
|
||||
{
|
||||
ignoreDeclarationSort: true,
|
||||
},
|
||||
],
|
||||
'node/no-deprecated-api': 'error',
|
||||
'no-bitwise': 'error',
|
||||
'no-continue': 'error',
|
||||
'no-dupe-keys': 'error',
|
||||
'no-duplicate-case': 'error',
|
||||
'no-duplicate-imports': 'error',
|
||||
'no-empty-character-class': 'error',
|
||||
'no-empty-pattern': 'error',
|
||||
'no-ex-assign': 'error',
|
||||
'no-extend-native': 'error',
|
||||
'no-extra-bind': 'error',
|
||||
'no-extra-boolean-cast': 'error',
|
||||
'no-extra-label': 'error',
|
||||
'no-func-assign': 'error',
|
||||
'no-implicit-globals': 'error',
|
||||
'no-implied-eval': 'error',
|
||||
'no-invalid-regexp': 'error',
|
||||
'no-inner-declarations': 'error',
|
||||
'no-lone-blocks': 'error',
|
||||
'no-multi-assign': 'error',
|
||||
'no-misleading-character-class': 'error',
|
||||
'no-new-symbol': 'error',
|
||||
'no-obj-calls': 'error',
|
||||
'no-param-reassign': ['error', { props: true }],
|
||||
'no-process-exit': 'error',
|
||||
'no-prototype-builtins': 'error',
|
||||
'no-return-await': 'error',
|
||||
'no-self-compare': 'error',
|
||||
'no-shadow-restricted-names': 'error',
|
||||
'no-sparse-arrays': 'error',
|
||||
'no-this-before-super': 'error',
|
||||
// rely on typescript
|
||||
'no-undef': 'off',
|
||||
'no-unreachable': 'error',
|
||||
'no-unsafe-finally': 'error',
|
||||
'no-useless-call': 'error',
|
||||
'no-useless-catch': 'error',
|
||||
'no-useless-concat': 'error',
|
||||
'no-useless-computed-key': 'error',
|
||||
'no-useless-escape': 'error',
|
||||
'no-useless-rename': 'error',
|
||||
'no-useless-return': 'error',
|
||||
'no-void': 'error',
|
||||
'one-var-declaration-per-line': 'error',
|
||||
'prefer-object-spread': 'error',
|
||||
'prefer-promise-reject-errors': 'error',
|
||||
'prefer-rest-params': 'error',
|
||||
'prefer-spread': 'error',
|
||||
'prefer-template': 'error',
|
||||
'require-atomic-updates': 'error',
|
||||
'symbol-description': 'error',
|
||||
'vars-on-top': 'error',
|
||||
'@typescript-eslint/explicit-member-accessibility': 'error',
|
||||
'@typescript-eslint/no-this-alias': 'error',
|
||||
'@typescript-eslint/no-explicit-any': 'error',
|
||||
'@typescript-eslint/no-useless-constructor': 'error',
|
||||
'@typescript-eslint/unified-signatures': 'error',
|
||||
'@typescript-eslint/explicit-function-return-type': 'error',
|
||||
'@typescript-eslint/no-non-null-assertion': 'error',
|
||||
'@typescript-eslint/no-unused-vars': 'error',
|
||||
'no-template-curly-in-string': 'error',
|
||||
'sort-keys': 'error',
|
||||
'prefer-destructuring': 'error',
|
||||
},
|
||||
},
|
||||
/**
|
||||
* Alerting Services overrides
|
||||
*/
|
||||
|
|
|
|||
151
api_docs/metrics_entities.json
Normal file
151
api_docs/metrics_entities.json
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
{
|
||||
"id": "metricsEntities",
|
||||
"client": {
|
||||
"classes": [],
|
||||
"functions": [],
|
||||
"interfaces": [],
|
||||
"enums": [],
|
||||
"misc": [],
|
||||
"objects": []
|
||||
},
|
||||
"server": {
|
||||
"classes": [],
|
||||
"functions": [],
|
||||
"interfaces": [],
|
||||
"enums": [],
|
||||
"misc": [],
|
||||
"objects": [],
|
||||
"setup": {
|
||||
"id": "def-server.MetricsEntitiesPluginSetup",
|
||||
"type": "Interface",
|
||||
"label": "MetricsEntitiesPluginSetup",
|
||||
"description": [],
|
||||
"tags": [],
|
||||
"children": [
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-server.MetricsEntitiesPluginSetup.getMetricsEntitiesClient",
|
||||
"type": "Function",
|
||||
"label": "getMetricsEntitiesClient",
|
||||
"description": [],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/server/types.ts",
|
||||
"lineNumber": 15
|
||||
},
|
||||
"signature": [
|
||||
"GetMetricsEntitiesClientType"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/server/types.ts",
|
||||
"lineNumber": 14
|
||||
},
|
||||
"lifecycle": "setup",
|
||||
"initialIsOpen": true
|
||||
},
|
||||
"start": {
|
||||
"id": "def-server.MetricsEntitiesPluginStart",
|
||||
"type": "Type",
|
||||
"label": "MetricsEntitiesPluginStart",
|
||||
"tags": [],
|
||||
"description": [],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/server/types.ts",
|
||||
"lineNumber": 18
|
||||
},
|
||||
"signature": [
|
||||
"void"
|
||||
],
|
||||
"lifecycle": "start",
|
||||
"initialIsOpen": true
|
||||
}
|
||||
},
|
||||
"common": {
|
||||
"classes": [],
|
||||
"functions": [],
|
||||
"interfaces": [],
|
||||
"enums": [],
|
||||
"misc": [
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-common.ELASTIC_NAME",
|
||||
"type": "string",
|
||||
"label": "ELASTIC_NAME",
|
||||
"description": [
|
||||
"\nGlobal prefix for all the transform jobs"
|
||||
],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/common/constants.ts",
|
||||
"lineNumber": 21
|
||||
},
|
||||
"signature": [
|
||||
"\"estc\""
|
||||
],
|
||||
"initialIsOpen": false
|
||||
},
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-common.METRICS_ENTITIES_TRANSFORMS",
|
||||
"type": "string",
|
||||
"label": "METRICS_ENTITIES_TRANSFORMS",
|
||||
"description": [
|
||||
"\nTransforms route"
|
||||
],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/common/constants.ts",
|
||||
"lineNumber": 16
|
||||
},
|
||||
"initialIsOpen": false
|
||||
},
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-common.METRICS_ENTITIES_URL",
|
||||
"type": "string",
|
||||
"label": "METRICS_ENTITIES_URL",
|
||||
"description": [
|
||||
"\nBase route"
|
||||
],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/common/constants.ts",
|
||||
"lineNumber": 11
|
||||
},
|
||||
"signature": [
|
||||
"\"/api/metrics_entities\""
|
||||
],
|
||||
"initialIsOpen": false
|
||||
},
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-common.PLUGIN_ID",
|
||||
"type": "string",
|
||||
"label": "PLUGIN_ID",
|
||||
"description": [],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/common/index.ts",
|
||||
"lineNumber": 8
|
||||
},
|
||||
"signature": [
|
||||
"\"metricsEntities\""
|
||||
],
|
||||
"initialIsOpen": false
|
||||
},
|
||||
{
|
||||
"tags": [],
|
||||
"id": "def-common.PLUGIN_NAME",
|
||||
"type": "string",
|
||||
"label": "PLUGIN_NAME",
|
||||
"description": [],
|
||||
"source": {
|
||||
"path": "x-pack/plugins/metrics_entities/common/index.ts",
|
||||
"lineNumber": 9
|
||||
},
|
||||
"signature": [
|
||||
"\"metrics_entities\""
|
||||
],
|
||||
"initialIsOpen": false
|
||||
}
|
||||
],
|
||||
"objects": []
|
||||
}
|
||||
}
|
||||
26
api_docs/metrics_entities.mdx
Normal file
26
api_docs/metrics_entities.mdx
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
id: kibMetricsEntitiesPluginApi
|
||||
slug: /kibana-dev-docs/metricsEntitiesPluginApi
|
||||
title: metricsEntities
|
||||
image: https://source.unsplash.com/400x175/?github
|
||||
summary: API docs for the metricsEntities plugin
|
||||
date: 2020-11-16
|
||||
tags: ['contributor', 'dev', 'apidocs', 'kibana', 'metricsEntities']
|
||||
warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
|
||||
---
|
||||
|
||||
import metricsEntitiesObj from './metrics_entities.json';
|
||||
|
||||
## Server
|
||||
|
||||
### Setup
|
||||
<DocDefinitionList data={[metricsEntitiesObj.server.setup]}/>
|
||||
|
||||
### Start
|
||||
<DocDefinitionList data={[metricsEntitiesObj.server.start]}/>
|
||||
|
||||
## Common
|
||||
|
||||
### Consts, variables and types
|
||||
<DocDefinitionList data={metricsEntitiesObj.common.misc}/>
|
||||
|
||||
|
|
@ -474,6 +474,12 @@ using the CURL scripts in the scripts folder.
|
|||
|Visualize geo data from Elasticsearch or 3rd party geo-services.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/metrics_entities/README.md[metricsEntities]
|
||||
|This is the metrics and entities plugin where you add can add transforms for your project
|
||||
and group those transforms into modules. You can also re-use existing transforms in your
|
||||
modules as well.
|
||||
|
||||
|
||||
|{kib-repo}blob/{branch}/x-pack/plugins/ml/readme.md[ml]
|
||||
|This plugin provides access to the machine learning features provided by
|
||||
Elastic.
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ it('produces the right watch and ignore list', () => {
|
|||
<absolute path>/x-pack/plugins/lists/server/scripts,
|
||||
<absolute path>/x-pack/plugins/security_solution/scripts,
|
||||
<absolute path>/x-pack/plugins/security_solution/server/lib/detection_engine/scripts,
|
||||
<absolute path>/x-pack/plugins/metrics_entities/server/scripts,
|
||||
]
|
||||
`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) {
|
|||
fromRoot('x-pack/plugins/lists/server/scripts'),
|
||||
fromRoot('x-pack/plugins/security_solution/scripts'),
|
||||
fromRoot('x-pack/plugins/security_solution/server/lib/detection_engine/scripts'),
|
||||
fromRoot('x-pack/plugins/metrics_entities/server/scripts'),
|
||||
];
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"x-pack/typings/**/*",
|
||||
"x-pack/tasks/**/*",
|
||||
"x-pack/plugins/lists/**/*",
|
||||
"x-pack/plugins/security_solution/**/*",
|
||||
"x-pack/plugins/security_solution/**/*"
|
||||
],
|
||||
"exclude": [
|
||||
"x-pack/plugins/security_solution/cypress/**/*"
|
||||
|
|
@ -110,6 +110,7 @@
|
|||
{ "path": "./x-pack/plugins/licensing/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/logstash/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/maps/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/metrics_entities/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/ml/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/monitoring/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/observability/tsconfig.json" },
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@
|
|||
{ "path": "./x-pack/plugins/licensing/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/logstash/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/maps/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/metrics_entities/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/ml/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/monitoring/tsconfig.json" },
|
||||
{ "path": "./x-pack/plugins/observability/tsconfig.json" },
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@
|
|||
"xpack.logstash": ["plugins/logstash"],
|
||||
"xpack.main": "legacy/plugins/xpack_main",
|
||||
"xpack.maps": ["plugins/maps"],
|
||||
"xpack.metricsEntities": "plugins/metrics_entities",
|
||||
"xpack.ml": ["plugins/ml"],
|
||||
"xpack.monitoring": ["plugins/monitoring"],
|
||||
"xpack.osquery": ["plugins/osquery"],
|
||||
|
|
|
|||
324
x-pack/plugins/metrics_entities/README.md
Executable file
324
x-pack/plugins/metrics_entities/README.md
Executable file
|
|
@ -0,0 +1,324 @@
|
|||
# metrics_entities
|
||||
|
||||
This is the metrics and entities plugin where you add can add transforms for your project
|
||||
and group those transforms into modules. You can also re-use existing transforms in your
|
||||
modules as well.
|
||||
|
||||
## Turn on experimental flags
|
||||
During at least phase 1 of this development, please add these to your `kibana.dev.yml` file to turn on the feature:
|
||||
|
||||
```ts
|
||||
xpack.metricsEntities.enabled: true
|
||||
xpack.securitySolution.enableExperimental: ['metricsEntitiesEnabled']
|
||||
```
|
||||
|
||||
## Quick start on using scripts to call the API
|
||||
|
||||
The scripts rely on CURL and jq:
|
||||
|
||||
- [CURL](https://curl.haxx.se)
|
||||
- [jq](https://stedolan.github.io/jq/)
|
||||
|
||||
Install curl and jq
|
||||
|
||||
```sh
|
||||
brew update
|
||||
brew install curl
|
||||
brew install jq
|
||||
```
|
||||
|
||||
Open `$HOME/.zshrc` or `${HOME}.bashrc` depending on your SHELL output from `echo $SHELL`
|
||||
and add these environment variables:
|
||||
|
||||
```sh
|
||||
export ELASTICSEARCH_USERNAME=${user}
|
||||
export ELASTICSEARCH_PASSWORD=${password}
|
||||
export ELASTICSEARCH_URL=https://${ip}:9200
|
||||
export KIBANA_URL=http://localhost:5601
|
||||
```
|
||||
|
||||
source `$HOME/.zshrc` or `${HOME}.bashrc` to ensure variables are set:
|
||||
|
||||
```sh
|
||||
source ~/.zshrc
|
||||
```
|
||||
|
||||
Restart Kibana and ensure that you are using `--no-base-path` as changing the base path is a feature but will
|
||||
get in the way of the CURL scripts written as is.
|
||||
|
||||
Go to the scripts folder `cd kibana/x-pack/plugins/metrics_entities/server/scripts` and can run some of the scripts
|
||||
such as:
|
||||
|
||||
```sh
|
||||
./post_transforms.sh ./post_examples/all.json
|
||||
```
|
||||
|
||||
which will post transforms from the `all.json`
|
||||
|
||||
You can also delete them by running:
|
||||
|
||||
```sh
|
||||
./delete_transforms.sh ./delete_examples/all.json
|
||||
```
|
||||
|
||||
See the folder for other curl scripts that exercise parts of the REST API and feel free to add your own examples
|
||||
in the folder as well.
|
||||
|
||||
## Quick start on how to add a transform
|
||||
|
||||
You will want to figure out how you want your transform from within Kibana roughly using
|
||||
the UI and then copy the JSON. The JSON you will want to change and paste within a folder
|
||||
which represents a module.
|
||||
|
||||
For example, for the `host_entities` and a `host_entities_mapping` we created a folder called host_entities
|
||||
here:
|
||||
|
||||
```sh
|
||||
sever/modules/host_entities
|
||||
```
|
||||
|
||||
Then we add two files, a subset of the transform JSON and a mapping like so:
|
||||
|
||||
```sh
|
||||
server/modules/host_entities/host_entities_mapping.json <--- this is the mappings
|
||||
server/modules/host_entities/host_entities.json <--- This is a subset of the transform JSON
|
||||
index.ts <--- Import/export your json here
|
||||
```
|
||||
|
||||
The mappings can be normal mapping like so with `host_entities_mapping.json`:
|
||||
```json
|
||||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "host_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"os": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"version": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
One caveat is that you need to add this to the meta section to tell it what the name will be:
|
||||
```json
|
||||
"_meta": {
|
||||
"index": "host_ent"
|
||||
},
|
||||
```
|
||||
|
||||
Keep the name short as there is only 65 characters for a transform job and we prepend extra information to the mapping such as:
|
||||
* prefix
|
||||
* name of estc
|
||||
|
||||
Although not required, a `"dynamic": "strict"` is strongly encouraged to prevent mapping guesses from elastic and it will be better for us
|
||||
to spot errors quicker in the mappings such as type-o's if this is set to strict.
|
||||
|
||||
Next, for the transform, you should add a subset that doesn't have any additional settings or meta associated like so for `host_entities.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "host_ent",
|
||||
"description": "[host.name entities] grouped by @timestamp, host.name, os.name, and os.version, and aggregated on host.name",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"host.name": {
|
||||
"terms": {
|
||||
"field": "host.name"
|
||||
}
|
||||
},
|
||||
"host.os.name": {
|
||||
"terms": {
|
||||
"field": "host.os.name",
|
||||
"missing_bucket": true
|
||||
}
|
||||
},
|
||||
"host.os.version": {
|
||||
"terms": {
|
||||
"field": "host.os.version",
|
||||
"missing_bucket": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.host.name.value_count": {
|
||||
"value_count": {
|
||||
"field": "host.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Look in the `server/modules` for other examples, but it should be that clear cut. The final part is to wire everything up in the code by touching a few files
|
||||
to either add this to an existing module or create your own module. In `server/module/host_entities` we add an `index.ts` like so that does an import/export
|
||||
of the JSON:
|
||||
|
||||
```sh
|
||||
import hostEntities from './host_entities.json';
|
||||
import hostEntitiesMapping from './host_entities_mapping.json';
|
||||
export { hostEntities, hostEntitiesMapping };
|
||||
```
|
||||
|
||||
Then in `modules/index.ts` we add a new module name if we are creating a new module to the `export enum ModuleNames {` like so:
|
||||
|
||||
```ts
|
||||
// Import your host entities you just made
|
||||
import { hostEntities, hostEntitiesMapping } from './host_entities';
|
||||
|
||||
/**
|
||||
* These module names will map 1 to 1 to the REST interface.
|
||||
*/
|
||||
export enum ModuleNames {
|
||||
hostSummaryMetrics = 'host_metrics',
|
||||
hostSummaryEntities = 'host_entities', // <-- Add the entities/transform and give it a enum name and a module name
|
||||
networkSummaryEntities = 'network_entities',
|
||||
networkSummaryMetrics = 'network_metrics',
|
||||
userSummaryEntities = 'user_entities',
|
||||
userSummaryMetrics = 'user_metrics',
|
||||
}
|
||||
```
|
||||
|
||||
If you're not creating a new module but rather you are adding to an existing module, you can skip the above step. Next, you
|
||||
just need to add your installable transform and installable mapping to the two data structures of `installableTransforms` and
|
||||
`installableMappings` like so:
|
||||
|
||||
```ts
|
||||
/**
|
||||
* Add any new folders as modules with their names below and grouped with
|
||||
* key values.
|
||||
*/
|
||||
export const installableTransforms: Record<ModuleNames, Transforms[]> = {
|
||||
[ModuleNames.hostSummaryMetrics]: [hostMetrics],
|
||||
[ModuleNames.hostSummaryEntities]: [hostEntities], // <-- Adds my new module name and transform to a new array.
|
||||
[ModuleNames.networkSummaryEntities]: [
|
||||
destinationIpEntities, // <-- If instead I am adding to an existing module, I just add it to the array like these show
|
||||
sourceIpEntities,
|
||||
destinationCountryIsoCodeEntities,
|
||||
sourceCountryIsoCodeEntities,
|
||||
],
|
||||
[ModuleNames.networkSummaryMetrics]: [ipMetrics],
|
||||
[ModuleNames.userSummaryEntities]: [userEntities],
|
||||
[ModuleNames.userSummaryMetrics]: [userMetrics],
|
||||
};
|
||||
|
||||
/**
|
||||
* For all the mapping types, add each with their names below and grouped with
|
||||
* key values.
|
||||
*/
|
||||
export const installableMappings: Record<ModuleNames, Mappings[]> = {
|
||||
[ModuleNames.hostSummaryMetrics]: [hostMetricsMapping],
|
||||
[ModuleNames.hostSummaryEntities]: [hostEntitiesMapping], // <-- Adds my new module name and mapping to a new array.
|
||||
[ModuleNames.networkSummaryEntities]: [ // <-- If instead I am adding to an existing module, I just add it to the array like these show
|
||||
sourceIpEntitiesMapping,
|
||||
destinationIpEntitiesMapping,
|
||||
destinationCountryIsoCodeEntitiesMapping,
|
||||
sourceCountryIsoCodeEntitiesMapping,
|
||||
],
|
||||
[ModuleNames.networkSummaryMetrics]: [ipMetricsMapping],
|
||||
[ModuleNames.userSummaryEntities]: [userEntitiesMapping],
|
||||
[ModuleNames.userSummaryMetrics]: [userMetricsMapping],
|
||||
};
|
||||
```
|
||||
|
||||
And after that, you should check out if there are any existing e2e tests or unit tests to update here to ensure that your mapping and transform will
|
||||
pass ci. Create a pull request and your mapping and transform are completed.
|
||||
|
||||
To call into the code to activate your module and create your transforms and mappings would be the following where you substitute your
|
||||
${KIBANA_URL} with your kibana URL and the ${SPACE_URL} with any space id you have. If you're using the default space then you would use
|
||||
an empty string:
|
||||
```json
|
||||
POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms
|
||||
{
|
||||
"prefix": "all",
|
||||
"modules": [
|
||||
"host_entities",
|
||||
],
|
||||
"indices": [
|
||||
"auditbeat-*",
|
||||
"endgame-*",
|
||||
"filebeat-*",
|
||||
"logs-*",
|
||||
"packetbeat-*",
|
||||
"winlogbeat-*",
|
||||
"-*elastic-cloud-logs-*"
|
||||
],
|
||||
"auto_start": true,
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
},
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Very similar to the regular transform REST API, with the caveats that you define which modules you want to install, the prefix name you want to use, and
|
||||
if you want to `auto_start` it or not. The rest such as `settings`, `query` will be the same as the transforms API. They will also push those same setting into
|
||||
each of your transforms within your module(s) as the same setting for each individual ones.
|
||||
|
||||
## TODO List
|
||||
During the phase 1, phase 2, phase N, this TODO will exist as a reminder and notes for what still needs to be developed. These are not in a priority order, but
|
||||
are notes during the phased approach. As we approach production and the feature flags are removed these TODO's should be removed in favor of Kibana issues or regular
|
||||
left over TODO's in the code base.
|
||||
|
||||
- Add these properties to the route which are:
|
||||
- disable_transforms/exclude flag to exclude 1 or more transforms within a module,
|
||||
- pipeline flag,
|
||||
- Change the REST routes on post to change the indexes for whichever indexes you want
|
||||
- Unit tests to ensure the data of the mapping.json includes the correct fields such as
|
||||
_meta, at least one alias, a mapping section, etc...
|
||||
- Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time
|
||||
- Add a sort of @timestamp to the output destination indexes?
|
||||
- Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'],
|
||||
- Add schema validation choosing some schema library (io-ts or Kibana Schema or ... )
|
||||
- Add unit tests
|
||||
- Add e2e tests
|
||||
- Move ui code into this plugin from security_solutions? (maybe?)
|
||||
- UI code could be within `kibana/packages` instead of in here directly and I think we will be better off.
|
||||
21
x-pack/plugins/metrics_entities/common/constants.ts
Normal file
21
x-pack/plugins/metrics_entities/common/constants.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Base route
|
||||
*/
|
||||
export const METRICS_ENTITIES_URL = '/api/metrics_entities';
|
||||
|
||||
/**
|
||||
* Transforms route
|
||||
*/
|
||||
export const METRICS_ENTITIES_TRANSFORMS = `${METRICS_ENTITIES_URL}/transforms`;
|
||||
|
||||
/**
|
||||
* Global prefix for all the transform jobs
|
||||
*/
|
||||
export const ELASTIC_NAME = 'estc';
|
||||
11
x-pack/plugins/metrics_entities/common/index.ts
Normal file
11
x-pack/plugins/metrics_entities/common/index.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const PLUGIN_ID = 'metricsEntities';
|
||||
export const PLUGIN_NAME = 'metrics_entities';
|
||||
|
||||
export * from './constants';
|
||||
12
x-pack/plugins/metrics_entities/jest.config.js
Normal file
12
x-pack/plugins/metrics_entities/jest.config.js
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
preset: '@kbn/test',
|
||||
rootDir: '../../..',
|
||||
roots: ['<rootDir>/x-pack/plugins/metrics_entities'],
|
||||
};
|
||||
10
x-pack/plugins/metrics_entities/kibana.json
Normal file
10
x-pack/plugins/metrics_entities/kibana.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"id": "metricsEntities",
|
||||
"version": "8.0.0",
|
||||
"kibanaVersion": "kibana",
|
||||
"configPath": ["xpack", "metricsEntities"],
|
||||
"server": true,
|
||||
"ui": false,
|
||||
"requiredPlugins": ["data", "dataEnhanced"],
|
||||
"optionalPlugins": []
|
||||
}
|
||||
14
x-pack/plugins/metrics_entities/server/config.ts
Normal file
14
x-pack/plugins/metrics_entities/server/config.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { TypeOf, schema } from '@kbn/config-schema';
|
||||
|
||||
export const ConfigSchema = schema.object({
|
||||
enabled: schema.boolean({ defaultValue: false }),
|
||||
});
|
||||
|
||||
export type ConfigType = TypeOf<typeof ConfigSchema>;
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export class ErrorWithStatusCode extends Error {
|
||||
private readonly statusCode: number;
|
||||
|
||||
constructor(message: string, statusCode: number) {
|
||||
super(message);
|
||||
this.statusCode = statusCode;
|
||||
}
|
||||
|
||||
public getStatusCode = (): number => this.statusCode;
|
||||
}
|
||||
21
x-pack/plugins/metrics_entities/server/index.ts
Normal file
21
x-pack/plugins/metrics_entities/server/index.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { PluginInitializerContext } from '../../../../src/core/server';
|
||||
|
||||
import { ConfigSchema } from './config';
|
||||
import { MetricsEntitiesPlugin } from './plugin';
|
||||
|
||||
// This exports static code and TypeScript types,
|
||||
// as well as, Kibana Platform `plugin()` initializer.
|
||||
|
||||
export const config = { schema: ConfigSchema };
|
||||
export const plugin = (initializerContext: PluginInitializerContext): MetricsEntitiesPlugin => {
|
||||
return new MetricsEntitiesPlugin(initializerContext);
|
||||
};
|
||||
|
||||
export { MetricsEntitiesPluginSetup, MetricsEntitiesPluginStart } from './types';
|
||||
4
x-pack/plugins/metrics_entities/server/modules/README.md
Normal file
4
x-pack/plugins/metrics_entities/server/modules/README.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# Modules
|
||||
|
||||
This is where all the module types exist so you can load different bundled modules
|
||||
with a REST endpoint.
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
{
|
||||
"id": "host_ent",
|
||||
"description": "[host.name entities] grouped by @timestamp, host.name, os.name, and os.version, and aggregated on host.name",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"host.name": {
|
||||
"terms": {
|
||||
"field": "host.name"
|
||||
}
|
||||
},
|
||||
"host.os.name": {
|
||||
"terms": {
|
||||
"field": "host.os.name",
|
||||
"missing_bucket": true
|
||||
}
|
||||
},
|
||||
"host.os.version": {
|
||||
"terms": {
|
||||
"field": "host.os.version",
|
||||
"missing_bucket": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.host.name.value_count": {
|
||||
"value_count": {
|
||||
"field": "host.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "host_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"os": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"version": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import hostEntities from './host_entities.json';
|
||||
import hostEntitiesMapping from './host_entities_mapping.json';
|
||||
export { hostEntities, hostEntitiesMapping };
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"id": "host_met",
|
||||
"description": "[host.name metrics] grouped by @timestamp and aggregated on host.name",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.host.name.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "host.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "host_met"
|
||||
},
|
||||
"properties": {
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"community_id": {
|
||||
"properties": {
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import hostMetrics from './host_metrics.json';
|
||||
import hostMetricsMapping from './host_metrics_mapping.json';
|
||||
|
||||
export { hostMetrics, hostMetricsMapping };
|
||||
71
x-pack/plugins/metrics_entities/server/modules/index.ts
Normal file
71
x-pack/plugins/metrics_entities/server/modules/index.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { hostMetrics, hostMetricsMapping } from './host_metrics';
|
||||
import { userMetrics, userMetricsMapping } from './user_metrics';
|
||||
import { ipMetrics, ipMetricsMapping } from './network_metrics';
|
||||
import { hostEntities, hostEntitiesMapping } from './host_entities';
|
||||
import {
|
||||
destinationCountryIsoCodeEntities,
|
||||
destinationCountryIsoCodeEntitiesMapping,
|
||||
destinationIpEntities,
|
||||
destinationIpEntitiesMapping,
|
||||
sourceCountryIsoCodeEntities,
|
||||
sourceCountryIsoCodeEntitiesMapping,
|
||||
sourceIpEntities,
|
||||
sourceIpEntitiesMapping,
|
||||
} from './network_entities';
|
||||
import { Mappings, Transforms } from './types';
|
||||
import { userEntities, userEntitiesMapping } from './user_entities';
|
||||
|
||||
/**
|
||||
* These module names will map 1 to 1 to the REST interface.
|
||||
*/
|
||||
export enum ModuleNames {
|
||||
hostSummaryMetrics = 'host_metrics',
|
||||
hostSummaryEntities = 'host_entities',
|
||||
networkSummaryEntities = 'network_entities',
|
||||
networkSummaryMetrics = 'network_metrics',
|
||||
userSummaryEntities = 'user_entities',
|
||||
userSummaryMetrics = 'user_metrics',
|
||||
}
|
||||
|
||||
/**
|
||||
* Add any new folders as modules with their names below and grouped with
|
||||
* key values.
|
||||
*/
|
||||
export const installableTransforms: Record<ModuleNames, Transforms[]> = {
|
||||
[ModuleNames.hostSummaryMetrics]: [hostMetrics],
|
||||
[ModuleNames.hostSummaryEntities]: [hostEntities],
|
||||
[ModuleNames.networkSummaryEntities]: [
|
||||
destinationIpEntities,
|
||||
sourceIpEntities,
|
||||
destinationCountryIsoCodeEntities,
|
||||
sourceCountryIsoCodeEntities,
|
||||
],
|
||||
[ModuleNames.networkSummaryMetrics]: [ipMetrics],
|
||||
[ModuleNames.userSummaryEntities]: [userEntities],
|
||||
[ModuleNames.userSummaryMetrics]: [userMetrics],
|
||||
};
|
||||
|
||||
/**
|
||||
* For all the mapping types, add each with their names below and grouped with
|
||||
* key values.
|
||||
*/
|
||||
export const installableMappings: Record<ModuleNames, Mappings[]> = {
|
||||
[ModuleNames.hostSummaryMetrics]: [hostMetricsMapping],
|
||||
[ModuleNames.hostSummaryEntities]: [hostEntitiesMapping],
|
||||
[ModuleNames.networkSummaryEntities]: [
|
||||
sourceIpEntitiesMapping,
|
||||
destinationIpEntitiesMapping,
|
||||
destinationCountryIsoCodeEntitiesMapping,
|
||||
sourceCountryIsoCodeEntitiesMapping,
|
||||
],
|
||||
[ModuleNames.networkSummaryMetrics]: [ipMetricsMapping],
|
||||
[ModuleNames.userSummaryEntities]: [userEntitiesMapping],
|
||||
[ModuleNames.userSummaryMetrics]: [userMetricsMapping],
|
||||
};
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"id": "dest_iso_ent",
|
||||
"description": "[destination.geo.country_iso_code entities] grouped by @timestamp and aggregated on source.bytes, destination.bytes, network.community_id, destination.ip, and source.ip",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"destination.geo.country_iso_code": {
|
||||
"terms": {
|
||||
"field": "destination.geo.country_iso_code"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.destination.geo.country_iso_code.value_count": {
|
||||
"value_count": {
|
||||
"field": "destination.geo.country_iso_code"
|
||||
}
|
||||
},
|
||||
"metrics.source.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "source.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.destination.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "destination.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.network.community_id.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "network.community_id"
|
||||
}
|
||||
},
|
||||
"metrics.source.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
},
|
||||
"metrics.destination.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "dest_iso_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"community_id": {
|
||||
"properties": {
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"id": "dest_ip_ent",
|
||||
"description": "[destination.ip entities] grouped by @timestamp and aggregated on destination.ip, source.bytes, destination.bytes, network.community_id, and source.ip",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"destination.ip": {
|
||||
"terms": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.destination.ip.value_count": {
|
||||
"value_count": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
},
|
||||
"metrics.source.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "source.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.destination.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "destination.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.network.community_id.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "network.community_id"
|
||||
}
|
||||
},
|
||||
"metrics.source.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "dest_ip_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"community_id": {
|
||||
"properties": {
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import sourceIpEntities from './source_ip_entities.json';
|
||||
import destinationIpEntities from './destination_ip_entities.json';
|
||||
import sourceIpEntitiesMapping from './source_ip_entities_mapping.json';
|
||||
import destinationIpEntitiesMapping from './destination_ip_entities_mapping.json';
|
||||
import destinationCountryIsoCodeEntities from './destination_country_iso_code_entities.json';
|
||||
import destinationCountryIsoCodeEntitiesMapping from './destination_country_iso_code_entities_mapping.json';
|
||||
import sourceCountryIsoCodeEntities from './source_country_iso_code_entities.json';
|
||||
import sourceCountryIsoCodeEntitiesMapping from './source_country_iso_code_entities_mapping.json';
|
||||
|
||||
export {
|
||||
sourceIpEntities,
|
||||
destinationIpEntities,
|
||||
sourceCountryIsoCodeEntities,
|
||||
sourceCountryIsoCodeEntitiesMapping,
|
||||
destinationCountryIsoCodeEntities,
|
||||
destinationCountryIsoCodeEntitiesMapping,
|
||||
sourceIpEntitiesMapping,
|
||||
destinationIpEntitiesMapping,
|
||||
};
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"id": "src_iso_ent",
|
||||
"description": "[source.geo.country_iso_code entities] grouped by @timestamp and aggregated on source.geo.country_iso_code, source.bytes, destination.bytes, network.community_id, source.ip, and destination.ip",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"source.geo.country_iso_code": {
|
||||
"terms": {
|
||||
"field": "source.geo.country_iso_code"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.source.geo.country_iso_code.value_count": {
|
||||
"value_count": {
|
||||
"field": "source.geo.country_iso_code"
|
||||
}
|
||||
},
|
||||
"metrics.source.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "source.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.destination.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "destination.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.network.community_id.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "network.community_id"
|
||||
}
|
||||
},
|
||||
"metrics.source.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
},
|
||||
"metrics.destination.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "src_iso_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"community_id": {
|
||||
"properties": {
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"geo": {
|
||||
"properties": {
|
||||
"country_iso_code": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"id": "src_ip_ent",
|
||||
"description": "[source.ip entities] grouped by @timestamp and aggregated on destination.ip, source.bytes, destination.bytes, network.community_id, and destination.ip",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"source.ip": {
|
||||
"terms": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.source.ip.value_count": {
|
||||
"value_count": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
},
|
||||
"metrics.source.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "source.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.destination.bytes.sum": {
|
||||
"sum": {
|
||||
"field": "destination.bytes"
|
||||
}
|
||||
},
|
||||
"metrics.network.community_id.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "network.community_id"
|
||||
}
|
||||
},
|
||||
"metrics.destination.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "src_ip_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"community_id": {
|
||||
"properties": {
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import ipMetrics from './ip_metrics.json';
|
||||
import ipMetricsMapping from './ip_metrics_mapping.json';
|
||||
|
||||
export { ipMetrics, ipMetricsMapping };
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
{
|
||||
"id": "ip_met",
|
||||
"description": "[source.ip metrics] grouped by @timestamp, source.ip, destination.ip and aggregated on tls.version, suricata.eve.tls.version, zeek.ssl.version, dns.question.name, and zeek.dns.query",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.source.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
},
|
||||
"metrics.destination.ip.cardinality": {
|
||||
"cardinality": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
},
|
||||
"metrics.network": {
|
||||
"filter": {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "source.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "destination.ip"
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"events.value_count": {
|
||||
"value_count": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
},
|
||||
"tls": {
|
||||
"filter": {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "tls.version"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "suricata.eve.tls.version"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "zeek.ssl.version"
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"version.value_count": {
|
||||
"value_count": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"metrics.dns": {
|
||||
"filter": {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "dns.question.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"suricata.eve.dns.type": {
|
||||
"value": "query"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "zeek.dns.query"
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"queries.value_count": {
|
||||
"value_count": {
|
||||
"field": "@timestamp"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "ip_met"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"source": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"cardinality": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bytes": {
|
||||
"properties": {
|
||||
"sum": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"network": {
|
||||
"properties": {
|
||||
"events": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"tls": {
|
||||
"properties": {
|
||||
"version": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"dns": {
|
||||
"properties": {
|
||||
"queries": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
38
x-pack/plugins/metrics_entities/server/modules/types.ts
Normal file
38
x-pack/plugins/metrics_entities/server/modules/types.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Loose type for the mappings
|
||||
*/
|
||||
export interface Mappings {
|
||||
[key: string]: unknown;
|
||||
mappings: {
|
||||
[key: string]: unknown;
|
||||
_meta: {
|
||||
index: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Loose type for the transforms. id is marked optional so we can delete it before
|
||||
* pushing it through elastic client.
|
||||
* TODO: Can we use stricter pre-defined typings for the transforms here or is this ours because we define it slightly different?
|
||||
*/
|
||||
export interface Transforms {
|
||||
[key: string]: unknown;
|
||||
id: string;
|
||||
dest?: Partial<{
|
||||
index: string;
|
||||
pipeline: string;
|
||||
}>;
|
||||
source?: Partial<{}>;
|
||||
settings?: Partial<{
|
||||
max_page_search_size: number;
|
||||
docs_per_second: number | null;
|
||||
}>;
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import userEntities from './user_entities.json';
|
||||
import userEntitiesMapping from './user_entities_mapping.json';
|
||||
export { userEntities, userEntitiesMapping };
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"id": "user_ent",
|
||||
"description": "[user.name entities] grouped by @timestamp and aggregated on user.name, and event.categories of success, failure, and unknown",
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
},
|
||||
"user.name": {
|
||||
"terms": {
|
||||
"field": "user.name"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.event.authentication": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.category": "authentication"
|
||||
}
|
||||
},
|
||||
"aggs": {
|
||||
"success.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "success"
|
||||
}
|
||||
}
|
||||
},
|
||||
"failure.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "failure"
|
||||
}
|
||||
}
|
||||
},
|
||||
"unknown.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "unknown"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "user_ent"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"event": {
|
||||
"properties": {
|
||||
"authentication": {
|
||||
"properties": {
|
||||
"failure": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"success": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"unknown": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"user": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import userMetrics from './user_metrics.json';
|
||||
import userMetricsMapping from './user_metrics_mapping.json';
|
||||
|
||||
export { userMetrics, userMetricsMapping };
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
{
|
||||
"id": "user_met",
|
||||
"description": "[event.category authentication metrics] grouped by @timestamp and aggregated on success, failure, and unknown",
|
||||
"source": {
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"term": {
|
||||
"event.category": "authentication"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pivot": {
|
||||
"group_by": {
|
||||
"@timestamp": {
|
||||
"date_histogram": {
|
||||
"field": "@timestamp",
|
||||
"calendar_interval": "1h"
|
||||
}
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"metrics.event.authentication.success.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "success"
|
||||
}
|
||||
}
|
||||
},
|
||||
"metrics.event.authentication.failure.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "failure"
|
||||
}
|
||||
}
|
||||
},
|
||||
"metrics.event.authentication.unknown.value_count": {
|
||||
"filter": {
|
||||
"term": {
|
||||
"event.outcome": "unknown"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"mappings": {
|
||||
"_meta": {
|
||||
"index": "user_met"
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": {
|
||||
"metrics": {
|
||||
"properties": {
|
||||
"event": {
|
||||
"properties": {
|
||||
"authentication": {
|
||||
"properties": {
|
||||
"failure": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"success": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
},
|
||||
"unknown": {
|
||||
"properties": {
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
89
x-pack/plugins/metrics_entities/server/plugin.ts
Normal file
89
x-pack/plugins/metrics_entities/server/plugin.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
CoreSetup,
|
||||
CoreStart,
|
||||
Logger,
|
||||
Plugin,
|
||||
PluginInitializerContext,
|
||||
} from '../../../../src/core/server';
|
||||
|
||||
import {
|
||||
ContextProvider,
|
||||
ContextProviderReturn,
|
||||
MetricsEntitiesPluginSetup,
|
||||
MetricsEntitiesPluginStart,
|
||||
MetricsEntitiesRequestHandlerContext,
|
||||
} from './types';
|
||||
import { getTransforms, postTransforms } from './routes';
|
||||
import { MetricsEntitiesClient } from './services/metrics_entities_client';
|
||||
import { deleteTransforms } from './routes/delete_transforms';
|
||||
|
||||
export class MetricsEntitiesPlugin
|
||||
implements Plugin<MetricsEntitiesPluginSetup, MetricsEntitiesPluginStart> {
|
||||
private readonly logger: Logger;
|
||||
private kibanaVersion: string;
|
||||
|
||||
constructor(initializerContext: PluginInitializerContext) {
|
||||
this.logger = initializerContext.logger.get();
|
||||
this.kibanaVersion = initializerContext.env.packageInfo.version;
|
||||
}
|
||||
|
||||
public setup(core: CoreSetup): MetricsEntitiesPluginSetup {
|
||||
const router = core.http.createRouter();
|
||||
|
||||
core.http.registerRouteHandlerContext<MetricsEntitiesRequestHandlerContext, 'metricsEntities'>(
|
||||
'metricsEntities',
|
||||
this.createRouteHandlerContext()
|
||||
);
|
||||
|
||||
// Register server side APIs
|
||||
// TODO: Add all of these into a separate file and call that file called init_routes.ts
|
||||
getTransforms(router);
|
||||
postTransforms(router);
|
||||
deleteTransforms(router);
|
||||
|
||||
return {
|
||||
getMetricsEntitiesClient: (esClient): MetricsEntitiesClient =>
|
||||
new MetricsEntitiesClient({
|
||||
esClient,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
logger: this.logger,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
public start(core: CoreStart): void {
|
||||
this.logger.debug('Starting plugin');
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.logger.debug('Stopping plugin');
|
||||
}
|
||||
|
||||
private createRouteHandlerContext = (): ContextProvider => {
|
||||
return async (context): ContextProviderReturn => {
|
||||
const {
|
||||
core: {
|
||||
elasticsearch: {
|
||||
client: { asCurrentUser: esClient },
|
||||
},
|
||||
},
|
||||
} = context;
|
||||
return {
|
||||
getMetricsEntitiesClient: (): MetricsEntitiesClient =>
|
||||
new MetricsEntitiesClient({
|
||||
esClient,
|
||||
kibanaVersion: this.kibanaVersion,
|
||||
logger: this.logger,
|
||||
}),
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
import { IRouter } from '../../../../../src/core/server';
|
||||
import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants';
|
||||
import { ModuleNames } from '../modules';
|
||||
|
||||
import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client';
|
||||
|
||||
/**
|
||||
* Deletes transforms.
|
||||
* NOTE: We use a POST rather than a DELETE on purpose here to ensure that there
|
||||
* are not problems with the body being sent.
|
||||
* @param router The router to delete the collection of transforms
|
||||
*/
|
||||
export const deleteTransforms = (router: IRouter): void => {
|
||||
router.post(
|
||||
{
|
||||
path: `${METRICS_ENTITIES_TRANSFORMS}/_delete`,
|
||||
validate: {
|
||||
// TODO: Add the validation instead of allowing handler to have access to raw non-validated in runtime
|
||||
body: schema.object({}, { unknowns: 'allow' }),
|
||||
query: schema.object({}, { unknowns: 'allow' }),
|
||||
},
|
||||
},
|
||||
async (context, request, response) => {
|
||||
// TODO: Type this through validation above and remove the weird casting of: "as { modules: ModuleNames };"
|
||||
// TODO: Validate for runtime that the module exists or not and throw before pushing the module name lower
|
||||
// TODO: Change modules to be part of the body and become an array of values
|
||||
// TODO: Wrap this in a try catch block and report errors
|
||||
const { modules, prefix = '', suffix = '' } = request.body as {
|
||||
modules: ModuleNames[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
};
|
||||
const metrics = getMetricsEntitiesClient(context);
|
||||
await metrics.deleteTransforms({ modules, prefix, suffix });
|
||||
|
||||
return response.custom({
|
||||
statusCode: 204,
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IRouter } from '../../../../../src/core/server';
|
||||
import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants';
|
||||
|
||||
import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client';
|
||||
|
||||
/**
|
||||
* Returns all transforms from all modules
|
||||
* TODO: Add support for specific modules and prefix
|
||||
* @param router The router to get the collection of transforms
|
||||
*/
|
||||
export const getTransforms = (router: IRouter): void => {
|
||||
router.get(
|
||||
{
|
||||
path: METRICS_ENTITIES_TRANSFORMS,
|
||||
// TODO: Add the validation instead of false
|
||||
// TODO: Add the prefix and module support
|
||||
validate: false,
|
||||
},
|
||||
async (context, _, response) => {
|
||||
const metrics = getMetricsEntitiesClient(context);
|
||||
const summaries = await metrics.getTransforms();
|
||||
return response.ok({
|
||||
body: {
|
||||
summaries,
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
10
x-pack/plugins/metrics_entities/server/routes/index.ts
Normal file
10
x-pack/plugins/metrics_entities/server/routes/index.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './delete_transforms';
|
||||
export * from './get_transforms';
|
||||
export * from './post_transforms';
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
|
||||
import { IRouter } from '../../../../../src/core/server';
|
||||
import { METRICS_ENTITIES_TRANSFORMS } from '../../common/constants';
|
||||
import { ModuleNames } from '../modules';
|
||||
|
||||
import { getMetricsEntitiesClient } from './utils/get_metrics_entities_client';
|
||||
|
||||
/**
|
||||
* Creates transforms.
|
||||
* @param router The router to get the collection of transforms
|
||||
*/
|
||||
export const postTransforms = (router: IRouter): void => {
|
||||
router.post(
|
||||
{
|
||||
path: METRICS_ENTITIES_TRANSFORMS,
|
||||
validate: {
|
||||
// TODO: Add the validation instead of allowing handler to have access to raw non-validated in runtime
|
||||
body: schema.object({}, { unknowns: 'allow' }),
|
||||
query: schema.object({}, { unknowns: 'allow' }),
|
||||
},
|
||||
},
|
||||
async (context, request, response) => {
|
||||
// TODO: Type this through validation above and remove the weird casting of: "as { modules: ModuleNames };"
|
||||
// TODO: Validate for runtime that the module exists or not and throw before pushing the module name lower
|
||||
// TODO: Change modules to be part of the body and become an array of values
|
||||
// TODO: Wrap this in a try catch block and report errors
|
||||
const {
|
||||
modules,
|
||||
auto_start: autoStart = false,
|
||||
settings: {
|
||||
max_page_search_size: maxPageSearchSize = 500,
|
||||
docs_per_second: docsPerSecond = null,
|
||||
} = {
|
||||
docsPerSecond: null,
|
||||
maxPageSearchSize: 500,
|
||||
},
|
||||
frequency = '1m',
|
||||
indices,
|
||||
query = { match_all: {} },
|
||||
prefix = '',
|
||||
suffix = '',
|
||||
sync = {
|
||||
time: {
|
||||
delay: '60s',
|
||||
field: '@timestamp',
|
||||
},
|
||||
},
|
||||
} = request.body as {
|
||||
modules: ModuleNames[];
|
||||
auto_start: boolean;
|
||||
indices: string[];
|
||||
// We can blow up at 65 character+ for transform id. We need to validate the prefix + transform jobs and return an error
|
||||
prefix: string;
|
||||
query: object;
|
||||
suffix: string;
|
||||
frequency: string;
|
||||
settings: {
|
||||
max_page_search_size: number;
|
||||
docs_per_second: number;
|
||||
};
|
||||
sync: {
|
||||
time: {
|
||||
delay: string;
|
||||
field: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
const metrics = getMetricsEntitiesClient(context);
|
||||
await metrics.postTransforms({
|
||||
autoStart,
|
||||
docsPerSecond,
|
||||
frequency,
|
||||
indices,
|
||||
maxPageSearchSize,
|
||||
modules,
|
||||
prefix,
|
||||
query,
|
||||
suffix,
|
||||
sync,
|
||||
});
|
||||
|
||||
return response.custom({
|
||||
body: { acknowledged: true },
|
||||
statusCode: 201,
|
||||
});
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ErrorWithStatusCode } from '../../error_with_status_code';
|
||||
import { MetricsEntitiesClient } from '../../services/metrics_entities_client';
|
||||
import type { MetricsEntitiesRequestHandlerContext } from '../../types';
|
||||
|
||||
export const getMetricsEntitiesClient = (
|
||||
context: MetricsEntitiesRequestHandlerContext
|
||||
): MetricsEntitiesClient => {
|
||||
const metricsEntities = context.metricsEntities?.getMetricsEntitiesClient();
|
||||
if (metricsEntities == null) {
|
||||
throw new ErrorWithStatusCode('Metrics Entities is not found as a plugin', 404);
|
||||
} else {
|
||||
return metricsEntities;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './get_metrics_entities_client';
|
||||
32
x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh
Executable file
32
x-pack/plugins/metrics_entities/server/scripts/check_env_variables.sh
Executable file
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
# Add this to the start of any scripts to detect if env variables are set
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "${ELASTICSEARCH_USERNAME}" ]; then
|
||||
echo "Set ELASTICSEARCH_USERNAME in your environment"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${ELASTICSEARCH_PASSWORD}" ]; then
|
||||
echo "Set ELASTICSEARCH_PASSWORD in your environment"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${ELASTICSEARCH_URL}" ]; then
|
||||
echo "Set ELASTICSEARCH_URL in your environment"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${KIBANA_URL}" ]; then
|
||||
echo "Set KIBANA_URL in your environment"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"prefix": "all",
|
||||
"modules": [
|
||||
"host_metrics",
|
||||
"host_entities",
|
||||
"network_metrics",
|
||||
"network_entities",
|
||||
"user_entities",
|
||||
"user_metrics"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"prefix": "auditbeat",
|
||||
"modules": [
|
||||
"host_metrics",
|
||||
"host_entities",
|
||||
"network_metrics",
|
||||
"network_entities",
|
||||
"user_entities",
|
||||
"user_metrics"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"modules": ["network_entities"]
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"modules": ["user_entities"]
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"modules": ["host_metrics", "host_entities"]
|
||||
}
|
||||
23
x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh
Executable file
23
x-pack/plugins/metrics_entities/server/scripts/delete_transforms.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
set -e
|
||||
./check_env_variables.sh
|
||||
|
||||
# Uses a default if no argument is specified
|
||||
FILE=${1:-./post_examples/one_module.json}
|
||||
|
||||
# Example: ./delete_transforms.sh ./delete_examples/one_module.json
|
||||
curl -s -k \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'kbn-xsrf: 123' \
|
||||
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
|
||||
-X POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms/_delete \
|
||||
-d @${FILE} \
|
||||
| jq .
|
||||
16
x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh
Executable file
16
x-pack/plugins/metrics_entities/server/scripts/get_transforms.sh
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
set -e
|
||||
./check_env_variables.sh
|
||||
|
||||
# Example: ./get_transforms.sh
|
||||
curl -s -k \
|
||||
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
|
||||
-X GET ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms | jq .
|
||||
17
x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh
Executable file
17
x-pack/plugins/metrics_entities/server/scripts/hard_reset.sh
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
# TODO Make this work
|
||||
|
||||
set -e
|
||||
./check_env_variables.sh
|
||||
|
||||
# remove all templates
|
||||
# add all templates again and start them
|
||||
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
{
|
||||
"prefix": "all",
|
||||
"modules": [
|
||||
"host_metrics",
|
||||
"host_entities",
|
||||
"network_metrics",
|
||||
"network_entities",
|
||||
"user_entities",
|
||||
"user_metrics"
|
||||
],
|
||||
"indices": [
|
||||
"auditbeat-*",
|
||||
"endgame-*",
|
||||
"filebeat-*",
|
||||
"logs-*",
|
||||
"packetbeat-*",
|
||||
"winlogbeat-*",
|
||||
"-*elastic-cloud-logs-*"
|
||||
],
|
||||
"auto_start": true,
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
},
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"modules": [
|
||||
"host_metrics",
|
||||
"host_entities",
|
||||
"network_metrics",
|
||||
"network_entities",
|
||||
"user_entities",
|
||||
"user_metrics"
|
||||
],
|
||||
"indices": ["auditbeat-*"],
|
||||
"auto_start": true,
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
},
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"modules": ["network_entities"],
|
||||
"indices": ["auditbeat-*"]
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"modules": ["network_metrics"],
|
||||
"indices": [
|
||||
"auditbeat-*",
|
||||
"endgame-*",
|
||||
"filebeat-*",
|
||||
"logs-*",
|
||||
"packetbeat-*",
|
||||
"winlogbeat-*",
|
||||
"-*elastic-cloud-logs-*"
|
||||
],
|
||||
"auto_start": true,
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"modules": ["network_metrics"],
|
||||
"indices": ["auditbeat-*"],
|
||||
"auto_start": true,
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"modules": ["host_metrics"],
|
||||
"indices": ["auditbeat-*"],
|
||||
"auto_start": true,
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"modules": ["host_metrics"],
|
||||
"indices": ["auditbeat-*"],
|
||||
"prefix": ["default_"]
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"modules": ["network_metrics", "network_entities"],
|
||||
"indices": [
|
||||
"auditbeat-*",
|
||||
"endgame-*",
|
||||
"filebeat-*",
|
||||
"logs-*",
|
||||
"packetbeat-*",
|
||||
"winlogbeat-*",
|
||||
"-*elastic-cloud-logs-*"
|
||||
],
|
||||
"auto_start": true,
|
||||
"query": {
|
||||
"range": {
|
||||
"@timestamp": {
|
||||
"gte": "now-1d/d",
|
||||
"format": "strict_date_optional_time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"settings": {
|
||||
"max_page_search_size": 5000
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"modules": ["host_metrics", "host_entities"],
|
||||
"indices": ["auditbeat-*"]
|
||||
}
|
||||
24
x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh
Executable file
24
x-pack/plugins/metrics_entities/server/scripts/post_transforms.sh
Executable file
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
set -e
|
||||
./check_env_variables.sh
|
||||
|
||||
# Uses a default if no argument is specified
|
||||
FILE=${1:-./post_examples/one_module_auditbeat.json}
|
||||
|
||||
# Example: ./post_transforms.sh ./post_examples/one_module_auditbeat.json
|
||||
# Example: ./post_transforms.sh ./post_examples/one_module_namespace_auditbeat.json
|
||||
curl -s -k \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'kbn-xsrf: 123' \
|
||||
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
|
||||
-X POST ${KIBANA_URL}${SPACE_URL}/api/metrics_entities/transforms \
|
||||
-d @${FILE} \
|
||||
| jq .
|
||||
13
x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh
Executable file
13
x-pack/plugins/metrics_entities/server/scripts/update_transforms.sh
Executable file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License
|
||||
# 2.0; you may not use this file except in compliance with the Elastic License
|
||||
# 2.0.
|
||||
#
|
||||
|
||||
set -e
|
||||
./check_env_variables.sh
|
||||
|
||||
# TODO Make this work
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { ModuleNames, installableMappings, installableTransforms } from '../modules';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import { uninstallMappings } from './uninstall_mappings';
|
||||
import { uninstallTransforms } from './uninstall_transforms';
|
||||
|
||||
interface DeleteTransformsOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
logger: Logger;
|
||||
modules: ModuleNames[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
}
|
||||
|
||||
export const deleteTransforms = async ({
|
||||
esClient,
|
||||
logger,
|
||||
modules,
|
||||
prefix,
|
||||
suffix,
|
||||
}: DeleteTransformsOptions): Promise<void> => {
|
||||
for (const moduleName of modules) {
|
||||
const mappings = installableMappings[moduleName];
|
||||
const transforms = installableTransforms[moduleName];
|
||||
|
||||
await uninstallTransforms({ esClient, logger, prefix, suffix, transforms });
|
||||
await uninstallMappings({ esClient, logger, mappings, prefix, suffix });
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
interface GetTransformsOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
// TODO: Type the Promise<unknown> to a stronger type
|
||||
export const getTransforms = async ({ esClient }: GetTransformsOptions): Promise<unknown> => {
|
||||
const { body } = await esClient.transform.getTransform({
|
||||
size: 1000,
|
||||
transform_id: '*',
|
||||
});
|
||||
return body;
|
||||
};
|
||||
15
x-pack/plugins/metrics_entities/server/services/index.ts
Normal file
15
x-pack/plugins/metrics_entities/server/services/index.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './delete_transforms';
|
||||
export * from './get_transforms';
|
||||
export * from './install_mappings';
|
||||
export * from './install_transforms';
|
||||
export * from './metrics_entities_client';
|
||||
export * from './post_transforms';
|
||||
export * from './uninstall_mappings';
|
||||
export * from './uninstall_transforms';
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { Mappings } from '../modules/types';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import {
|
||||
computeMappingId,
|
||||
getIndexExists,
|
||||
logMappingDebug,
|
||||
logMappingError,
|
||||
logMappingInfo,
|
||||
} from './utils';
|
||||
|
||||
interface CreateMappingOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
mappings: Mappings[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
logger: Logger;
|
||||
kibanaVersion: string;
|
||||
}
|
||||
|
||||
export const installMappings = async ({
|
||||
esClient,
|
||||
kibanaVersion,
|
||||
mappings,
|
||||
prefix,
|
||||
suffix,
|
||||
logger,
|
||||
}: CreateMappingOptions): Promise<void> => {
|
||||
for (const mapping of mappings) {
|
||||
const { index } = mapping.mappings._meta;
|
||||
const mappingId = computeMappingId({ id: index, prefix, suffix });
|
||||
const exists = await getIndexExists(esClient, mappingId);
|
||||
const computedBody = {
|
||||
...mapping,
|
||||
...{
|
||||
mappings: {
|
||||
...mapping.mappings,
|
||||
_meta: {
|
||||
...mapping.mappings._meta,
|
||||
...{
|
||||
created_by: 'metrics_entities',
|
||||
index: mappingId,
|
||||
version: kibanaVersion,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
if (!exists) {
|
||||
try {
|
||||
logMappingInfo({ id: mappingId, logger, message: 'does not exist, creating the mapping' });
|
||||
await esClient.indices.create({
|
||||
body: computedBody,
|
||||
index: mappingId,
|
||||
});
|
||||
} catch (error) {
|
||||
logMappingError({
|
||||
error,
|
||||
id: mappingId,
|
||||
logger,
|
||||
message: 'cannot install mapping',
|
||||
postBody: computedBody,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
logMappingDebug({
|
||||
id: mappingId,
|
||||
logger,
|
||||
message: 'mapping already exists. It will not be recreated',
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { Transforms } from '../modules/types';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import {
|
||||
computeMappingId,
|
||||
computeTransformId,
|
||||
getTransformExists,
|
||||
logTransformDebug,
|
||||
logTransformError,
|
||||
logTransformInfo,
|
||||
} from './utils';
|
||||
|
||||
interface CreateTransformOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
transforms: Transforms[];
|
||||
autoStart: boolean;
|
||||
indices: string[];
|
||||
frequency: string;
|
||||
logger: Logger;
|
||||
query: object;
|
||||
docsPerSecond: number | null;
|
||||
maxPageSearchSize: number;
|
||||
sync: {
|
||||
time: {
|
||||
delay: string;
|
||||
field: string;
|
||||
};
|
||||
};
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
}
|
||||
|
||||
export const installTransforms = async ({
|
||||
autoStart,
|
||||
esClient,
|
||||
frequency,
|
||||
indices,
|
||||
docsPerSecond,
|
||||
logger,
|
||||
maxPageSearchSize,
|
||||
prefix,
|
||||
suffix,
|
||||
transforms,
|
||||
query,
|
||||
sync,
|
||||
}: CreateTransformOptions): Promise<void> => {
|
||||
for (const transform of transforms) {
|
||||
const destIndex = transform?.dest?.index ?? transform.id;
|
||||
const computedMappingIndex = computeMappingId({ id: destIndex, prefix, suffix });
|
||||
const { id, ...transformNoId } = {
|
||||
...transform,
|
||||
...{ source: { ...transform.source, index: indices, query } },
|
||||
...{ dest: { ...transform.dest, index: computedMappingIndex } },
|
||||
...{
|
||||
settings: {
|
||||
...transform.settings,
|
||||
docs_per_second: docsPerSecond,
|
||||
max_page_search_size: maxPageSearchSize,
|
||||
},
|
||||
},
|
||||
frequency,
|
||||
sync,
|
||||
};
|
||||
|
||||
const computedName = computeTransformId({ id, prefix, suffix });
|
||||
const exists = await getTransformExists(esClient, computedName);
|
||||
if (!exists) {
|
||||
try {
|
||||
logTransformInfo({
|
||||
id: computedName,
|
||||
logger,
|
||||
message: 'does not exist, creating the transform',
|
||||
});
|
||||
await esClient.transform.putTransform({
|
||||
body: transformNoId,
|
||||
defer_validation: true,
|
||||
transform_id: computedName,
|
||||
});
|
||||
|
||||
if (autoStart) {
|
||||
logTransformInfo({
|
||||
id: computedName,
|
||||
logger,
|
||||
message: 'is being auto started',
|
||||
});
|
||||
await esClient.transform.startTransform({
|
||||
transform_id: computedName,
|
||||
});
|
||||
} else {
|
||||
logTransformInfo({
|
||||
id: computedName,
|
||||
logger,
|
||||
message: 'is not being auto started',
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logTransformError({
|
||||
error,
|
||||
id: computedName,
|
||||
logger,
|
||||
message: 'Could not create and/or start',
|
||||
postBody: transformNoId,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
logTransformDebug({
|
||||
id: computedName,
|
||||
logger,
|
||||
message: 'already exists. It will not be recreated',
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import { getTransforms } from './get_transforms';
|
||||
import {
|
||||
ConstructorOptions,
|
||||
DeleteTransformsOptions,
|
||||
PostTransformsOptions,
|
||||
} from './metrics_entities_client_types';
|
||||
import { postTransforms } from './post_transforms';
|
||||
import { deleteTransforms } from './delete_transforms';
|
||||
|
||||
export class MetricsEntitiesClient {
|
||||
private readonly esClient: ElasticsearchClient;
|
||||
private readonly logger: Logger;
|
||||
private readonly kibanaVersion: string;
|
||||
|
||||
constructor({ esClient, logger, kibanaVersion }: ConstructorOptions) {
|
||||
this.esClient = esClient;
|
||||
this.logger = logger;
|
||||
this.kibanaVersion = kibanaVersion;
|
||||
}
|
||||
|
||||
// TODO: Type the unknown to be stronger
|
||||
public getTransforms = async (): Promise<unknown> => {
|
||||
const { esClient, logger } = this;
|
||||
return getTransforms({ esClient, logger });
|
||||
};
|
||||
|
||||
public postTransforms = async ({
|
||||
autoStart,
|
||||
frequency,
|
||||
docsPerSecond,
|
||||
maxPageSearchSize,
|
||||
modules,
|
||||
indices,
|
||||
prefix,
|
||||
suffix,
|
||||
query,
|
||||
sync,
|
||||
}: PostTransformsOptions): Promise<void> => {
|
||||
const { esClient, logger, kibanaVersion } = this;
|
||||
return postTransforms({
|
||||
autoStart,
|
||||
docsPerSecond,
|
||||
esClient,
|
||||
frequency,
|
||||
indices,
|
||||
kibanaVersion,
|
||||
logger,
|
||||
maxPageSearchSize,
|
||||
modules,
|
||||
prefix,
|
||||
query,
|
||||
suffix,
|
||||
sync,
|
||||
});
|
||||
};
|
||||
|
||||
public deleteTransforms = async ({
|
||||
modules,
|
||||
prefix,
|
||||
suffix,
|
||||
}: DeleteTransformsOptions): Promise<void> => {
|
||||
const { esClient, logger } = this;
|
||||
return deleteTransforms({ esClient, logger, modules, prefix, suffix });
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
import { ModuleNames } from '../modules';
|
||||
|
||||
export interface ConstructorOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
logger: Logger;
|
||||
kibanaVersion: string;
|
||||
}
|
||||
|
||||
export interface PostTransformsOptions {
|
||||
modules: ModuleNames[];
|
||||
autoStart: boolean;
|
||||
frequency: string;
|
||||
indices: string[];
|
||||
docsPerSecond: number | null;
|
||||
maxPageSearchSize: number;
|
||||
prefix: string;
|
||||
query: object;
|
||||
suffix: string;
|
||||
sync: {
|
||||
time: {
|
||||
delay: string;
|
||||
field: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export interface DeleteTransformsOptions {
|
||||
modules: ModuleNames[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { ModuleNames, installableMappings, installableTransforms } from '../modules';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import { installMappings } from './install_mappings';
|
||||
import { installTransforms } from './install_transforms';
|
||||
|
||||
interface PostTransformsOptions {
|
||||
logger: Logger;
|
||||
esClient: ElasticsearchClient;
|
||||
modules: ModuleNames[];
|
||||
autoStart: boolean;
|
||||
frequency: string;
|
||||
indices: string[];
|
||||
docsPerSecond: number | null;
|
||||
kibanaVersion: string;
|
||||
maxPageSearchSize: number;
|
||||
query: object;
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
sync: {
|
||||
time: {
|
||||
delay: string;
|
||||
field: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export const postTransforms = async ({
|
||||
autoStart,
|
||||
logger,
|
||||
esClient,
|
||||
frequency,
|
||||
indices,
|
||||
docsPerSecond,
|
||||
kibanaVersion,
|
||||
maxPageSearchSize,
|
||||
modules,
|
||||
prefix,
|
||||
suffix,
|
||||
query,
|
||||
sync,
|
||||
}: PostTransformsOptions): Promise<void> => {
|
||||
for (const moduleName of modules) {
|
||||
const mappings = installableMappings[moduleName];
|
||||
const transforms = installableTransforms[moduleName];
|
||||
|
||||
await installMappings({ esClient, kibanaVersion, logger, mappings, prefix, suffix });
|
||||
await installTransforms({
|
||||
autoStart,
|
||||
docsPerSecond,
|
||||
esClient,
|
||||
frequency,
|
||||
indices,
|
||||
logger,
|
||||
maxPageSearchSize,
|
||||
prefix,
|
||||
query,
|
||||
suffix,
|
||||
sync,
|
||||
transforms,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
// TODO: Write this
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { Mappings } from '../modules/types';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import { computeMappingId, logMappingInfo } from './utils';
|
||||
import { logMappingError } from './utils/log_mapping_error';
|
||||
|
||||
interface UninstallMappingOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
mappings: Mappings[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
export const uninstallMappings = async ({
|
||||
esClient,
|
||||
logger,
|
||||
mappings,
|
||||
prefix,
|
||||
suffix,
|
||||
}: UninstallMappingOptions): Promise<void> => {
|
||||
const indices = mappings.map((mapping) => {
|
||||
const { index } = mapping.mappings._meta;
|
||||
return computeMappingId({ id: index, prefix, suffix });
|
||||
});
|
||||
logMappingInfo({
|
||||
id: indices.join(),
|
||||
logger,
|
||||
message: 'deleting indices',
|
||||
});
|
||||
try {
|
||||
await esClient.indices.delete({
|
||||
allow_no_indices: true,
|
||||
ignore_unavailable: true,
|
||||
index: indices,
|
||||
});
|
||||
} catch (error) {
|
||||
logMappingError({
|
||||
error,
|
||||
id: indices.join(),
|
||||
logger,
|
||||
message: 'could not delete index',
|
||||
postBody: undefined,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
import { Transforms } from '../modules/types';
|
||||
import type { Logger } from '../../../../../src/core/server';
|
||||
|
||||
import {
|
||||
computeTransformId,
|
||||
getTransformExists,
|
||||
logTransformError,
|
||||
logTransformInfo,
|
||||
} from './utils';
|
||||
|
||||
interface UninstallTransformsOptions {
|
||||
esClient: ElasticsearchClient;
|
||||
transforms: Transforms[];
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Uninstalls all the transforms underneath a given module
|
||||
*/
|
||||
export const uninstallTransforms = async ({
|
||||
esClient,
|
||||
logger,
|
||||
prefix,
|
||||
suffix,
|
||||
transforms,
|
||||
}: UninstallTransformsOptions): Promise<void> => {
|
||||
transforms.forEach(async (transform) => {
|
||||
const { id } = transform;
|
||||
const computedId = computeTransformId({ id, prefix, suffix });
|
||||
const exists = await getTransformExists(esClient, computedId);
|
||||
if (exists) {
|
||||
logTransformInfo({
|
||||
id: computedId,
|
||||
logger,
|
||||
message: 'stopping transform',
|
||||
});
|
||||
try {
|
||||
await esClient.transform.stopTransform({
|
||||
allow_no_match: true,
|
||||
force: true,
|
||||
timeout: '5s',
|
||||
transform_id: computedId,
|
||||
wait_for_completion: true,
|
||||
});
|
||||
} catch (error) {
|
||||
logTransformError({
|
||||
error,
|
||||
id: computedId,
|
||||
logger,
|
||||
message: 'Could not stop transform, still attempting to delete it',
|
||||
postBody: undefined,
|
||||
});
|
||||
}
|
||||
logTransformInfo({
|
||||
id: computedId,
|
||||
logger,
|
||||
message: 'deleting transform',
|
||||
});
|
||||
try {
|
||||
await esClient.transform.deleteTransform({
|
||||
force: true,
|
||||
transform_id: computedId,
|
||||
});
|
||||
} catch (error) {
|
||||
logTransformError({
|
||||
error,
|
||||
id: computedId,
|
||||
logger,
|
||||
message: 'Could not create and/or start',
|
||||
postBody: undefined,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
logTransformInfo({
|
||||
id: computedId,
|
||||
logger,
|
||||
message: 'transform does not exist to delete',
|
||||
});
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { computeTransformId } from './compute_transform_id';
|
||||
|
||||
export const computeMappingId = ({
|
||||
prefix,
|
||||
id,
|
||||
suffix,
|
||||
}: {
|
||||
prefix: string;
|
||||
id: string;
|
||||
suffix: string;
|
||||
}): string => {
|
||||
// TODO: This causes issues if above 65 character limit. We should limit the prefix
|
||||
// and anything else on the incoming routes to avoid this causing an issue. We should still
|
||||
// throw here in case I change the prefix or other names and cause issues.
|
||||
const computedId = computeTransformId({ id, prefix, suffix });
|
||||
return `.${computedId}`;
|
||||
};
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ELASTIC_NAME } from '../../../common';
|
||||
|
||||
export const computeTransformId = ({
|
||||
prefix,
|
||||
id,
|
||||
suffix,
|
||||
}: {
|
||||
prefix: string;
|
||||
id: string;
|
||||
suffix: string;
|
||||
}): string => {
|
||||
const prefixExists = prefix.trim() !== '';
|
||||
const suffixExists = suffix.trim() !== '';
|
||||
|
||||
// TODO: Check for invalid characters on the main route for prefixExists and suffixExists and do an invalidation
|
||||
// if either have invalid characters for a job name. Might want to add that same check within the API too at a top level?
|
||||
if (prefixExists && suffixExists) {
|
||||
return `${ELASTIC_NAME}_${prefix}_${id}_${suffix}`;
|
||||
} else if (prefixExists) {
|
||||
return `${ELASTIC_NAME}_${prefix}_${id}`;
|
||||
} else if (suffixExists) {
|
||||
return `${ELASTIC_NAME}_${id}_${suffix}`;
|
||||
} else {
|
||||
return `${ELASTIC_NAME}_${id}`;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
/**
|
||||
* Tried and true, copied forever again and again, the way we check if an index exists
|
||||
* with the least amount of privileges.
|
||||
* @param esClient The client to check if the index already exists
|
||||
* @param index The index to check for
|
||||
* @returns true if it exists, otherwise false
|
||||
*/
|
||||
export const getIndexExists = async (
|
||||
esClient: ElasticsearchClient,
|
||||
index: string
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
const { body: response } = await esClient.search({
|
||||
allow_no_indices: true,
|
||||
body: {
|
||||
terminate_after: 1,
|
||||
},
|
||||
index,
|
||||
size: 0,
|
||||
});
|
||||
return response._shards.total > 0;
|
||||
} catch (err) {
|
||||
if (err.body?.status === 404) {
|
||||
return false;
|
||||
} else {
|
||||
throw err.body ? err.body : err;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
// TODO: Move indent to configuration part or flip to default false
|
||||
export const getJSON = (body: unknown, indent: boolean = true): string =>
|
||||
indent ? JSON.stringify(body, null, 2) : JSON.stringify(body);
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from 'kibana/server';
|
||||
|
||||
export const getTransformExists = async (
|
||||
esClient: ElasticsearchClient,
|
||||
id: string
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
const {
|
||||
body: { count },
|
||||
} = await esClient.transform.getTransform({
|
||||
size: 1000,
|
||||
transform_id: id,
|
||||
});
|
||||
return count > 0;
|
||||
} catch (err) {
|
||||
if (err.body?.status === 404) {
|
||||
return false;
|
||||
} else {
|
||||
throw err.body ? err.body : err;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './compute_mapping_index';
|
||||
export * from './compute_transform_id';
|
||||
export * from './get_index_exists';
|
||||
export * from './get_transform_exists';
|
||||
export * from './log_mapping_debug';
|
||||
export * from './log_mapping_error';
|
||||
export * from './log_mapping_info';
|
||||
export * from './log_transform_debug';
|
||||
export * from './log_transform_error';
|
||||
export * from './log_transform_info';
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
export const logMappingDebug = ({
|
||||
logger,
|
||||
id,
|
||||
message,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
message: string;
|
||||
}): void => {
|
||||
logger.debug(`mapping id: "${id}", ${message}`);
|
||||
};
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
import { getJSON } from './get_json';
|
||||
|
||||
export const logMappingError = ({
|
||||
logger,
|
||||
id,
|
||||
message,
|
||||
error,
|
||||
postBody,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
error: unknown;
|
||||
message: string;
|
||||
postBody: {} | undefined;
|
||||
}): void => {
|
||||
const postString = postBody != null ? `, post body: "${getJSON(postBody)}"` : '';
|
||||
logger.error(`${message}, mapping id: "${id}"${postString}, error: ${getJSON(error)}`);
|
||||
};
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
export const logMappingInfo = ({
|
||||
logger,
|
||||
id,
|
||||
message,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
message: string;
|
||||
}): void => {
|
||||
logger.info(`mapping id: "${id}", ${message}`);
|
||||
};
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
export const logTransformDebug = ({
|
||||
logger,
|
||||
id,
|
||||
message,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
message: string;
|
||||
}): void => {
|
||||
logger.debug(`transform id: "${id}", ${message}`);
|
||||
};
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
import { getJSON } from './get_json';
|
||||
|
||||
export const logTransformError = ({
|
||||
id,
|
||||
logger,
|
||||
error,
|
||||
postBody,
|
||||
message,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
error: unknown;
|
||||
message: string;
|
||||
postBody: {} | undefined;
|
||||
}): void => {
|
||||
const postString = postBody != null ? `, post body: "${getJSON(postBody)}"` : '';
|
||||
logger.error(`${message}, transform id: ${id}${postString}, response error: ${getJSON(error)}`);
|
||||
};
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Logger } from '../../../../../../src/core/server';
|
||||
|
||||
export const logTransformInfo = ({
|
||||
logger,
|
||||
id,
|
||||
message,
|
||||
}: {
|
||||
logger: Logger;
|
||||
id: string;
|
||||
message: string;
|
||||
}): void => {
|
||||
logger.info(`transform id: "${id}", ${message}`);
|
||||
};
|
||||
36
x-pack/plugins/metrics_entities/server/types.ts
Normal file
36
x-pack/plugins/metrics_entities/server/types.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient, IContextProvider, RequestHandlerContext } from 'kibana/server';
|
||||
|
||||
import { MetricsEntitiesClient } from './services/metrics_entities_client';
|
||||
|
||||
export type GetMetricsEntitiesClientType = (esClient: ElasticsearchClient) => MetricsEntitiesClient;
|
||||
|
||||
export interface MetricsEntitiesPluginSetup {
|
||||
getMetricsEntitiesClient: GetMetricsEntitiesClientType;
|
||||
}
|
||||
|
||||
export type MetricsEntitiesPluginStart = void;
|
||||
|
||||
export type ContextProvider = IContextProvider<
|
||||
MetricsEntitiesRequestHandlerContext,
|
||||
'metricsEntities'
|
||||
>;
|
||||
|
||||
export interface MetricsEntitiesApiRequestHandlerContext {
|
||||
getMetricsEntitiesClient: () => MetricsEntitiesClient;
|
||||
}
|
||||
|
||||
export interface MetricsEntitiesRequestHandlerContext extends RequestHandlerContext {
|
||||
metricsEntities?: MetricsEntitiesApiRequestHandlerContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
export type ContextProviderReturn = Promise<MetricsEntitiesApiRequestHandlerContext>;
|
||||
27
x-pack/plugins/metrics_entities/tsconfig.json
Normal file
27
x-pack/plugins/metrics_entities/tsconfig.json
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"extends": "../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"outDir": "./target/types",
|
||||
"emitDeclarationOnly": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true
|
||||
},
|
||||
"include": [
|
||||
"common/**/*",
|
||||
"public/**/*",
|
||||
"server/**/*",
|
||||
// have to declare *.json explicitly due to https://github.com/microsoft/TypeScript/issues/25636
|
||||
"server/**/*.json",
|
||||
"../../../typings/**/*"
|
||||
],
|
||||
"references": [
|
||||
{ "path": "../../../src/core/tsconfig.json" },
|
||||
{ "path": "../spaces/tsconfig.json" },
|
||||
{ "path": "../security/tsconfig.json" },
|
||||
{ "path": "../licensing/tsconfig.json" },
|
||||
{ "path": "../features/tsconfig.json" },
|
||||
{ "path": "../../../src/plugins/usage_collection/tsconfig.json" },
|
||||
{ "path": "../../../src/plugins/kibana_utils/tsconfig.json" }
|
||||
]
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { TransformConfigSchema } from './transforms/types';
|
||||
import { ENABLE_CASE_CONNECTOR } from '../../cases/common';
|
||||
|
||||
export const APP_ID = 'securitySolution';
|
||||
|
|
@ -38,6 +39,7 @@ export const DEFAULT_INTERVAL_PAUSE = true;
|
|||
export const DEFAULT_INTERVAL_TYPE = 'manual';
|
||||
export const DEFAULT_INTERVAL_VALUE = 300000; // ms
|
||||
export const DEFAULT_TIMEPICKER_QUICK_RANGES = 'timepicker:quickRanges';
|
||||
export const DEFAULT_TRANSFORMS = 'securitySolution:transforms';
|
||||
export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled';
|
||||
export const GLOBAL_HEADER_HEIGHT = 98; // px
|
||||
export const FILTERS_GLOBAL_HEIGHT = 109; // px
|
||||
|
|
@ -106,6 +108,38 @@ export const IP_REPUTATION_LINKS_SETTING_DEFAULT = `[
|
|||
{ "name": "talosIntelligence.com", "url_template": "https://talosintelligence.com/reputation_center/lookup?search={{ip}}" }
|
||||
]`;
|
||||
|
||||
/** The default settings for the transforms */
|
||||
export const defaultTransformsSetting: TransformConfigSchema = {
|
||||
enabled: false,
|
||||
auto_start: true,
|
||||
auto_create: true,
|
||||
query: {
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: 'now-1d/d',
|
||||
format: 'strict_date_optional_time',
|
||||
},
|
||||
},
|
||||
},
|
||||
retention_policy: {
|
||||
time: {
|
||||
field: '@timestamp',
|
||||
max_age: '1w',
|
||||
},
|
||||
},
|
||||
max_page_search_size: 5000,
|
||||
settings: [
|
||||
{
|
||||
prefix: 'all',
|
||||
indices: ['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
data_sources: [
|
||||
['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'],
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
export const DEFAULT_TRANSFORMS_SETTING = JSON.stringify(defaultTransformsSetting, null, 2);
|
||||
|
||||
/**
|
||||
* Id for the signals alerting type
|
||||
*/
|
||||
|
|
@ -214,3 +248,10 @@ export const showAllOthersBucket: string[] = [
|
|||
'destination.ip',
|
||||
'user.name',
|
||||
];
|
||||
|
||||
/**
|
||||
* Used for transforms for metrics_entities. If the security_solutions pulls in
|
||||
* the metrics_entities plugin, then it should pull this constant from there rather
|
||||
* than use it from here.
|
||||
*/
|
||||
export const ELASTIC_NAME = 'estc';
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ export type ExperimentalFeatures = typeof allowedExperimentalValues;
|
|||
*/
|
||||
const allowedExperimentalValues = Object.freeze({
|
||||
trustedAppsByPolicyEnabled: false,
|
||||
metricsEntitiesEnabled: false,
|
||||
eventFilteringEnabled: false,
|
||||
hostIsolationEnabled: false,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -72,9 +72,13 @@ export interface AuthenticationBucket {
|
|||
doc_count: number;
|
||||
failures: {
|
||||
doc_count: number;
|
||||
// TODO: Keep this or make a new structure?
|
||||
value?: number;
|
||||
};
|
||||
successes: {
|
||||
doc_count: number;
|
||||
// TODO: Keep this or make a new structure?
|
||||
value?: number;
|
||||
};
|
||||
authentication: {
|
||||
hits: {
|
||||
|
|
|
|||
|
|
@ -16,9 +16,11 @@ export * from './uncommon_processes';
|
|||
|
||||
export enum HostsQueries {
|
||||
authentications = 'authentications',
|
||||
authenticationsEntities = 'authenticationsEntities',
|
||||
details = 'hostDetails',
|
||||
firstOrLastSeen = 'firstOrLastSeen',
|
||||
hosts = 'hosts',
|
||||
hostsEntities = 'hostsEntities',
|
||||
overview = 'overviewHost',
|
||||
uncommonProcesses = 'uncommonProcesses',
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue