[Synthetics] add synthetics-private-location command (#189531)

# Summary

Quickily start Fleet, enroll Elastic Agent, and create a private
location.

## Usage

```
node x-pack/scripts/synthetics_private_location.js
```

For available options, run `--help`.

## Prerequistes

This script requires `docker` and the following `kibama.yml`
configuration.

```
# Create an agent policy for Fleet Server.
xpack.fleet.agentPolicies:
  - name: Fleet Server policy
    id: fleet-server-policy
    is_default_fleet_server: true
    # is_managed: true # Useful to mimic cloud environment
    description: Fleet server policy
    namespace: default
    package_policies:
      - name: Fleet Server
        package:
          name: fleet_server
        inputs:
          - type: fleet-server
            keep_enabled: true
            vars:
              - name: host
                value: 0.0.0.0
                frozen: true
              - name: port
                value: 8220
                frozen: true

# Set a default Fleet Server host.
xpack.fleet.fleetServerHosts:
  - id: default-fleet-server
    name: Default Fleet server
    is_default: true
    host_urls: ['https://host.docker.internal:8220'] # For running a Fleet Server Docker container

# Set a default Elasticsearch output.
xpack.fleet.outputs:
  - id: es-default-output
    name: Default output
    type: elasticsearch
    is_default: true
    is_default_monitoring: true
    hosts: ['http://host.docker.internal:9200'] # For enrolling dockerized agents
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Shahzad <shahzad31comp@gmail.com>
This commit is contained in:
Dominique Clarke 2024-08-09 07:02:02 -04:00 committed by GitHub
parent 1391836d7d
commit a049461f44
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 567 additions and 0 deletions

1
.github/CODEOWNERS vendored
View file

@ -862,6 +862,7 @@ packages/kbn-stdio-dev-helpers @elastic/kibana-operations
packages/kbn-storybook @elastic/kibana-operations
x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team
x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team
x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team
x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture @elastic/response-ops
x-pack/test/plugin_api_perf/plugins/task_manager_performance @elastic/response-ops
x-pack/plugins/task_manager @elastic/response-ops

View file

@ -870,6 +870,7 @@
"@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
"@kbn/std": "link:packages/kbn-std",
"@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics",
"@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location",
"@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture",
"@kbn/task-manager-performance-plugin": "link:x-pack/test/plugin_api_perf/plugins/task_manager_performance",
"@kbn/task-manager-plugin": "link:x-pack/plugins/task_manager",

View file

@ -1718,6 +1718,8 @@
"@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"],
"@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"],
"@kbn/synthetics-plugin/*": ["x-pack/plugins/observability_solution/synthetics/*"],
"@kbn/synthetics-private-location": ["x-pack/packages/kbn-synthetics-private-location"],
"@kbn/synthetics-private-location/*": ["x-pack/packages/kbn-synthetics-private-location/*"],
"@kbn/task-manager-fixture-plugin": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture"],
"@kbn/task-manager-fixture-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture/*"],
"@kbn/task-manager-performance-plugin": ["x-pack/test/plugin_api_perf/plugins/task_manager_performance"],

View file

@ -0,0 +1,56 @@
# @kbn/synthetics-private-location
Quickily start Fleet, enroll Elastic Agent, and create a private location.
## Usage
```
node x-pack/scripts/synthetics_private_location.js
```
For available options, run `--help`.
## Prerequistes
This script requires `docker` and the following `kibama.yml` configuration.
```
# Create an agent policy for Fleet Server.
xpack.fleet.agentPolicies:
- name: Fleet Server policy
id: fleet-server-policy
is_default_fleet_server: true
# is_managed: true # Useful to mimic cloud environment
description: Fleet server policy
namespace: default
package_policies:
- name: Fleet Server
package:
name: fleet_server
inputs:
- type: fleet-server
keep_enabled: true
vars:
- name: host
value: 0.0.0.0
frozen: true
- name: port
value: 8220
frozen: true
# Set a default Fleet Server host.
xpack.fleet.fleetServerHosts:
- id: default-fleet-server
name: Default Fleet server
is_default: true
host_urls: ['https://host.docker.internal:8220'] # For running a Fleet Server Docker container
# Set a default Elasticsearch output.
xpack.fleet.outputs:
- id: es-default-output
name: Default output
type: elasticsearch
is_default: true
is_default_monitoring: true
hosts: ['http://host.docker.internal:9200'] # For enrolling dockerized agents
```

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export type { CliOptions } from './src/types';
export { run } from './src/run';
export { cli } from './src/cli';
// export { cleanup } from './src/cleanup';
export { DEFAULTS } from './src/constants';

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
module.exports = {
preset: '@kbn/test',
rootDir: '../../..',
roots: ['<rootDir>/x-pack/packages/kbn-synthetics-private-location'],
};

View file

@ -0,0 +1,5 @@
{
"type": "shared-common",
"id": "@kbn/synthetics-private-location",
"owner": "@elastic/obs-ux-management-team"
}

View file

@ -0,0 +1,6 @@
{
"name": "@kbn/synthetics-private-location",
"private": true,
"version": "1.0.0",
"license": "Elastic License 2.0"
}

View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ToolingLog } from '@kbn/tooling-log';
import { parseCliOptions } from './lib/parse_cli_options';
import { CliOptions } from './types';
import { run } from './run';
export async function cli(cliOptions?: CliOptions) {
const options = cliOptions ?? parseCliOptions();
const logger = new ToolingLog({ level: 'info', writeTo: process.stdout });
return run(options, logger);
}

View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { v4 as uuidv4 } from 'uuid';
export const DEFAULTS = {
LOCATION_NAME: `Default location ${uuidv4()}`,
AGENT_POLICY_NAME: `Synthetics agent policy ${uuidv4()}`,
ELASTICSEARCH_HOST: 'http://localhost:9200',
KIBANA_URL: 'http://localhost:5601',
KIBANA_USERNAME: 'elastic',
KIBANA_PASSWORD: 'changeme',
};

View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isError } from 'lodash';
import { ToolingLog } from '@kbn/tooling-log';
import { CliOptions } from '../types';
import type { KibanaAPIClient } from './kibana_api_client';
export async function createElasticAgentPolicy(
{ agentPolicyName }: CliOptions,
logger: ToolingLog,
kibanaApiClient: KibanaAPIClient
) {
try {
const response = await kibanaApiClient.sendRequest({
method: 'post',
url: 'api/fleet/agent_policies',
data: {
name: agentPolicyName,
description: '',
namespace: 'default',
monitoring_enabled: ['logs', 'metrics'],
inactivity_timeout: 1209600,
is_protected: false,
},
});
logger.info(`Generated elastic agent policy`);
return response.data;
} catch (error) {
if (isError(error)) {
logger.error(`Error generating elastic agent policy: ${error.message} ${error.stack}`);
}
throw error;
}
}

View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isError } from 'lodash';
import { ToolingLog } from '@kbn/tooling-log';
import { CliOptions } from '../types';
import { KibanaAPIClient } from './kibana_api_client';
export async function createPrivateLocation(
{ kibanaUrl, kibanaPassword, kibanaUsername, locationName }: CliOptions,
logger: ToolingLog,
kibanaApiClient: KibanaAPIClient,
agentPolicyId: string
) {
try {
const response = await kibanaApiClient.sendRequest({
method: 'post',
url: 'api/synthetics/private_locations',
data: {
label: locationName,
agentPolicyId,
},
});
logger.info(`Synthetics private location created successfully`);
return response.data;
} catch (error) {
if (isError(error)) {
logger.error(`Error creating synthetics private location: ${error.message} ${error.stack}`);
}
throw error;
}
}

View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { spawn, spawnSync } from 'child_process';
import * as path from 'path';
import { CliOptions } from '../types';
import { KibanaAPIClient } from './kibana_api_client';
export async function enrollAgent(
{ kibanaUrl, elasticsearchHost }: CliOptions,
enrollmentToken: string,
kibanaApiClient: KibanaAPIClient
) {
const formattedKibanaURL = new URL(kibanaUrl);
const formattedElasticsearchHost = new URL(elasticsearchHost);
if (formattedKibanaURL.hostname === 'localhost') {
formattedKibanaURL.hostname = 'host.docker.internal';
}
if (formattedElasticsearchHost.hostname === 'localhost') {
formattedElasticsearchHost.hostname = 'host.docker.internal';
}
const version = `${await kibanaApiClient.getKibanaVersion()}-SNAPSHOT`;
await new Promise((res, rej) => {
try {
const fleetProcess = spawn(
'docker',
[
'run',
'-e',
'FLEET_SERVER_ENABLE=1',
'-e',
`FLEET_SERVER_ELASTICSEARCH_HOST=${formattedElasticsearchHost.origin}`,
'-e',
'FLEET_SERVER_POLICY_ID=fleet-server-policy',
'-e',
'FLEET_INSECURE=1',
'-e',
`KIBANA_HOST=${formattedKibanaURL.origin}`,
'-e',
'KIBANA_USERNAME=elastic',
'-e',
'KIBANA_PASSWORD=changeme',
'-e',
'KIBANA_FLEET_SETUP=1',
'-p',
'8220:8220',
'--rm',
`docker.elastic.co/beats/elastic-agent:${version}`,
],
{
shell: true,
cwd: path.join(__dirname, '../'),
timeout: 120000,
}
);
setTimeout(res, 10_000);
fleetProcess.on('error', rej);
} catch (error) {
rej(error);
}
});
spawnSync(
'docker',
[
'run',
'-e',
'FLEET_URL=https://host.docker.internal:8220',
'-e',
'FLEET_ENROLL=1',
'-e',
`FLEET_ENROLLMENT_TOKEN=${enrollmentToken}`,
'-e',
'FLEET_INSECURE=1',
'--rm',
`docker.elastic.co/beats/elastic-agent-complete:${version}`,
],
{
stdio: 'inherit',
}
);
}

View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isError } from 'lodash';
import { ToolingLog } from '@kbn/tooling-log';
import { KibanaAPIClient } from './kibana_api_client';
import { CliOptions } from '../types';
export async function fetchAgentPolicyEnrollmentToken(
{ kibanaUrl, kibanaPassword, kibanaUsername }: CliOptions,
logger: ToolingLog,
kibanaApiClient: KibanaAPIClient,
agentPolicyId: string
) {
try {
const response = await kibanaApiClient.sendRequest({
method: 'get',
url: `api/fleet/enrollment_api_keys?kuery=policy_id:${agentPolicyId}`,
});
logger.info(`Fetching agent policy enrollment token`);
return response.data;
} catch (error) {
if (isError(error)) {
logger.error(`Error fetching agent enrollment token: ${error.message} ${error.stack}`);
}
throw error;
}
}

View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isError } from 'lodash';
import { ToolingLog } from '@kbn/tooling-log';
import { CliOptions } from '../types';
import { KibanaAPIClient } from './kibana_api_client';
export async function generateFleetServiceToken(
{ kibanaUrl, kibanaPassword, kibanaUsername }: CliOptions,
logger: ToolingLog,
kibanaApiClient: KibanaAPIClient
) {
try {
const response = await kibanaApiClient.sendRequest({
method: 'post',
url: 'api/fleet/service_tokens',
});
logger.info(`Generated fleet server service token saved`);
return response.data;
} catch (error) {
if (isError(error)) {
logger.error(`Error generating fleet server service token: ${error.message} ${error.stack}`);
}
throw error;
}
}

View file

@ -0,0 +1,76 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { isError } from 'lodash';
import { ToolingLog } from '@kbn/tooling-log';
import { KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
import fs from 'fs';
import https, { Agent } from 'https';
import axios from 'axios';
export class KibanaAPIClient {
private isHTTPS: boolean;
private httpsAgent: Agent | undefined;
constructor(
private kibanaUrl: string,
private kibanaUsername: string,
private kibanaPassword: string,
private logger: ToolingLog
) {
this.isHTTPS = new URL(kibanaUrl).protocol === 'https:';
this.httpsAgent = this.isHTTPS
? new https.Agent({
ca: fs.readFileSync(KBN_CERT_PATH),
key: fs.readFileSync(KBN_KEY_PATH),
// hard-coded set to false like in packages/kbn-cli-dev-mode/src/base_path_proxy_server.ts
rejectUnauthorized: false,
})
: undefined;
}
public async sendRequest({
method,
url,
data,
headers,
}: {
method: string;
url: string;
data?: Record<string, unknown>;
headers?: Record<string, unknown>;
}) {
try {
const response = await axios({
method,
url: `${this.kibanaUrl}/${url}`,
data,
headers: {
'kbn-xsrf': 'true',
'elastic-api-version': '2023-10-31',
...headers,
},
auth: {
username: this.kibanaUsername,
password: this.kibanaPassword,
},
httpsAgent: this.httpsAgent,
});
return response;
} catch (e) {
if (isError(e)) {
this.logger.error(`Error sending request to Kibana: ${e.message} ${e.stack}`);
}
throw e;
}
}
public async getKibanaVersion() {
const res = await this.sendRequest({ method: 'GET', url: 'api/status' });
return res.data.version.number;
}
}

View file

@ -0,0 +1,48 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Command } from 'commander';
import { CliOptions } from '../types';
import { DEFAULTS } from '../constants';
export function parseCliOptions(): CliOptions {
const program = new Command();
program
.name('synthetics_private_location.js')
.description(
'A script to start Fleet Server, enroll Elastic Agent, and create a Synthetics private location'
)
.option(
'--elasticsearch-host <address>',
'The address to the Elasticsearch cluster',
DEFAULTS.ELASTICSEARCH_HOST
)
.option('--kibana-url <address>', 'The address to the Kibana server', DEFAULTS.KIBANA_URL)
.option(
'--kibana-username <username>',
'The username for the Kibana server',
DEFAULTS.KIBANA_USERNAME
)
.option(
'--kibana-password <password>',
'The password for the Kibana server',
DEFAULTS.KIBANA_PASSWORD
)
.option(
'--location-name <name>',
'The name of the Synthetics private location',
DEFAULTS.LOCATION_NAME
)
.option(
'--agent-policy-name <name>',
'The name of the agent policy',
DEFAULTS.AGENT_POLICY_NAME
);
program.parse(process.argv);
return program.opts() as CliOptions;
}

View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ToolingLog } from '@kbn/tooling-log';
import type { CliOptions } from './types';
import { createElasticAgentPolicy } from './lib/create_agent_policy';
import { fetchAgentPolicyEnrollmentToken } from './lib/fetch_agent_policy_enrollment_token';
import { enrollAgent } from './lib/enroll_agent';
import { createPrivateLocation } from './lib/create_private_location';
import { KibanaAPIClient } from './lib/kibana_api_client';
export async function run(options: CliOptions, logger: ToolingLog) {
const kibanaClient = new KibanaAPIClient(
options.kibanaUrl,
options.kibanaUsername,
options.kibanaPassword,
logger
);
const {
item: { id: agentPolicyId },
} = await createElasticAgentPolicy(options, logger, kibanaClient);
await createPrivateLocation(options, logger, kibanaClient, agentPolicyId);
const { list } = await fetchAgentPolicyEnrollmentToken(
options,
logger,
kibanaClient,
agentPolicyId
);
const [enrollmentTokenConfig] = list;
const { api_key: enrollmentToken } = enrollmentTokenConfig;
enrollAgent(options, enrollmentToken, kibanaClient);
}

View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface CliOptions {
locationName: string;
agentPolicyName: string;
kibanaUrl: string;
kibanaUsername: string;
kibanaPassword: string;
elasticsearchHost: string;
}

View file

@ -0,0 +1,22 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"types": [
"jest",
"node",
"react"
]
},
"include": [
"**/*.ts",
"**/*.tsx",
],
"exclude": [
"target/**/*"
],
"kbn_references": [
"@kbn/tooling-log",
"@kbn/dev-utils",
]
}

View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
require('../../src/setup_node_env');
require('@kbn/synthetics-private-location').cli();

View file

@ -6716,6 +6716,10 @@
version "0.0.0"
uid ""
"@kbn/synthetics-private-location@link:x-pack/packages/kbn-synthetics-private-location":
version "0.0.0"
uid ""
"@kbn/task-manager-fixture-plugin@link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture":
version "0.0.0"
uid ""