[Osquery] Cypress automation for osquery manager integration (#108759)

x-pack/plugins/osquery/cypress/.gitignore

@@ -0,0 +1,2 @@
+videos
+screenshots

x-pack/plugins/osquery/cypress/README.md

@@ -20,7 +20,7 @@ A headless browser is a browser simulation program that does not have a user int
 
 #### FTR (CI)
 
-This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/security_solution_cypress`
+This is the configuration used by CI. It uses the FTR to spawn both a Kibana instance (http://localhost:5620) and an Elasticsearch instance (http://localhost:9220) with a preloaded minimum set of data (see preceding "Test data" section), and then executes cypress against this stack. You can find this configuration in `x-pack/test/osquery_cypress`
 
 ### Test Execution: Examples
 
@@ -36,7 +36,7 @@ yarn kbn bootstrap
 node scripts/build_kibana_platform_plugins
 
 # launch the cypress test runner
-cd x-pack/plugins/security_solution
+cd x-pack/plugins/osquery
 yarn cypress:run-as-ci
 ```
 #### FTR + Interactive
@@ -51,7 +51,7 @@ yarn kbn bootstrap
 node scripts/build_kibana_platform_plugins
 
 # launch the cypress test runner
-cd x-pack/plugins/security_solution
+cd x-pack/plugins/osquery
 yarn cypress:open-as-ci
 ```
 
@@ -98,16 +98,16 @@ We use es_archiver to manage the data that our Cypress tests need.
 
 1. Set up a clean instance of kibana and elasticsearch (if this is not possible, try to clean/minimize the data that you are going to archive).
 2. With the kibana and elasticsearch instance up and running, create the data that you need for your test.
-3. When you are sure that you have all the data you need run the following command from: `x-pack/plugins/security_solution`
+3. When you are sure that you have all the data you need run the following command from: `x-pack/plugins/osquery`
 
 ```sh
-node ../../../scripts/es_archiver save <nameOfTheFolderWhereDataIsSaved> <indexPatternsToBeSaved>  --dir ../../test/security_solution_cypress/es_archives --config ../../../test/functional/config.js --es-url http://<elasticsearchUsername>:<elasticsearchPassword>@<elasticsearchHost>:<elasticsearchPort>
+node ../../../scripts/es_archiver save <nameOfTheFolderWhereDataIsSaved> <indexPatternsToBeSaved>  --dir ../../test/osquery_cypress/es_archives --config ../../../test/functional/config.js --es-url http://<elasticsearchUsername>:<elasticsearchPassword>@<elasticsearchHost>:<elasticsearchPort>
 ```
 
 Example:
 
 ```sh
-node ../../../scripts/es_archiver save custom_rules ".kibana",".siem-signal*"  --dir ../../test/security_solution_cypress/es_archives --config ../../../test/functional/config.js --es-url http://elastic:changeme@localhost:9220
+node ../../../scripts/es_archiver save custom_rules ".kibana",".siem-signal*"  --dir ../../test/osquery_cypress/es_archives --config ../../../test/functional/config.js --es-url http://elastic:changeme@localhost:9220
 ```
 
 Note that the command will create the folder if it does not exist.

x-pack/plugins/osquery/cypress/integration/osquery_manager.spec.ts

@@ -8,13 +8,19 @@
 import { HEADER } from '../screens/osquery';
 import { OSQUERY_NAVIGATION_LINK } from '../screens/navigation';
 
-import { INTEGRATIONS, OSQUERY, openNavigationFlyout, navigateTo } from '../tasks/navigation';
+import { OSQUERY, NEW_LIVE_QUERY, openNavigationFlyout, navigateTo } from '../tasks/navigation';
 import { addIntegration } from '../tasks/integrations';
+import { checkResults, inputQuery, selectAllAgents, submitQuery } from '../tasks/live_query';
 
 describe('Osquery Manager', () => {
-  before(() => {
-    navigateTo(INTEGRATIONS);
-    addIntegration('Osquery Manager');
+  before(() => addIntegration(Cypress.env('OSQUERY_POLICY')));
+
+  it('Runs live queries', () => {
+    navigateTo(NEW_LIVE_QUERY);
+    selectAllAgents();
+    inputQuery();
+    submitQuery();
+    checkResults();
   });
 
   it('Displays Osquery on the navigation flyout once installed ', () => {

x-pack/plugins/osquery/cypress/screens/integrations.ts

@@ -8,3 +8,4 @@
 export const ADD_POLICY_BTN = '[data-test-subj="addIntegrationPolicyButton"]';
 export const CREATE_PACKAGE_POLICY_SAVE_BTN = '[data-test-subj="createPackagePolicySaveButton"]';
 export const INTEGRATIONS_CARD = '.euiCard__titleAnchor';
+export const SAVE_PACKAGE_CONFIRM = '[data-test-subj=confirmModalConfirmButton]';

x-pack/plugins/osquery/cypress/screens/live_query.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const AGENT_FIELD = '[data-test-subj="comboBoxInput"]';
+export const ALL_AGENTS_OPTION = '[title="All agents"]';
+export const LIVE_QUERY_EDITOR = '#osquery_editor';
+export const SUBMIT_BUTTON = '#submit-button';

x-pack/plugins/osquery/cypress/tasks/integrations.ts

@@ -5,16 +5,18 @@
  * 2.0.
  */
 
-import {
-  ADD_POLICY_BTN,
-  CREATE_PACKAGE_POLICY_SAVE_BTN,
-  INTEGRATIONS_CARD,
-} from '../screens/integrations';
+import { CREATE_PACKAGE_POLICY_SAVE_BTN, SAVE_PACKAGE_CONFIRM } from '../screens/integrations';
 
-export const addIntegration = (integration: string) => {
-  cy.get(INTEGRATIONS_CARD).contains(integration).click();
-  cy.get(ADD_POLICY_BTN).click();
+import { navigateTo, OSQUERY_INTEGRATION_PAGE } from './navigation';
+
+// TODO: allow adding integration version strings to this
+export const addIntegration = (policyId: string) => {
+  navigateTo(OSQUERY_INTEGRATION_PAGE, { qs: { policyId } });
   cy.get(CREATE_PACKAGE_POLICY_SAVE_BTN).click();
-  cy.get(CREATE_PACKAGE_POLICY_SAVE_BTN).should('not.exist');
-  cy.reload();
+  cy.get(SAVE_PACKAGE_CONFIRM).click();
+  // XXX: there is a race condition between the test going to the ui powered by the agent, and the agent having the integration ready to go
+  // so we wait.
+  // TODO: actually make this wait til the agent has been updated with the proper integration
+  cy.wait(5000);
+  return cy.reload();
 };

x-pack/plugins/osquery/cypress/tasks/live_query.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  AGENT_FIELD,
+  ALL_AGENTS_OPTION,
+  LIVE_QUERY_EDITOR,
+  SUBMIT_BUTTON,
+} from '../screens/live_query';
+
+export const selectAllAgents = () => {
+  cy.get(AGENT_FIELD).first().click();
+  return cy.get(ALL_AGENTS_OPTION).contains('All agents').click();
+};
+
+export const inputQuery = () => cy.get(LIVE_QUERY_EDITOR).type('select * from processes;');
+
+export const submitQuery = () => cy.get(SUBMIT_BUTTON).contains('Submit').click();
+
+export const checkResults = () =>
+  cy.get('[data-test-subj="dataGridRowCell"]').should('have.lengthOf.above', 0);

x-pack/plugins/osquery/cypress/tasks/navigation.ts

@@ -7,11 +7,13 @@
 
 import { TOGGLE_NAVIGATION_BTN } from '../screens/navigation';
 
-export const INTEGRATIONS = 'app/integrations#/';
 export const OSQUERY = 'app/osquery/live_queries';
-
-export const navigateTo = (page: string) => {
-  cy.visit(page);
+export const NEW_LIVE_QUERY = 'app/osquery/live_queries/new';
+export const OSQUERY_INTEGRATION_PAGE = '/app/fleet/integrations/osquery_manager/add-integration';
+export const navigateTo = (page: string, opts?: Partial<Cypress.VisitOptions>) => {
+  cy.visit(page, opts);
+  // There's a security warning toast that seemingly makes ui elements in the bottom right unavailable, so we close it
+  return cy.get('[data-test-subj="toastCloseButton"]').click();
 };
 
 export const openNavigationFlyout = () => {

x-pack/plugins/osquery/public/live_queries/form/index.tsx

@@ -328,6 +328,7 @@ const LiveQueryFormComponent: React.FC<LiveQueryFormProps> = ({
           )}
           <EuiFlexItem grow={false}>
             <EuiButton
+              id="submit-button"
               disabled={!enabled || !agentSelected || !queryValueProvided || isSubmitting}
               onClick={submit}
             >

x-pack/test/osquery_cypress/agent.ts

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ToolingLog } from '@kbn/dev-utils';
+import axios, { AxiosRequestConfig } from 'axios';
+import { copyFile } from 'fs/promises';
+import { ChildProcess, execFileSync, spawn } from 'child_process';
+import { resolve } from 'path';
+import { unlinkSync } from 'fs';
+import { Manager } from './resource_manager';
+
+interface AgentManagerParams {
+  user: string;
+  password: string;
+  kibanaUrl: string;
+  esHost: string;
+}
+
+export class AgentManager extends Manager {
+  private directoryPath: string;
+  private params: AgentManagerParams;
+  private log: ToolingLog;
+  private agentProcess?: ChildProcess;
+  private requestOptions: AxiosRequestConfig;
+  constructor(directoryPath: string, params: AgentManagerParams, log: ToolingLog) {
+    super();
+    // TODO: check if the file exists
+    this.directoryPath = directoryPath;
+    this.log = log;
+    this.params = params;
+    this.requestOptions = {
+      headers: {
+        'kbn-xsrf': 'kibana',
+      },
+      auth: {
+        username: this.params.user,
+        password: this.params.password,
+      },
+    };
+  }
+
+  public getBinaryPath() {
+    return resolve(this.directoryPath, 'elastic-agent');
+  }
+
+  public async setup() {
+    this.log.info('Running agent preconfig');
+    await axios.post(`${this.params.kibanaUrl}/api/fleet/agents/setup`, {}, this.requestOptions);
+
+    this.log.info('Updating the default agent output');
+    const {
+      data: {
+        items: [defaultOutput],
+      },
+    } = await axios.get(this.params.kibanaUrl + '/api/fleet/outputs', this.requestOptions);
+
+    await axios.put(
+      `${this.params.kibanaUrl}/api/fleet/outputs/${defaultOutput.id}`,
+      { hosts: [this.params.esHost] },
+      this.requestOptions
+    );
+
+    this.log.info('Getting agent enrollment key');
+    const { data: apiKeys } = await axios.get(
+      this.params.kibanaUrl + '/api/fleet/enrollment-api-keys',
+      this.requestOptions
+    );
+    const policy = apiKeys.list[1];
+
+    this.log.info('Enrolling the agent');
+    const args = [
+      'enroll',
+      '--insecure',
+      '-f',
+      // TODO: parse the host/port out of the logs for the fleet server
+      '--url=http://localhost:8220',
+      `--enrollment-token=${policy.api_key}`,
+    ];
+    const agentBinPath = this.getBinaryPath();
+    execFileSync(agentBinPath, args, { stdio: 'inherit' });
+
+    // Copy the config file
+    const configPath = resolve(__dirname, this.directoryPath, 'elastic-agent.yml');
+    this.log.info(`Copying agent config from ${configPath}`);
+    await copyFile(configPath, resolve('.', 'elastic-agent.yml'));
+
+    this.log.info('Running the agent');
+    this.agentProcess = spawn(agentBinPath, ['run', '-v'], { stdio: 'inherit' });
+
+    // Wait til we see the agent is online
+    let done = false;
+    let retries = 0;
+    while (!done) {
+      await new Promise((r) => setTimeout(r, 5000));
+      const { data: agents } = await axios.get(
+        `${this.params.kibanaUrl}/api/fleet/agents`,
+        this.requestOptions
+      );
+      done = agents.list[0]?.status === 'online';
+      if (++retries > 12) {
+        this.log.error('Giving up on enrolling the agent after a minute');
+        throw new Error('Agent timed out while coming online');
+      }
+    }
+    return { policyId: policy.policy_id as string };
+  }
+
+  protected _cleanup() {
+    this.log.info('Cleaning up the agent process');
+    if (this.agentProcess) {
+      if (!this.agentProcess.kill(9)) {
+        this.log.warning('Unable to kill agent process');
+      }
+
+      this.agentProcess.on('close', () => {
+        this.log.info('Agent process closed');
+      });
+      delete this.agentProcess;
+    }
+    unlinkSync(resolve('.', 'elastic-agent.yml'));
+  }
+}

x-pack/test/osquery_cypress/artifact_manager.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios, { AxiosResponse } from 'axios';
+import { get } from 'lodash';
+import { execSync } from 'child_process';
+import { writeFileSync, unlinkSync, rmdirSync } from 'fs';
+import { resolve } from 'path';
+import { ToolingLog } from '@kbn/dev-utils';
+import { Manager } from './resource_manager';
+
+const archMap: { [key: string]: string } = {
+  x64: 'x86_64',
+};
+
+type ArtifactName = 'elastic-agent' | 'fleet-server';
+
+async function getArtifact(
+  artifact: string,
+  urlExtractor: (data: AxiosResponse<any>, filename: string) => string,
+  log: ToolingLog,
+  version: string
+) {
+  log.info(`Fetching ${version} of ${artifact}`);
+  const agents = await axios(
+    `https://artifacts-api.elastic.co/v1/versions/${version}/builds/latest`
+  );
+  const arch = archMap[process.arch] ?? process.arch;
+  const dirName = `${artifact}-${version}-${process.platform}-${arch}`;
+  const filename = dirName + '.tar.gz';
+  const url = urlExtractor(agents.data, filename);
+  if (!url) {
+    log.error(`Could not find url for ${artifact}:  ${url}`);
+    throw new Error(`Unable to fetch ${artifact}`);
+  }
+  log.info(`Fetching ${filename} from ${url}`);
+  const agent = await axios(url as string, { responseType: 'arraybuffer' });
+  writeFileSync(filename, agent.data);
+  execSync(`tar xvf ${filename}`);
+  return resolve(filename);
+}
+
+// There has to be a better way to represent partial function application
+type ArtifactFetcher = (
+  log: Parameters<typeof getArtifact>[2],
+  version: Parameters<typeof getArtifact>[3]
+) => ReturnType<typeof getArtifact>;
+type ArtifactFetchers = {
+  [artifactName in ArtifactName]: ArtifactFetcher;
+};
+
+const fetchers: ArtifactFetchers = {
+  'elastic-agent': getArtifact.bind(null, 'elastic-agent', (data, filename) =>
+    get(data, ['build', 'projects', 'beats', 'packages', filename, 'url'])
+  ),
+  'fleet-server': getArtifact.bind(null, 'fleet-server', (data, filename) =>
+    get(data, ['build', 'projects', 'fleet-server', 'packages', filename, 'url'])
+  ),
+};
+
+export type FetchArtifactsParams = {
+  [artifactName in ArtifactName]?: string;
+};
+
+type ArtifactPaths = FetchArtifactsParams;
+export class ArtifactManager extends Manager {
+  private artifacts: ArtifactPaths;
+  private versions: FetchArtifactsParams;
+  private log: ToolingLog;
+
+  constructor(versions: FetchArtifactsParams, log: ToolingLog) {
+    super();
+    this.versions = versions;
+    this.log = log;
+    this.artifacts = {};
+  }
+
+  public fetchArtifacts = async () => {
+    this.log.info('Fetching artifacts');
+    await Promise.all(
+      Object.keys(this.versions).map(async (name: string) => {
+        const artifactName = name as ArtifactName;
+        const version = this.versions[artifactName];
+        if (!version) {
+          this.log.warning(`No version is specified for ${artifactName}, skipping`);
+          return;
+        }
+        const fetcher = fetchers[artifactName];
+        if (!fetcher) {
+          this.log.warning(`No fetcher is defined for ${artifactName}, skipping`);
+        }
+
+        this.artifacts[artifactName] = await fetcher(this.log, version);
+      })
+    );
+  };
+
+  public getArtifactDirectory(artifactName: string) {
+    const file = this.artifacts[artifactName as ArtifactName];
+    // this will break if the tarball name diverges from the directory that gets untarred
+    if (!file) {
+      throw new Error(`Unknown artifact ${artifactName}, unable to retreive directory`);
+    }
+    return file.replace('.tar.gz', '');
+  }
+
+  protected _cleanup() {
+    this.log.info('Cleaning up artifacts');
+    if (this.artifacts) {
+      for (const artifactName of Object.keys(this.artifacts)) {
+        const file = this.artifacts[artifactName as ArtifactName];
+        if (!file) {
+          this.log.warning(`Unknown artifact ${artifactName} encountered during cleanup, skipping`);
+          continue;
+        }
+        unlinkSync(file);
+        rmdirSync(this.getArtifactDirectory(artifactName), { recursive: true });
+      }
+    }
+  }
+}

x-pack/test/osquery_cypress/fleet_server.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ChildProcess, spawn } from 'child_process';
+import { copyFile } from 'fs/promises';
+import { unlinkSync } from 'fs';
+import { resolve } from 'path';
+import { ToolingLog } from '@kbn/dev-utils';
+import { Manager } from './resource_manager';
+export interface ElasticsearchConfig {
+  esHost: string;
+  user: string;
+  password: string;
+}
+
+export class FleetManager extends Manager {
+  private directoryPath: string;
+  private fleetProcess?: ChildProcess;
+  private esConfig: ElasticsearchConfig;
+  private log: ToolingLog;
+  constructor(directoryPath: string, esConfig: ElasticsearchConfig, log: ToolingLog) {
+    super();
+    // TODO: check if the file exists
+    this.esConfig = esConfig;
+    this.directoryPath = directoryPath;
+    this.log = log;
+  }
+  public async setup(): Promise<void> {
+    this.log.info('Setting fleet up');
+    await copyFile(resolve(__dirname, 'fleet_server.yml'), resolve('.', 'fleet-server.yml'));
+    return new Promise((res, rej) => {
+      const env = {
+        ELASTICSEARCH_HOSTS: this.esConfig.esHost,
+        ELASTICSEARCH_USERNAME: this.esConfig.user,
+        ELASTICSEARCH_PASSWORD: this.esConfig.password,
+      };
+      const file = resolve(this.directoryPath, 'fleet-server');
+      // TODO: handle logging properly
+      this.fleetProcess = spawn(file, [], { stdio: 'inherit', env });
+      this.fleetProcess.on('error', rej);
+      // TODO: actually wait for the fleet server to start listening
+      setTimeout(res, 15000);
+    });
+  }
+
+  protected _cleanup() {
+    this.log.info('Removing old fleet config');
+    if (this.fleetProcess) {
+      this.log.info('Closing fleet process');
+      if (!this.fleetProcess.kill(9)) {
+        this.log.warning('Unable to kill fleet server process');
+      }
+
+      this.fleetProcess.on('close', () => {
+        this.log.info('Fleet server process closed');
+      });
+      delete this.fleetProcess;
+    }
+    unlinkSync(resolve('.', 'fleet-server.yml'));
+  }
+}

x-pack/test/osquery_cypress/fleet_server.yml

@@ -0,0 +1,17 @@
+# mostly a stub config
+output:
+  elasticsearch:
+    hosts: '${ELASTICSEARCH_HOSTS:localhost:9220}'
+    username: '${ELASTICSEARCH_USERNAME:elastic}'
+    password: '${ELASTICSEARCH_PASSWORD:changeme}'
+
+fleet:
+  agent:
+    id: 1e4954ce-af37-4731-9f4a-407b08e69e42
+    logging:
+      level: '${LOG_LEVEL:DEBUG}'
+
+logging:
+  to_stderr: true
+
+http.enabled: true

x-pack/test/osquery_cypress/resource_manager.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const CLEANUP_EVENTS = ['SIGINT', 'exit', 'uncaughtException', 'unhandledRejection'];
+export class Manager {
+  private cleaned = false;
+  constructor() {
+    const cleanup = () => this.cleanup();
+    CLEANUP_EVENTS.forEach((ev) => process.on(ev, cleanup));
+  }
+  // This must be a synchronous method because it is used in the unhandledException and exit event handlers
+  public cleanup() {
+    // Since this can be called multiple places we proxy it with some protection
+    if (this._cleanup && !this.cleaned) {
+      this.cleaned = true;
+      this._cleanup();
+    }
+  }
+  protected _cleanup?(): void;
+}

x-pack/test/osquery_cypress/runner.ts

@@ -12,70 +12,159 @@ import { withProcRunner } from '@kbn/dev-utils';
 
 import { FtrProviderContext } from './ftr_provider_context';
 
-export async function OsqueryCypressCliTestRunner({ getService }: FtrProviderContext) {
+import { ArtifactManager, FetchArtifactsParams } from './artifact_manager';
+import { setupUsers } from './users';
+import { AgentManager } from './agent';
+import { FleetManager } from './fleet_server';
+
+interface SetupParams {
+  artifacts: FetchArtifactsParams;
+}
+
+async function withFleetAgent(
+  { getService }: FtrProviderContext,
+  params: SetupParams,
+  runner: (runnerEnv: Record<string, string>) => Promise<void>
+) {
   const log = getService('log');
   const config = getService('config');
 
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:run'],
-      cwd: resolve(__dirname, '../../plugins/osquery'),
-      env: {
-        FORCE_COLOR: '1',
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_baseUrl: Url.format(config.get('servers.kibana')),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_protocol: config.get('servers.kibana.protocol'),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_hostname: config.get('servers.kibana.hostname'),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_configport: config.get('servers.kibana.port'),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        CYPRESS_KIBANA_URL: Url.format({
-          protocol: config.get('servers.kibana.protocol'),
-          hostname: config.get('servers.kibana.hostname'),
-          port: config.get('servers.kibana.port'),
-        }),
-        ...process.env,
-      },
-      wait: true,
-    });
+  const artifactManager = new ArtifactManager(params.artifacts, log);
+  await artifactManager.fetchArtifacts();
+
+  const esHost = Url.format(config.get('servers.elasticsearch'));
+  const esConfig = {
+    user: config.get('servers.elasticsearch.username'),
+    password: config.get('servers.elasticsearch.password'),
+    esHost,
+  };
+  const fleetManager = new FleetManager(
+    artifactManager.getArtifactDirectory('fleet-server'),
+    esConfig,
+    log
+  );
+
+  const agentManager = new AgentManager(
+    artifactManager.getArtifactDirectory('elastic-agent'),
+    {
+      ...esConfig,
+      kibanaUrl: Url.format({
+        protocol: config.get('servers.kibana.protocol'),
+        hostname: config.get('servers.kibana.hostname'),
+        port: config.get('servers.kibana.port'),
+      }),
+    },
+    log
+  );
+
+  // Since the managers will create uncaughtException event handlers we need to exit manually
+  process.on('uncaughtException', (err) => {
+    // eslint-disable-next-line no-console
+    console.error('Encountered error; exiting after cleanup.', err);
+    process.exit(1);
   });
+
+  await fleetManager.setup();
+  const { policyId } = await agentManager.setup();
+  await setupUsers(esConfig);
+  try {
+    await runner({
+      CYPRESS_OSQUERY_POLICY: policyId,
+    });
+  } finally {
+    fleetManager.cleanup();
+    agentManager.cleanup();
+    artifactManager.cleanup();
+  }
 }
 
-export async function OsqueryCypressVisualTestRunner({ getService }: FtrProviderContext) {
-  const log = getService('log');
-  const config = getService('config');
-
-  await withProcRunner(log, async (procs) => {
-    await procs.run('cypress', {
-      cmd: 'yarn',
-      args: ['cypress:open'],
-      cwd: resolve(__dirname, '../../plugins/osquery'),
-      env: {
-        FORCE_COLOR: '1',
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_baseUrl: Url.format(config.get('servers.kibana')),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_protocol: config.get('servers.kibana.protocol'),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_hostname: config.get('servers.kibana.hostname'),
-        // eslint-disable-next-line @typescript-eslint/naming-convention
-        CYPRESS_configport: config.get('servers.kibana.port'),
-        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
-        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
-        CYPRESS_KIBANA_URL: Url.format({
-          protocol: config.get('servers.kibana.protocol'),
-          hostname: config.get('servers.kibana.hostname'),
-          port: config.get('servers.kibana.port'),
-        }),
-        ...process.env,
+export async function OsqueryCypressCliTestRunner(context: FtrProviderContext) {
+  const log = context.getService('log');
+  const config = context.getService('config');
+  await withFleetAgent(
+    context,
+    {
+      artifacts: {
+        'elastic-agent': '7.15.0-SNAPSHOT',
+        'fleet-server': '7.15.0-SNAPSHOT',
       },
-      wait: true,
-    });
-  });
+    },
+    (runnerEnv) =>
+      withProcRunner(log, async (procs) => {
+        await procs.run('cypress', {
+          cmd: 'yarn',
+          args: ['cypress:run'],
+          cwd: resolve(__dirname, '../../plugins/osquery'),
+          env: {
+            FORCE_COLOR: '1',
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            CYPRESS_baseUrl: Url.format(config.get('servers.kibana')),
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            CYPRESS_protocol: config.get('servers.kibana.protocol'),
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            CYPRESS_hostname: config.get('servers.kibana.hostname'),
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            CYPRESS_configport: config.get('servers.kibana.port'),
+            CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
+            CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
+            CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
+            CYPRESS_KIBANA_URL: Url.format({
+              protocol: config.get('servers.kibana.protocol'),
+              hostname: config.get('servers.kibana.hostname'),
+              port: config.get('servers.kibana.port'),
+            }),
+            ...runnerEnv,
+            ...process.env,
+          },
+          wait: true,
+        });
+      })
+  );
+}
+
+export async function OsqueryCypressVisualTestRunner(context: FtrProviderContext) {
+  const log = context.getService('log');
+  const config = context.getService('config');
+
+  await withFleetAgent(
+    context,
+    {
+      artifacts: {
+        'elastic-agent': '7.15.0-SNAPSHOT',
+        'fleet-server': '7.15.0-SNAPSHOT',
+      },
+    },
+    (runnerEnv) =>
+      withProcRunner(
+        log,
+        async (procs) =>
+          await procs.run('cypress', {
+            cmd: 'yarn',
+            args: ['cypress:open'],
+            cwd: resolve(__dirname, '../../plugins/osquery'),
+            env: {
+              FORCE_COLOR: '1',
+              // eslint-disable-next-line @typescript-eslint/naming-convention
+              CYPRESS_baseUrl: Url.format(config.get('servers.kibana')),
+              // eslint-disable-next-line @typescript-eslint/naming-convention
+              CYPRESS_protocol: config.get('servers.kibana.protocol'),
+              // eslint-disable-next-line @typescript-eslint/naming-convention
+              CYPRESS_hostname: config.get('servers.kibana.hostname'),
+              // eslint-disable-next-line @typescript-eslint/naming-convention
+              CYPRESS_configport: config.get('servers.kibana.port'),
+              CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
+              CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
+              CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
+              CYPRESS_KIBANA_URL: Url.format({
+                protocol: config.get('servers.kibana.protocol'),
+                hostname: config.get('servers.kibana.hostname'),
+                port: config.get('servers.kibana.port'),
+              }),
+              ...runnerEnv,
+              ...process.env,
+            },
+            wait: true,
+          })
+      )
+  );
 }

x-pack/test/osquery_cypress/users.ts

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import axios from 'axios';
+import { ElasticsearchConfig } from './fleet_server';
+
+interface Roles {
+  [roleName: string]: {
+    indices: [
+      {
+        names: string[];
+        privileges: string[];
+        allow_restricted_indices: boolean;
+      }
+    ];
+    applications: [
+      {
+        application: string;
+        privileges: string[];
+        resources: string[];
+      }
+    ];
+    transient_metadata: {
+      enabled: boolean;
+    };
+  };
+}
+
+interface Users {
+  [username: string]: { roles: string[] };
+}
+
+export const ROLES: Roles = {
+  read: {
+    indices: [
+      {
+        names: ['logs-*'],
+        privileges: ['read'],
+        allow_restricted_indices: false,
+      },
+    ],
+    applications: [
+      {
+        application: 'kibana-.kibana',
+        privileges: ['feature_osquery.read'],
+        resources: ['*'],
+      },
+    ],
+    transient_metadata: {
+      enabled: true,
+    },
+  },
+  all: {
+    indices: [
+      {
+        names: ['logs-*'],
+        privileges: ['read'],
+        allow_restricted_indices: false,
+      },
+    ],
+    applications: [
+      {
+        application: 'kibana-.kibana',
+        privileges: ['feature_osquery.all'],
+        resources: ['*'],
+      },
+    ],
+    transient_metadata: {
+      enabled: true,
+    },
+  },
+};
+
+export const USERS: Users = {
+  osqueryRead: {
+    roles: ['osquery_read'],
+  },
+  osqueryAll: {
+    roles: ['osquery_all'],
+  },
+};
+
+export const setupUsers = async (config: ElasticsearchConfig) => {
+  const { esHost, user: username, password } = config;
+  const params = {
+    auth: { username, password },
+  };
+  await Promise.all(
+    Object.keys(ROLES).map((role) =>
+      axios.put(`${esHost}/_security/role/osquery_${role}`, ROLES[role], params)
+    )
+  );
+  await Promise.all(
+    Object.keys(USERS).map((newUsername) =>
+      axios.put(`${esHost}/_security/user/${newUsername}`, { password, ...USERS[username] }, params)
+    )
+  );
+};