[8.16] [EDR Workflows] Improve agent downloader (#196135) (#197183)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[EDR Workflows] Improve agent downloader
(#196135)](https://github.com/elastic/kibana/pull/196135)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Tomasz
Ciecierski","email":"tomasz.ciecierski@elastic.co"},"sourceCommit":{"committedDate":"2024-10-22T08:51:55Z","message":"[EDR
Workflows] Improve agent downloader
(#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Defend
Workflows","v8.16.0","backport:version","v8.17.0"],"title":"[EDR
Workflows] Improve agent
downloader","number":196135,"url":"https://github.com/elastic/kibana/pull/196135","mergeCommit":{"message":"[EDR
Workflows] Improve agent downloader
(#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816"}},"sourceBranch":"main","suggestedTargetBranches":["8.16","8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/196135","number":196135,"mergeCommit":{"message":"[EDR
Workflows] Improve agent downloader
(#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Tomasz Ciecierski <tomasz.ciecierski@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-22 21:35:58 +11:00 committed by GitHub
parent 5d1530f5aa
commit 27e881576e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 409 additions and 57 deletions

View file

@ -0,0 +1,203 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services';
import { downloadAndStoreAgent } from '../common/agent_downloads_service';
import type { ToolingLog } from '@kbn/tooling-log';
import { agentDownloaderRunner } from './agent_downloader';
import type { RunContext } from '@kbn/dev-cli-runner';
jest.mock('../common/fleet_services');
jest.mock('../common/agent_downloads_service');
describe('agentDownloaderRunner', () => {
let log: ToolingLog;
beforeEach(() => {
log = {
info: jest.fn(),
error: jest.fn(),
} as unknown as ToolingLog;
jest.clearAllMocks();
});
const version = '8.15.0';
let closestMatch = false;
const url = 'http://example.com/agent.tar.gz';
const fileName = 'elastic-agent-8.15.0.tar.gz';
it('downloads and stores the specified version', async () => {
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
await agentDownloaderRunner({
flags: { version, closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
expect(getAgentFileName).toHaveBeenCalledWith(version);
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
});
it('logs an error if the download fails', async () => {
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock).mockRejectedValue(new Error('Download failed'));
await agentDownloaderRunner({
flags: { version, closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
expect(getAgentFileName).toHaveBeenCalledWith(version);
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
expect(log.error).toHaveBeenCalledWith(
'Failed to download or store version 8.15.0: Download failed'
);
});
it('downloads and stores the previous patch version if the specified version fails', async () => {
const fallbackVersion = '8.15.0';
const fallbackFileName = 'elastic-agent-8.15.0.tar.gz';
(getAgentDownloadUrl as jest.Mock)
.mockResolvedValueOnce({ url })
.mockResolvedValueOnce({ url });
(getAgentFileName as jest.Mock)
.mockReturnValueOnce('elastic-agent-8.15.1')
.mockReturnValueOnce('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock)
.mockRejectedValueOnce(new Error('Download failed'))
.mockResolvedValueOnce(undefined);
await agentDownloaderRunner({
flags: { version: '8.15.1', closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log);
expect(getAgentDownloadUrl).toHaveBeenCalledWith(fallbackVersion, closestMatch, log);
expect(getAgentFileName).toHaveBeenCalledWith('8.15.1');
expect(getAgentFileName).toHaveBeenCalledWith(fallbackVersion);
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz');
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fallbackFileName);
expect(log.error).toHaveBeenCalledWith(
'Failed to download or store version 8.15.1: Download failed'
);
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
});
it('logs an error if all downloads fail', async () => {
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
(getAgentFileName as jest.Mock)
.mockReturnValueOnce('elastic-agent-8.15.1')
.mockReturnValueOnce('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock)
.mockRejectedValueOnce(new Error('Download failed'))
.mockRejectedValueOnce(new Error('Download failed'));
await agentDownloaderRunner({
flags: { version: '8.15.1', closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log);
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.0', closestMatch, log);
expect(getAgentFileName).toHaveBeenCalledWith('8.15.1');
expect(getAgentFileName).toHaveBeenCalledWith('8.15.0');
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz');
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.0.tar.gz');
expect(log.error).toHaveBeenCalledWith(
'Failed to download or store version 8.15.1: Download failed'
);
expect(log.error).toHaveBeenCalledWith(
'Failed to download or store version 8.15.0: Download failed'
);
});
it('does not attempt fallback when patch version is 0', async () => {
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
await agentDownloaderRunner({
flags: { version: '8.15.0', closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledTimes(1); // Only one call for 8.15.0
expect(getAgentFileName).toHaveBeenCalledTimes(1);
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
});
it('logs an error for an invalid version format', async () => {
const invalidVersion = '7.x.x';
await expect(
agentDownloaderRunner({
flags: { version: invalidVersion, closestMatch },
log,
} as unknown as RunContext)
).rejects.toThrow('Invalid version format');
});
it('passes the closestMatch flag correctly', async () => {
closestMatch = true;
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
await agentDownloaderRunner({
flags: { version, closestMatch },
log,
} as unknown as RunContext);
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
});
it('throws an error when version is not provided', async () => {
await expect(
agentDownloaderRunner({
flags: { closestMatch },
log,
} as unknown as RunContext)
).rejects.toThrow('version argument is required');
});
it('logs the correct messages when both version and fallback version are processed', async () => {
const primaryVersion = '8.15.1';
(getAgentDownloadUrl as jest.Mock)
.mockResolvedValueOnce({ url })
.mockResolvedValueOnce({ url });
(getAgentFileName as jest.Mock)
.mockReturnValueOnce('elastic-agent-8.15.1')
.mockReturnValueOnce('elastic-agent-8.15.0');
(downloadAndStoreAgent as jest.Mock)
.mockRejectedValueOnce(new Error('Download failed')) // Fail on primary
.mockResolvedValueOnce(undefined); // Success on fallback
await agentDownloaderRunner({
flags: { version: primaryVersion, closestMatch },
log,
} as unknown as RunContext);
expect(log.error).toHaveBeenCalledWith(
'Failed to download or store version 8.15.1: Download failed'
);
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
});
});

View file

@ -8,24 +8,72 @@
import { ok } from 'assert';
import type { RunFn } from '@kbn/dev-cli-runner';
import type { ToolingLog } from '@kbn/tooling-log';
import semver from 'semver';
import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services';
import { downloadAndStoreAgent } from '../common/agent_downloads_service';
// Decrement the patch version by 1 and preserve pre-release tag (if any)
const decrementPatchVersion = (version: string): string | null => {
const parsedVersion = semver.parse(version);
if (!parsedVersion) {
return null;
}
const newPatchVersion = parsedVersion.patch - 1;
// Create a new version string with the decremented patch - removing any possible pre-release tag
const newVersion = `${parsedVersion.major}.${parsedVersion.minor}.${newPatchVersion}`;
return semver.valid(newVersion) ? newVersion : null;
};
// Generate a list of versions to attempt downloading, including a fallback to the previous patch (GA)
const getVersionsToDownload = (version: string): string[] => {
const parsedVersion = semver.parse(version);
if (!parsedVersion) return [];
// If patch version is 0, return only the current version.
if (parsedVersion.patch === 0) {
return [version];
}
const decrementedVersion = decrementPatchVersion(version);
return decrementedVersion ? [version, decrementedVersion] : [version];
};
// Download and store the Elastic Agent for the specified version(s)
const downloadAndStoreElasticAgent = async (
version: string,
closestMatch: boolean,
log: ToolingLog
) => {
const downloadUrlResponse = await getAgentDownloadUrl(version, closestMatch, log);
const fileNameNoExtension = getAgentFileName(version);
const agentFile = `${fileNameNoExtension}.tar.gz`;
await downloadAndStoreAgent(downloadUrlResponse.url, agentFile);
): Promise<void> => {
const versionsToDownload = getVersionsToDownload(version);
// Although we have a list of versions to try downloading, we only need to download one, and will return as soon as it succeeds.
for (const versionToDownload of versionsToDownload) {
try {
const { url } = await getAgentDownloadUrl(versionToDownload, closestMatch, log);
const fileName = `${getAgentFileName(versionToDownload)}.tar.gz`;
await downloadAndStoreAgent(url, fileName);
log.info(`Successfully downloaded and stored version ${versionToDownload}`);
return; // Exit once successful
} catch (error) {
log.error(`Failed to download or store version ${versionToDownload}: ${error.message}`);
}
}
log.error(`Failed to download agent for any available version: ${versionsToDownload.join(', ')}`);
};
export const agentDownloaderRunner: RunFn = async (cliContext) => {
ok(cliContext.flags.version, 'version argument is required');
const { version } = cliContext.flags;
ok(version, 'version argument is required');
// Validate version format
if (!semver.valid(version as string)) {
throw new Error('Invalid version format');
}
await downloadAndStoreElasticAgent(
cliContext.flags.version as string,
version as string,
cliContext.flags.closestMatch as boolean,
cliContext.log
);

View file

@ -0,0 +1,77 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
// Adjust path if needed
import { downloadAndStoreAgent, isAgentDownloadFromDiskAvailable } from './agent_downloads_service';
import fs from 'fs';
import nodeFetch from 'node-fetch';
import { finished } from 'stream/promises';
jest.mock('fs');
jest.mock('node-fetch');
jest.mock('stream/promises', () => ({
finished: jest.fn(),
}));
jest.mock('../../../common/endpoint/data_loaders/utils', () => ({
createToolingLogger: jest.fn(() => ({
debug: jest.fn(),
info: jest.fn(),
error: jest.fn(),
})),
}));
describe('AgentDownloadStorage', () => {
const url = 'http://example.com/agent.tar.gz';
const fileName = 'elastic-agent-7.10.0.tar.gz';
beforeEach(() => {
jest.clearAllMocks(); // Ensure no previous test state affects the current one
});
it('downloads and stores the agent if not cached', async () => {
(fs.existsSync as unknown as jest.Mock).mockReturnValue(false);
(fs.createWriteStream as unknown as jest.Mock).mockReturnValue({
on: jest.fn(),
end: jest.fn(),
});
(nodeFetch as unknown as jest.Mock).mockResolvedValue({ body: { pipe: jest.fn() } });
(finished as unknown as jest.Mock).mockResolvedValue(undefined);
const result = await downloadAndStoreAgent(url, fileName);
expect(result).toEqual({
url,
filename: fileName,
directory: expect.any(String),
fullFilePath: expect.stringContaining(fileName), // Dynamically match the file path
});
});
it('reuses cached agent if available', async () => {
(fs.existsSync as unknown as jest.Mock).mockReturnValue(true);
const result = await downloadAndStoreAgent(url, fileName);
expect(result).toEqual({
url,
filename: fileName,
directory: expect.any(String),
fullFilePath: expect.stringContaining(fileName), // Dynamically match the path
});
});
it('checks if agent download is available from disk', () => {
(fs.existsSync as unknown as jest.Mock).mockReturnValue(true);
const result = isAgentDownloadFromDiskAvailable(fileName);
expect(result).toEqual({
filename: fileName,
directory: expect.any(String),
fullFilePath: expect.stringContaining(fileName), // Dynamically match the path
});
});
});

View file

@ -5,6 +5,7 @@
* 2.0.
*/
import pRetry from 'p-retry';
import { mkdir, readdir, stat, unlink } from 'fs/promises';
import { join } from 'path';
import fs from 'fs';
@ -24,7 +25,7 @@ export interface DownloadedAgentInfo {
interface AgentDownloadStorageSettings {
/**
* Last time a cleanup was ran. Date in ISO format
* Last time a cleanup was performed. Date in ISO format
*/
lastCleanup: string;
@ -47,7 +48,7 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
constructor() {
super('agent_download_storage_settings.json', {
defaultSettings: {
maxFileAge: 1.728e8, // 2 days
maxFileAge: 1.728e8, // 2 days in milliseconds
lastCleanup: new Date().toISOString(),
},
});
@ -55,20 +56,25 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
this.downloadsDirFullPath = this.buildPath(this.downloadsDirName);
}
/**
* Ensures the download directory exists on disk
*/
protected async ensureExists(): Promise<void> {
await super.ensureExists();
if (!this.downloadsFolderExists) {
await mkdir(this.downloadsDirFullPath, { recursive: true });
this.log.debug(`Created directory [this.downloadsDirFullPath] for cached agent downloads`);
this.log.debug(`Created directory [${this.downloadsDirFullPath}] for cached agent downloads`);
this.downloadsFolderExists = true;
}
}
/**
* Gets the file paths for a given download URL and optional file name.
*/
public getPathsForUrl(agentDownloadUrl: string, agentFileName?: string): DownloadedAgentInfo {
const filename = agentFileName
? agentFileName
: agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#');
const filename =
agentFileName || agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#');
const directory = this.downloadsDirFullPath;
const fullFilePath = this.buildPath(join(this.downloadsDirName, filename));
@ -79,59 +85,67 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
};
}
/**
* Downloads the agent and stores it locally. Reuses existing downloads if available.
*/
public async downloadAndStore(
agentDownloadUrl: string,
agentFileName?: string
): Promise<DownloadedAgentInfo> {
this.log.debug(`Downloading and storing: ${agentDownloadUrl}`);
// TODO: should we add "retry" attempts to file downloads?
this.log.debug(`Starting download: ${agentDownloadUrl}`);
await this.ensureExists();
const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl, agentFileName);
// If download is already present on disk, then just return that info. No need to re-download it
// Return cached version if the file already exists
if (fs.existsSync(newDownloadInfo.fullFilePath)) {
this.log.debug(`Download already cached at [${newDownloadInfo.fullFilePath}]`);
return newDownloadInfo;
}
try {
const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath);
await pRetry(
async (attempt) => {
this.log.info(
`Attempt ${attempt} - Downloading agent from [${agentDownloadUrl}] to [${newDownloadInfo.fullFilePath}]`
);
const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath);
await handleProcessInterruptions(
async () => {
const { body } = await nodeFetch(agentDownloadUrl);
await finished(body.pipe(outputStream));
await handleProcessInterruptions(
async () => {
const { body } = await nodeFetch(agentDownloadUrl);
await finished(body.pipe(outputStream));
},
() => fs.unlinkSync(newDownloadInfo.fullFilePath) // Clean up on interruption
);
this.log.info(`Successfully downloaded agent to [${newDownloadInfo.fullFilePath}]`);
},
() => {
fs.unlinkSync(newDownloadInfo.fullFilePath);
{
retries: 2, // 2 retries = 3 total attempts (1 initial + 2 retries)
onFailedAttempt: (error) => {
this.log.error(`Download attempt ${error.attemptNumber} failed: ${error.message}`);
// Cleanup failed download
return unlink(newDownloadInfo.fullFilePath);
},
}
);
} catch (e) {
// Try to clean up download case it failed halfway through
await unlink(newDownloadInfo.fullFilePath);
throw e;
} catch (error) {
throw new Error(`Download failed after multiple attempts: ${error.message}`);
}
await this.cleanupDownloads();
return newDownloadInfo;
}
public async cleanupDownloads(): Promise<{ deleted: string[] }> {
this.log.debug(`Performing cleanup of cached Agent downlaods`);
this.log.debug('Performing cleanup of cached Agent downloads');
const settings = await this.get();
const maxAgeDate = new Date();
const maxAgeDate = new Date(Date.now() - settings.maxFileAge);
const response: { deleted: string[] } = { deleted: [] };
maxAgeDate.setMilliseconds(settings.maxFileAge * -1); // `* -1` to set time back
// If cleanup already happen within the file age, then nothing to do. Exit.
if (settings.lastCleanup > maxAgeDate.toISOString()) {
this.log.debug('Skipping cleanup, as it was performed recently.');
return response;
}
@ -140,41 +154,48 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
lastCleanup: new Date().toISOString(),
});
const deleteFilePromises: Array<Promise<unknown>> = [];
const allFiles = await readdir(this.downloadsDirFullPath);
try {
const allFiles = await readdir(this.downloadsDirFullPath);
const deleteFilePromises = allFiles.map(async (fileName) => {
const filePath = join(this.downloadsDirFullPath, fileName);
const fileStats = await stat(filePath);
if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) {
try {
await unlink(filePath);
response.deleted.push(filePath);
} catch (err) {
this.log.error(`Failed to delete file [${filePath}]: ${err.message}`);
}
}
});
for (const fileName of allFiles) {
const filePath = join(this.downloadsDirFullPath, fileName);
const fileStats = await stat(filePath);
if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) {
deleteFilePromises.push(unlink(filePath));
response.deleted.push(filePath);
}
await Promise.allSettled(deleteFilePromises);
this.log.debug(`Deleted ${response.deleted.length} file(s)`);
return response;
} catch (err) {
this.log.error(`Error during cleanup: ${err.message}`);
return response;
}
await Promise.allSettled(deleteFilePromises);
this.log.debug(`Deleted [${response.deleted.length}] file(s)`);
this.log.verbose(`files deleted:\n`, response.deleted.join('\n'));
return response;
}
/**
* Checks if a specific agent download is available locally.
*/
public isAgentDownloadFromDiskAvailable(filename: string): DownloadedAgentInfo | undefined {
if (fs.existsSync(join(this.downloadsDirFullPath, filename))) {
const filePath = join(this.downloadsDirFullPath, filename);
if (fs.existsSync(filePath)) {
return {
filename,
/** The local directory where downloads are stored */
directory: this.downloadsDirFullPath,
/** The full local file path and name */
fullFilePath: join(this.downloadsDirFullPath, filename),
fullFilePath: filePath,
};
}
}
}
const agentDownloadsClient = new AgentDownloadStorage();
export const agentDownloadsClient = new AgentDownloadStorage();
export interface DownloadAndStoreAgentResponse extends DownloadedAgentInfo {
url: string;
@ -203,12 +224,15 @@ export const downloadAndStoreAgent = async (
};
/**
* Cleans up the old agent downloads on disk.
* Cleans up old agent downloads on disk.
*/
export const cleanupDownloads = async (): ReturnType<AgentDownloadStorage['cleanupDownloads']> => {
return agentDownloadsClient.cleanupDownloads();
};
/**
* Checks if a specific agent download is available from disk.
*/
export const isAgentDownloadFromDiskAvailable = (
fileName: string
): DownloadedAgentInfo | undefined => {