mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
# Backport This will backport the following commits from `main` to `8.16`: - [[EDR Workflows] Improve agent downloader (#196135)](https://github.com/elastic/kibana/pull/196135) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Tomasz Ciecierski","email":"tomasz.ciecierski@elastic.co"},"sourceCommit":{"committedDate":"2024-10-22T08:51:55Z","message":"[EDR Workflows] Improve agent downloader (#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Defend Workflows","v8.16.0","backport:version","v8.17.0"],"title":"[EDR Workflows] Improve agent downloader","number":196135,"url":"https://github.com/elastic/kibana/pull/196135","mergeCommit":{"message":"[EDR Workflows] Improve agent downloader (#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816"}},"sourceBranch":"main","suggestedTargetBranches":["8.16","8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/196135","number":196135,"mergeCommit":{"message":"[EDR Workflows] Improve agent downloader (#196135)","sha":"c5067fdd06425541d6eb5a9ef5260c9ea9a86816"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Tomasz Ciecierski <tomasz.ciecierski@elastic.co>
This commit is contained in:
parent
5d1530f5aa
commit
27e881576e
4 changed files with 409 additions and 57 deletions
|
@ -0,0 +1,203 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services';
|
||||
import { downloadAndStoreAgent } from '../common/agent_downloads_service';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import { agentDownloaderRunner } from './agent_downloader';
|
||||
import type { RunContext } from '@kbn/dev-cli-runner';
|
||||
|
||||
jest.mock('../common/fleet_services');
|
||||
jest.mock('../common/agent_downloads_service');
|
||||
|
||||
describe('agentDownloaderRunner', () => {
|
||||
let log: ToolingLog;
|
||||
|
||||
beforeEach(() => {
|
||||
log = {
|
||||
info: jest.fn(),
|
||||
error: jest.fn(),
|
||||
} as unknown as ToolingLog;
|
||||
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
const version = '8.15.0';
|
||||
let closestMatch = false;
|
||||
const url = 'http://example.com/agent.tar.gz';
|
||||
const fileName = 'elastic-agent-8.15.0.tar.gz';
|
||||
|
||||
it('downloads and stores the specified version', async () => {
|
||||
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
|
||||
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version, closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
|
||||
expect(getAgentFileName).toHaveBeenCalledWith(version);
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
|
||||
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
|
||||
});
|
||||
|
||||
it('logs an error if the download fails', async () => {
|
||||
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
|
||||
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock).mockRejectedValue(new Error('Download failed'));
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version, closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
|
||||
expect(getAgentFileName).toHaveBeenCalledWith(version);
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'Failed to download or store version 8.15.0: Download failed'
|
||||
);
|
||||
});
|
||||
|
||||
it('downloads and stores the previous patch version if the specified version fails', async () => {
|
||||
const fallbackVersion = '8.15.0';
|
||||
const fallbackFileName = 'elastic-agent-8.15.0.tar.gz';
|
||||
|
||||
(getAgentDownloadUrl as jest.Mock)
|
||||
.mockResolvedValueOnce({ url })
|
||||
.mockResolvedValueOnce({ url });
|
||||
(getAgentFileName as jest.Mock)
|
||||
.mockReturnValueOnce('elastic-agent-8.15.1')
|
||||
.mockReturnValueOnce('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock)
|
||||
.mockRejectedValueOnce(new Error('Download failed'))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version: '8.15.1', closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log);
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith(fallbackVersion, closestMatch, log);
|
||||
expect(getAgentFileName).toHaveBeenCalledWith('8.15.1');
|
||||
expect(getAgentFileName).toHaveBeenCalledWith(fallbackVersion);
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz');
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fallbackFileName);
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'Failed to download or store version 8.15.1: Download failed'
|
||||
);
|
||||
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
|
||||
});
|
||||
|
||||
it('logs an error if all downloads fail', async () => {
|
||||
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
|
||||
(getAgentFileName as jest.Mock)
|
||||
.mockReturnValueOnce('elastic-agent-8.15.1')
|
||||
.mockReturnValueOnce('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock)
|
||||
.mockRejectedValueOnce(new Error('Download failed'))
|
||||
.mockRejectedValueOnce(new Error('Download failed'));
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version: '8.15.1', closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.1', closestMatch, log);
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith('8.15.0', closestMatch, log);
|
||||
expect(getAgentFileName).toHaveBeenCalledWith('8.15.1');
|
||||
expect(getAgentFileName).toHaveBeenCalledWith('8.15.0');
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.1.tar.gz');
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, 'elastic-agent-8.15.0.tar.gz');
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'Failed to download or store version 8.15.1: Download failed'
|
||||
);
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'Failed to download or store version 8.15.0: Download failed'
|
||||
);
|
||||
});
|
||||
|
||||
it('does not attempt fallback when patch version is 0', async () => {
|
||||
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
|
||||
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version: '8.15.0', closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledTimes(1); // Only one call for 8.15.0
|
||||
expect(getAgentFileName).toHaveBeenCalledTimes(1);
|
||||
expect(downloadAndStoreAgent).toHaveBeenCalledWith(url, fileName);
|
||||
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
|
||||
});
|
||||
|
||||
it('logs an error for an invalid version format', async () => {
|
||||
const invalidVersion = '7.x.x';
|
||||
|
||||
await expect(
|
||||
agentDownloaderRunner({
|
||||
flags: { version: invalidVersion, closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext)
|
||||
).rejects.toThrow('Invalid version format');
|
||||
});
|
||||
|
||||
it('passes the closestMatch flag correctly', async () => {
|
||||
closestMatch = true;
|
||||
|
||||
(getAgentDownloadUrl as jest.Mock).mockResolvedValue({ url });
|
||||
(getAgentFileName as jest.Mock).mockReturnValue('elastic-agent-8.15.0');
|
||||
(downloadAndStoreAgent as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version, closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(getAgentDownloadUrl).toHaveBeenCalledWith(version, closestMatch, log);
|
||||
});
|
||||
|
||||
it('throws an error when version is not provided', async () => {
|
||||
await expect(
|
||||
agentDownloaderRunner({
|
||||
flags: { closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext)
|
||||
).rejects.toThrow('version argument is required');
|
||||
});
|
||||
|
||||
it('logs the correct messages when both version and fallback version are processed', async () => {
|
||||
const primaryVersion = '8.15.1';
|
||||
|
||||
(getAgentDownloadUrl as jest.Mock)
|
||||
.mockResolvedValueOnce({ url })
|
||||
.mockResolvedValueOnce({ url });
|
||||
|
||||
(getAgentFileName as jest.Mock)
|
||||
.mockReturnValueOnce('elastic-agent-8.15.1')
|
||||
.mockReturnValueOnce('elastic-agent-8.15.0');
|
||||
|
||||
(downloadAndStoreAgent as jest.Mock)
|
||||
.mockRejectedValueOnce(new Error('Download failed')) // Fail on primary
|
||||
.mockResolvedValueOnce(undefined); // Success on fallback
|
||||
|
||||
await agentDownloaderRunner({
|
||||
flags: { version: primaryVersion, closestMatch },
|
||||
log,
|
||||
} as unknown as RunContext);
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
'Failed to download or store version 8.15.1: Download failed'
|
||||
);
|
||||
expect(log.info).toHaveBeenCalledWith('Successfully downloaded and stored version 8.15.0');
|
||||
});
|
||||
});
|
|
@ -8,24 +8,72 @@
|
|||
import { ok } from 'assert';
|
||||
import type { RunFn } from '@kbn/dev-cli-runner';
|
||||
import type { ToolingLog } from '@kbn/tooling-log';
|
||||
import semver from 'semver';
|
||||
import { getAgentDownloadUrl, getAgentFileName } from '../common/fleet_services';
|
||||
import { downloadAndStoreAgent } from '../common/agent_downloads_service';
|
||||
|
||||
// Decrement the patch version by 1 and preserve pre-release tag (if any)
|
||||
const decrementPatchVersion = (version: string): string | null => {
|
||||
const parsedVersion = semver.parse(version);
|
||||
if (!parsedVersion) {
|
||||
return null;
|
||||
}
|
||||
const newPatchVersion = parsedVersion.patch - 1;
|
||||
// Create a new version string with the decremented patch - removing any possible pre-release tag
|
||||
const newVersion = `${parsedVersion.major}.${parsedVersion.minor}.${newPatchVersion}`;
|
||||
return semver.valid(newVersion) ? newVersion : null;
|
||||
};
|
||||
|
||||
// Generate a list of versions to attempt downloading, including a fallback to the previous patch (GA)
|
||||
const getVersionsToDownload = (version: string): string[] => {
|
||||
const parsedVersion = semver.parse(version);
|
||||
if (!parsedVersion) return [];
|
||||
// If patch version is 0, return only the current version.
|
||||
if (parsedVersion.patch === 0) {
|
||||
return [version];
|
||||
}
|
||||
|
||||
const decrementedVersion = decrementPatchVersion(version);
|
||||
return decrementedVersion ? [version, decrementedVersion] : [version];
|
||||
};
|
||||
|
||||
// Download and store the Elastic Agent for the specified version(s)
|
||||
const downloadAndStoreElasticAgent = async (
|
||||
version: string,
|
||||
closestMatch: boolean,
|
||||
log: ToolingLog
|
||||
) => {
|
||||
const downloadUrlResponse = await getAgentDownloadUrl(version, closestMatch, log);
|
||||
const fileNameNoExtension = getAgentFileName(version);
|
||||
const agentFile = `${fileNameNoExtension}.tar.gz`;
|
||||
await downloadAndStoreAgent(downloadUrlResponse.url, agentFile);
|
||||
): Promise<void> => {
|
||||
const versionsToDownload = getVersionsToDownload(version);
|
||||
|
||||
// Although we have a list of versions to try downloading, we only need to download one, and will return as soon as it succeeds.
|
||||
for (const versionToDownload of versionsToDownload) {
|
||||
try {
|
||||
const { url } = await getAgentDownloadUrl(versionToDownload, closestMatch, log);
|
||||
const fileName = `${getAgentFileName(versionToDownload)}.tar.gz`;
|
||||
|
||||
await downloadAndStoreAgent(url, fileName);
|
||||
log.info(`Successfully downloaded and stored version ${versionToDownload}`);
|
||||
return; // Exit once successful
|
||||
} catch (error) {
|
||||
log.error(`Failed to download or store version ${versionToDownload}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
log.error(`Failed to download agent for any available version: ${versionsToDownload.join(', ')}`);
|
||||
};
|
||||
|
||||
export const agentDownloaderRunner: RunFn = async (cliContext) => {
|
||||
ok(cliContext.flags.version, 'version argument is required');
|
||||
const { version } = cliContext.flags;
|
||||
|
||||
ok(version, 'version argument is required');
|
||||
|
||||
// Validate version format
|
||||
if (!semver.valid(version as string)) {
|
||||
throw new Error('Invalid version format');
|
||||
}
|
||||
|
||||
await downloadAndStoreElasticAgent(
|
||||
cliContext.flags.version as string,
|
||||
version as string,
|
||||
cliContext.flags.closestMatch as boolean,
|
||||
cliContext.log
|
||||
);
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
// Adjust path if needed
|
||||
|
||||
import { downloadAndStoreAgent, isAgentDownloadFromDiskAvailable } from './agent_downloads_service';
|
||||
import fs from 'fs';
|
||||
import nodeFetch from 'node-fetch';
|
||||
import { finished } from 'stream/promises';
|
||||
|
||||
jest.mock('fs');
|
||||
jest.mock('node-fetch');
|
||||
jest.mock('stream/promises', () => ({
|
||||
finished: jest.fn(),
|
||||
}));
|
||||
jest.mock('../../../common/endpoint/data_loaders/utils', () => ({
|
||||
createToolingLogger: jest.fn(() => ({
|
||||
debug: jest.fn(),
|
||||
info: jest.fn(),
|
||||
error: jest.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
describe('AgentDownloadStorage', () => {
|
||||
const url = 'http://example.com/agent.tar.gz';
|
||||
const fileName = 'elastic-agent-7.10.0.tar.gz';
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks(); // Ensure no previous test state affects the current one
|
||||
});
|
||||
|
||||
it('downloads and stores the agent if not cached', async () => {
|
||||
(fs.existsSync as unknown as jest.Mock).mockReturnValue(false);
|
||||
(fs.createWriteStream as unknown as jest.Mock).mockReturnValue({
|
||||
on: jest.fn(),
|
||||
end: jest.fn(),
|
||||
});
|
||||
(nodeFetch as unknown as jest.Mock).mockResolvedValue({ body: { pipe: jest.fn() } });
|
||||
(finished as unknown as jest.Mock).mockResolvedValue(undefined);
|
||||
|
||||
const result = await downloadAndStoreAgent(url, fileName);
|
||||
|
||||
expect(result).toEqual({
|
||||
url,
|
||||
filename: fileName,
|
||||
directory: expect.any(String),
|
||||
fullFilePath: expect.stringContaining(fileName), // Dynamically match the file path
|
||||
});
|
||||
});
|
||||
|
||||
it('reuses cached agent if available', async () => {
|
||||
(fs.existsSync as unknown as jest.Mock).mockReturnValue(true);
|
||||
|
||||
const result = await downloadAndStoreAgent(url, fileName);
|
||||
|
||||
expect(result).toEqual({
|
||||
url,
|
||||
filename: fileName,
|
||||
directory: expect.any(String),
|
||||
fullFilePath: expect.stringContaining(fileName), // Dynamically match the path
|
||||
});
|
||||
});
|
||||
|
||||
it('checks if agent download is available from disk', () => {
|
||||
(fs.existsSync as unknown as jest.Mock).mockReturnValue(true);
|
||||
|
||||
const result = isAgentDownloadFromDiskAvailable(fileName);
|
||||
|
||||
expect(result).toEqual({
|
||||
filename: fileName,
|
||||
directory: expect.any(String),
|
||||
fullFilePath: expect.stringContaining(fileName), // Dynamically match the path
|
||||
});
|
||||
});
|
||||
});
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import pRetry from 'p-retry';
|
||||
import { mkdir, readdir, stat, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import fs from 'fs';
|
||||
|
@ -24,7 +25,7 @@ export interface DownloadedAgentInfo {
|
|||
|
||||
interface AgentDownloadStorageSettings {
|
||||
/**
|
||||
* Last time a cleanup was ran. Date in ISO format
|
||||
* Last time a cleanup was performed. Date in ISO format
|
||||
*/
|
||||
lastCleanup: string;
|
||||
|
||||
|
@ -47,7 +48,7 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
|
|||
constructor() {
|
||||
super('agent_download_storage_settings.json', {
|
||||
defaultSettings: {
|
||||
maxFileAge: 1.728e8, // 2 days
|
||||
maxFileAge: 1.728e8, // 2 days in milliseconds
|
||||
lastCleanup: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
@ -55,20 +56,25 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
|
|||
this.downloadsDirFullPath = this.buildPath(this.downloadsDirName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the download directory exists on disk
|
||||
*/
|
||||
protected async ensureExists(): Promise<void> {
|
||||
await super.ensureExists();
|
||||
|
||||
if (!this.downloadsFolderExists) {
|
||||
await mkdir(this.downloadsDirFullPath, { recursive: true });
|
||||
this.log.debug(`Created directory [this.downloadsDirFullPath] for cached agent downloads`);
|
||||
this.log.debug(`Created directory [${this.downloadsDirFullPath}] for cached agent downloads`);
|
||||
this.downloadsFolderExists = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the file paths for a given download URL and optional file name.
|
||||
*/
|
||||
public getPathsForUrl(agentDownloadUrl: string, agentFileName?: string): DownloadedAgentInfo {
|
||||
const filename = agentFileName
|
||||
? agentFileName
|
||||
: agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#');
|
||||
const filename =
|
||||
agentFileName || agentDownloadUrl.replace(/^https?:\/\//gi, '').replace(/\//g, '#');
|
||||
const directory = this.downloadsDirFullPath;
|
||||
const fullFilePath = this.buildPath(join(this.downloadsDirName, filename));
|
||||
|
||||
|
@ -79,59 +85,67 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the agent and stores it locally. Reuses existing downloads if available.
|
||||
*/
|
||||
public async downloadAndStore(
|
||||
agentDownloadUrl: string,
|
||||
agentFileName?: string
|
||||
): Promise<DownloadedAgentInfo> {
|
||||
this.log.debug(`Downloading and storing: ${agentDownloadUrl}`);
|
||||
|
||||
// TODO: should we add "retry" attempts to file downloads?
|
||||
this.log.debug(`Starting download: ${agentDownloadUrl}`);
|
||||
|
||||
await this.ensureExists();
|
||||
|
||||
const newDownloadInfo = this.getPathsForUrl(agentDownloadUrl, agentFileName);
|
||||
|
||||
// If download is already present on disk, then just return that info. No need to re-download it
|
||||
// Return cached version if the file already exists
|
||||
if (fs.existsSync(newDownloadInfo.fullFilePath)) {
|
||||
this.log.debug(`Download already cached at [${newDownloadInfo.fullFilePath}]`);
|
||||
return newDownloadInfo;
|
||||
}
|
||||
|
||||
try {
|
||||
const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath);
|
||||
await pRetry(
|
||||
async (attempt) => {
|
||||
this.log.info(
|
||||
`Attempt ${attempt} - Downloading agent from [${agentDownloadUrl}] to [${newDownloadInfo.fullFilePath}]`
|
||||
);
|
||||
const outputStream = fs.createWriteStream(newDownloadInfo.fullFilePath);
|
||||
|
||||
await handleProcessInterruptions(
|
||||
async () => {
|
||||
const { body } = await nodeFetch(agentDownloadUrl);
|
||||
await finished(body.pipe(outputStream));
|
||||
await handleProcessInterruptions(
|
||||
async () => {
|
||||
const { body } = await nodeFetch(agentDownloadUrl);
|
||||
await finished(body.pipe(outputStream));
|
||||
},
|
||||
() => fs.unlinkSync(newDownloadInfo.fullFilePath) // Clean up on interruption
|
||||
);
|
||||
this.log.info(`Successfully downloaded agent to [${newDownloadInfo.fullFilePath}]`);
|
||||
},
|
||||
() => {
|
||||
fs.unlinkSync(newDownloadInfo.fullFilePath);
|
||||
{
|
||||
retries: 2, // 2 retries = 3 total attempts (1 initial + 2 retries)
|
||||
onFailedAttempt: (error) => {
|
||||
this.log.error(`Download attempt ${error.attemptNumber} failed: ${error.message}`);
|
||||
// Cleanup failed download
|
||||
return unlink(newDownloadInfo.fullFilePath);
|
||||
},
|
||||
}
|
||||
);
|
||||
} catch (e) {
|
||||
// Try to clean up download case it failed halfway through
|
||||
await unlink(newDownloadInfo.fullFilePath);
|
||||
|
||||
throw e;
|
||||
} catch (error) {
|
||||
throw new Error(`Download failed after multiple attempts: ${error.message}`);
|
||||
}
|
||||
|
||||
await this.cleanupDownloads();
|
||||
|
||||
return newDownloadInfo;
|
||||
}
|
||||
|
||||
public async cleanupDownloads(): Promise<{ deleted: string[] }> {
|
||||
this.log.debug(`Performing cleanup of cached Agent downlaods`);
|
||||
this.log.debug('Performing cleanup of cached Agent downloads');
|
||||
|
||||
const settings = await this.get();
|
||||
const maxAgeDate = new Date();
|
||||
const maxAgeDate = new Date(Date.now() - settings.maxFileAge);
|
||||
const response: { deleted: string[] } = { deleted: [] };
|
||||
|
||||
maxAgeDate.setMilliseconds(settings.maxFileAge * -1); // `* -1` to set time back
|
||||
|
||||
// If cleanup already happen within the file age, then nothing to do. Exit.
|
||||
if (settings.lastCleanup > maxAgeDate.toISOString()) {
|
||||
this.log.debug('Skipping cleanup, as it was performed recently.');
|
||||
return response;
|
||||
}
|
||||
|
||||
|
@ -140,41 +154,48 @@ class AgentDownloadStorage extends SettingsStorage<AgentDownloadStorageSettings>
|
|||
lastCleanup: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const deleteFilePromises: Array<Promise<unknown>> = [];
|
||||
const allFiles = await readdir(this.downloadsDirFullPath);
|
||||
try {
|
||||
const allFiles = await readdir(this.downloadsDirFullPath);
|
||||
const deleteFilePromises = allFiles.map(async (fileName) => {
|
||||
const filePath = join(this.downloadsDirFullPath, fileName);
|
||||
const fileStats = await stat(filePath);
|
||||
if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) {
|
||||
try {
|
||||
await unlink(filePath);
|
||||
response.deleted.push(filePath);
|
||||
} catch (err) {
|
||||
this.log.error(`Failed to delete file [${filePath}]: ${err.message}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for (const fileName of allFiles) {
|
||||
const filePath = join(this.downloadsDirFullPath, fileName);
|
||||
const fileStats = await stat(filePath);
|
||||
|
||||
if (fileStats.isFile() && fileStats.birthtime < maxAgeDate) {
|
||||
deleteFilePromises.push(unlink(filePath));
|
||||
response.deleted.push(filePath);
|
||||
}
|
||||
await Promise.allSettled(deleteFilePromises);
|
||||
this.log.debug(`Deleted ${response.deleted.length} file(s)`);
|
||||
return response;
|
||||
} catch (err) {
|
||||
this.log.error(`Error during cleanup: ${err.message}`);
|
||||
return response;
|
||||
}
|
||||
|
||||
await Promise.allSettled(deleteFilePromises);
|
||||
|
||||
this.log.debug(`Deleted [${response.deleted.length}] file(s)`);
|
||||
this.log.verbose(`files deleted:\n`, response.deleted.join('\n'));
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a specific agent download is available locally.
|
||||
*/
|
||||
public isAgentDownloadFromDiskAvailable(filename: string): DownloadedAgentInfo | undefined {
|
||||
if (fs.existsSync(join(this.downloadsDirFullPath, filename))) {
|
||||
const filePath = join(this.downloadsDirFullPath, filename);
|
||||
if (fs.existsSync(filePath)) {
|
||||
return {
|
||||
filename,
|
||||
/** The local directory where downloads are stored */
|
||||
directory: this.downloadsDirFullPath,
|
||||
/** The full local file path and name */
|
||||
fullFilePath: join(this.downloadsDirFullPath, filename),
|
||||
fullFilePath: filePath,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const agentDownloadsClient = new AgentDownloadStorage();
|
||||
export const agentDownloadsClient = new AgentDownloadStorage();
|
||||
|
||||
export interface DownloadAndStoreAgentResponse extends DownloadedAgentInfo {
|
||||
url: string;
|
||||
|
@ -203,12 +224,15 @@ export const downloadAndStoreAgent = async (
|
|||
};
|
||||
|
||||
/**
|
||||
* Cleans up the old agent downloads on disk.
|
||||
* Cleans up old agent downloads on disk.
|
||||
*/
|
||||
export const cleanupDownloads = async (): ReturnType<AgentDownloadStorage['cleanupDownloads']> => {
|
||||
return agentDownloadsClient.cleanupDownloads();
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if a specific agent download is available from disk.
|
||||
*/
|
||||
export const isAgentDownloadFromDiskAvailable = (
|
||||
fileName: string
|
||||
): DownloadedAgentInfo | undefined => {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue