mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[Fleet] fix check deleted files task (#149689)
## Summary Fixed a bug where the hourly task was incorrectly setting all uploaded files to DELETED state. This is because the file doc `_index` property was expected to be in the form of e.g. `.fleet-file-data-agent`, but actually it contains a backing index name e.g. `.fleet-files-agent-000001` How I tested: - Enroll an agent and request a few diagnostics (requires latest changes from agent pr) - Wait for files to be uploaded, the files should be visible here in READY state: `GET .fleet-files-agent/_search` - Wait some time (up to 1h) for the task to run - Expect that the files are still in READY state after the task has run. - Previously all diagnostics files were set to DELETED in 1 hour <img width="1394" alt="image" src="https://user-images.githubusercontent.com/90178898/215104906-94e0364c-1e54-4287-87f7-7c560bf0e41e.png"> Example hit from metadata index: ``` { "_index": ".fleet-files-agent-000001", "_id": "2023-01-27T10-25-16Z-00.0c99a40e-5eec-42a4-bbeb-8c92e61f9e42", "_score": null, "_source": { "transithash": { "sha256": "d8187866600a3783490f93ffb0920b35180095c1b5392c3c561da516b7b17c82" }, "upload_id": "7695aeac-5837-44a3-8597-8d64aeed3aa2", "file": { "Status": "READY", "ext": "zip", "ChunkSize": 4194304, "size": 445046, "mime_type": "application/zip", "name": "elastic-agent-diagnostics-2023-01-27T10-25-16Z-00.zip", "hash": { "sha256": "", "md5": "" } }, "agent_id": "0c99a40e-5eec-42a4-bbeb-8c92e61f9e42", "action_id": "2023-01-27T10-25-16Z-00", "contents": null, "src": "agent", "upload_start": 1674815116965 }, "sort": [ 1674815116965 ] }, ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios
This commit is contained in:
Julia Bardi
GitHub
parent
a3cbb4e75b
commit
2e740c50e2
4 changed files with 34 additions and 20 deletions
|
|
@ -52,12 +52,17 @@ export const getFileWriteIndexName = (aliasName: string) => aliasName + '-000001
|
|||
* // return 'agent'
|
||||
*/
|
||||
export const getIntegrationNameFromFileDataIndexName = (indexName: string): string => {
|
||||
const integrationNameIndexPosition = FILE_STORAGE_DATA_INDEX_PATTERN.split('-').indexOf('*');
|
||||
return getIntegrationNameFromIndexName(indexName, FILE_STORAGE_DATA_INDEX_PATTERN);
|
||||
};
|
||||
|
||||
export const getIntegrationNameFromIndexName = (
|
||||
indexName: string,
|
||||
indexPattern: string
|
||||
): string => {
|
||||
const integrationNameIndexPosition = indexPattern.split('-').indexOf('*');
|
||||
|
||||
if (integrationNameIndexPosition === -1) {
|
||||
throw new Error(
|
||||
`Unable to parse index name. No '*' in index pattern: ${FILE_STORAGE_DATA_INDEX_PATTERN}`
|
||||
);
|
||||
throw new Error(`Unable to parse index name. No '*' in index pattern: ${indexPattern}`);
|
||||
}
|
||||
|
||||
const indexPieces = indexName.split('-');
|
||||
|
|
@ -66,7 +71,7 @@ export const getIntegrationNameFromFileDataIndexName = (indexName: string): stri
|
|||
return indexPieces[integrationNameIndexPosition];
|
||||
}
|
||||
|
||||
throw new Error(`Index name ${indexName} does not seem to be a File Data storage index`);
|
||||
throw new Error(`Index name ${indexName} does not seem to be a File storage index`);
|
||||
};
|
||||
|
||||
export const getFileStorageWriteIndexBody = (aliasName: string) => ({
|
||||
|
|
|
|||
|
|
@ -20,7 +20,10 @@ import { ES_SEARCH_LIMIT } from '../../../common/constants';
|
|||
import { fileIdsWithoutChunksByIndex, getFilesByStatus, updateFilesStatus } from '.';
|
||||
|
||||
const ENDPOINT_FILE_METADATA_INDEX = getFileMetadataIndexName('endpoint');
|
||||
const ENDPOINT_FILE_METADATA_BACKING_INDEX = `${ENDPOINT_FILE_METADATA_INDEX}-000001`;
|
||||
|
||||
const ENDPOINT_FILE_INDEX = getFileDataIndexName('endpoint');
|
||||
const ENDPOINT_FILE_BACKING_INDEX = `${ENDPOINT_FILE_INDEX}-000001`;
|
||||
|
||||
describe('files service', () => {
|
||||
let esClientMock: ElasticsearchClientMock;
|
||||
|
|
@ -49,11 +52,11 @@ describe('files service', () => {
|
|||
hits: {
|
||||
hits: [
|
||||
{
|
||||
_index: ENDPOINT_FILE_METADATA_INDEX,
|
||||
_index: ENDPOINT_FILE_METADATA_BACKING_INDEX,
|
||||
_id: 'someid1',
|
||||
},
|
||||
{
|
||||
_index: ENDPOINT_FILE_METADATA_INDEX,
|
||||
_index: ENDPOINT_FILE_METADATA_BACKING_INDEX,
|
||||
_id: 'someid2',
|
||||
},
|
||||
],
|
||||
|
|
@ -79,8 +82,8 @@ describe('files service', () => {
|
|||
{ signal: abortController.signal }
|
||||
);
|
||||
expect(result).toEqual([
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'someid1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'someid2' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'someid1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'someid2' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -91,22 +94,22 @@ describe('files service', () => {
|
|||
took: 5,
|
||||
timed_out: false,
|
||||
_shards: {
|
||||
total: 1,
|
||||
successful: 1,
|
||||
total: 2,
|
||||
successful: 2,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
},
|
||||
hits: {
|
||||
hits: [
|
||||
{
|
||||
_index: ENDPOINT_FILE_INDEX,
|
||||
_index: ENDPOINT_FILE_BACKING_INDEX,
|
||||
_id: 'keep1',
|
||||
_source: {
|
||||
bid: 'keep1',
|
||||
},
|
||||
},
|
||||
{
|
||||
_index: ENDPOINT_FILE_INDEX,
|
||||
_index: ENDPOINT_FILE_BACKING_INDEX,
|
||||
_id: 'keep2',
|
||||
_source: {
|
||||
bid: 'keep2',
|
||||
|
|
@ -117,10 +120,10 @@ describe('files service', () => {
|
|||
});
|
||||
|
||||
const files = [
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'keep1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'keep2' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'delete1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'delete2' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'keep1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'keep2' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'delete1' },
|
||||
{ _index: ENDPOINT_FILE_METADATA_BACKING_INDEX, _id: 'delete2' },
|
||||
];
|
||||
const { fileIdsByIndex: deletedFileIdsByIndex, allFileIds: allDeletedFileIds } =
|
||||
await fileIdsWithoutChunksByIndex(esClientMock, abortController, files);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import {
|
|||
import {
|
||||
getFileMetadataIndexName,
|
||||
getIntegrationNameFromFileDataIndexName,
|
||||
getIntegrationNameFromIndexName,
|
||||
} from '../../../common/services';
|
||||
|
||||
import { ES_SEARCH_LIMIT } from '../../../common/constants';
|
||||
|
|
@ -73,8 +74,13 @@ export async function fileIdsWithoutChunksByIndex(
|
|||
const noChunkFileIdsByIndex = files.reduce((acc, file) => {
|
||||
allFileIds.add(file._id);
|
||||
|
||||
const fileIds = acc[file._index];
|
||||
acc[file._index] = fileIds ? fileIds.add(file._id) : new Set([file._id]);
|
||||
const integration = getIntegrationNameFromIndexName(
|
||||
file._index,
|
||||
FILE_STORAGE_METADATA_INDEX_PATTERN
|
||||
);
|
||||
const metadataIndex = getFileMetadataIndexName(integration);
|
||||
const fileIds = acc[metadataIndex];
|
||||
acc[metadataIndex] = fileIds ? fileIds.add(file._id) : new Set([file._id]);
|
||||
return acc;
|
||||
}, {} as FileIdsByIndex);
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
} from '../services/files';
|
||||
|
||||
export const TYPE = 'fleet:check-deleted-files-task';
|
||||
export const VERSION = '1.0.0';
|
||||
export const VERSION = '1.0.1';
|
||||
const TITLE = 'Fleet Deleted Files Periodic Tasks';
|
||||
const TIMEOUT = '2m';
|
||||
const SCOPE = ['fleet'];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue