github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] clean up for deleted file task (#144624)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Joey F. Poon 2022-11-07 07:51:52 -06:00 • committed by GitHub
parent 7793d4dd46
commit 5de249ffa8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 24 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/fleet/kibana.json
View file

@ -8,7 +8,7 @@
"server": true,
"ui": true,
"configPath": ["xpack", "fleet"],
"requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share", "spaces", "security", "unifiedSearch", "savedObjectsTagging", "taskManager", "guidedOnboarding"],
"requiredPlugins": ["licensing", "data", "encryptedSavedObjects", "navigation", "customIntegrations", "share", "spaces", "security", "unifiedSearch", "savedObjectsTagging", "taskManager", "guidedOnboarding", "files"],
"optionalPlugins": ["features", "cloud", "usageCollection", "home", "globalSearch", "telemetry", "discover", "ingestPipelines"],
"extraPublicDirs": ["common"],
"requiredBundles": ["kibanaReact", "cloudChat", "esUiShared", "infra", "kibanaUtils", "usageCollection", "unifiedSearch"]

4
x-pack/plugins/fleet/server/constants/fleet_es_assets.ts
View file

@ -196,5 +196,5 @@ on_failure:
- 'failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}'`;
// File storage indexes supporting endpoint Upload/download
export const FILE_STORAGE_METADATA_INDEX = '.fleet-*-files';
export const FILE_STORAGE_DATA_INDEX = '.fleet-*-file-data';
export const FILE_STORAGE_METADATA_INDEX_PATTERN = '.fleet-*-files';
export const FILE_STORAGE_DATA_INDEX_PATTERN = '.fleet-*-file-data';

8
x-pack/plugins/fleet/server/services/files/index.test.ts
View file

@ -10,8 +10,8 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import { ES_SEARCH_LIMIT } from '../../../common/constants';
import {
FILE_STORAGE_DATA_INDEX,
FILE_STORAGE_METADATA_INDEX,
FILE_STORAGE_DATA_INDEX_PATTERN,
FILE_STORAGE_METADATA_INDEX_PATTERN,
} from '../../constants/fleet_es_assets';
import { fileIdsWithoutChunksByIndex, getFilesByStatus, updateFilesStatus } from '.';
@ -61,7 +61,7 @@ describe('files service', () => {
expect(esClientMock.search).toBeCalledWith(
{
index: FILE_STORAGE_METADATA_INDEX,
index: FILE_STORAGE_METADATA_INDEX_PATTERN,
body: {
size: ES_SEARCH_LIMIT,
query: {
@ -124,7 +124,7 @@ describe('files service', () => {
expect(esClientMock.search).toBeCalledWith(
{
index: FILE_STORAGE_DATA_INDEX,
index: FILE_STORAGE_DATA_INDEX_PATTERN,
body: {
size: ES_SEARCH_LIMIT,
query: {

14
x-pack/plugins/fleet/server/services/files/index.ts
View file

@ -7,13 +7,13 @@
import type { ElasticsearchClient } from '@kbn/core/server';
import type { UpdateByQueryResponse, SearchHit } from '@elastic/elasticsearch/lib/api/types';
import type { FileStatus } from '@kbn/files-plugin/common/types';
import {
FILE_STORAGE_DATA_INDEX,
FILE_STORAGE_METADATA_INDEX,
FILE_STORAGE_DATA_INDEX_PATTERN,
FILE_STORAGE_METADATA_INDEX_PATTERN,
} from '../../constants/fleet_es_assets';
import { ES_SEARCH_LIMIT } from '../../../common/constants';
import type { FILE_STATUS } from '../../types/files';
/**
* Gets files with given status
@ -25,11 +25,11 @@ import type { FILE_STATUS } from '../../types/files';
export async function getFilesByStatus(
esClient: ElasticsearchClient,
abortController: AbortController,
status: FILE_STATUS = 'READY'
status: FileStatus = 'READY'
): Promise<SearchHit[]> {
const result = await esClient.search(
{
index: FILE_STORAGE_METADATA_INDEX,
index: FILE_STORAGE_METADATA_INDEX_PATTERN,
body: {
size: ES_SEARCH_LIMIT,
query: {
@ -74,7 +74,7 @@ export async function fileIdsWithoutChunksByIndex(
const chunks = await esClient.search<{ bid: string }>(
{
index: FILE_STORAGE_DATA_INDEX,
index: FILE_STORAGE_DATA_INDEX_PATTERN,
body: {
size: ES_SEARCH_LIMIT,
query: {
@ -124,7 +124,7 @@ export function updateFilesStatus(
esClient: ElasticsearchClient,
abortController: AbortController,
fileIdsByIndex: FileIdsByIndex,
status: FILE_STATUS
status: FileStatus
): Promise<UpdateByQueryResponse[]> {
return Promise.all(
Object.entries(fileIdsByIndex).map(([index, fileIds]) => {

13
x-pack/plugins/fleet/server/tasks/check_deleted_files_task.test.ts
View file

@ -14,7 +14,10 @@ import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-ser
import { loggingSystemMock } from '@kbn/core/server/mocks';
import { createAppContextStartContractMock } from '../mocks';
import { FILE_STORAGE_DATA_INDEX, FILE_STORAGE_METADATA_INDEX } from '../constants/fleet_es_assets';
import {
FILE_STORAGE_DATA_INDEX_PATTERN,
FILE_STORAGE_METADATA_INDEX_PATTERN,
} from '../constants/fleet_es_assets';
import { appContextService } from '../services';
import { CheckDeletedFilesTask, TYPE, VERSION } from './check_deleted_files_task';
@ -115,12 +118,12 @@ describe('check deleted files task', () => {
hits: [
{
_id: 'metadata-testid1',
_index: FILE_STORAGE_METADATA_INDEX,
_index: FILE_STORAGE_METADATA_INDEX_PATTERN,
_source: { file: { status: 'READY' } },
},
{
_id: 'metadata-testid2',
_index: FILE_STORAGE_METADATA_INDEX,
_index: FILE_STORAGE_METADATA_INDEX_PATTERN,
_source: { file: { status: 'READY' } },
},
],
@ -144,7 +147,7 @@ describe('check deleted files task', () => {
hits: [
{
_id: 'data-testid1',
_index: FILE_STORAGE_DATA_INDEX,
_index: FILE_STORAGE_DATA_INDEX_PATTERN,
_source: {
bid: 'metadata-testid1',
},
@ -157,7 +160,7 @@ describe('check deleted files task', () => {
expect(esClient.updateByQuery).toHaveBeenCalledWith(
{
index: FILE_STORAGE_METADATA_INDEX,
index: FILE_STORAGE_METADATA_INDEX_PATTERN,
query: {
ids: {
values: ['metadata-testid2'],

8
x-pack/plugins/fleet/server/types/files/index.ts
View file

@ -1,8 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export type FILE_STATUS = 'AWAITING_UPLOAD' | 'UPLOADING' | 'READY' | 'UPLOAD_ERROR' | 'DELETED';

3
x-pack/plugins/fleet/tsconfig.json
View file

@ -27,7 +27,8 @@
{ "path": "../licensing/tsconfig.json" },
{ "path": "../../../src/plugins/data/tsconfig.json" },
{ "path": "../encrypted_saved_objects/tsconfig.json" },
{ "path": "../../../src/plugins/guided_onboarding/tsconfig.json" },
{ "path": "../../../src/plugins/guided_onboarding/tsconfig.json" },
{ "path": "../../../src/plugins/files/tsconfig.json"},
// optionalPlugins from ./kibana.json
{ "path": "../security/tsconfig.json" },