[Share] Add unused url cleanup task (#220138)

## Summary
This PR adds a background task to `share` plugin which periodically
deletes saved objects of type `url` which have been older than a value
controlled by `share.url_expiration.duration` config - the default is 1
year.
The task can be run manually by calling `POST
/internal/unused_urls_task/run` with `superuser` privileges.

Config options (with their default values):

```yaml
share.url_expiration.enabled: false # controls whether the task is enabled
share.url_expiration.duration: '1y' # controls the expiration threshold
share.url_expiration.check_interval: '7d' # controls how often the task runs 
share.url_expiration.url_limit: 10000 # controls how many urls should be fetched at once
```

Closes: #179146

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Krzysztof Kowalczyk 2025-06-18 16:29:29 +02:00 committed by GitHub
parent 9975f8d295
commit 5c441a49cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 1274 additions and 13 deletions

View file

@ -407,3 +407,4 @@ enabled:
- x-pack/platform/test/saved_object_api_integration/security_and_spaces/config_trial.ts
- x-pack/platform/test/saved_object_api_integration/spaces_only/config.ts
- x-pack/platform/test/saved_object_api_integration/user_profiles/config.ts
- src/platform/test/api_integration/apis/unused_urls_task/config.ts

2
.github/CODEOWNERS vendored
View file

@ -2786,6 +2786,7 @@ x-pack/solutions/observability/plugins/observability_shared/public/components/pr
# Shared UX
/x-pack/test_serverless/api_integration/test_suites/common/favorites @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/200985
/src/platform/test/api_integration/apis/short_url/**/*.ts @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/200209/files#r1846654156
/src/platform/test/api_integration/apis/unused_urls_task/**/*.ts @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/220138
/src/platform/test/functional/page_objects/share_page.ts @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/200209/files#r1846648444
/src/platform/test/accessibility/apps/kibana_overview_* @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/200209/files/cab99bce5ac2082fa77222beebe3b61ff836b94b#r1846659920
/x-pack/test/functional/services/sample_data @elastic/appex-sharedux # Assigned per https://github.com/elastic/kibana/pull/200142#discussion_r1846512756
@ -2807,6 +2808,7 @@ x-pack/solutions/observability/plugins/observability_shared/public/components/pr
/x-pack/test/functional/apps/advanced_settings @elastic/appex-sharedux
/src/platform/test/functional/services/monaco_editor.ts @elastic/appex-sharedux
/x-pack/test/functional/fixtures/kbn_archiver/global_search @elastic/appex-sharedux
/src/platform/test/api_integration/fixtures/unused_urls_task @elastic/appex-sharedux
/x-pack/test/plugin_functional/test_suites/global_search @elastic/appex-sharedux
/src/platform/test/plugin_functional/test_suites/shared_ux @elastic/appex-sharedux
/src/platform/test/plugin_functional/plugins/kbn_sample_panel_action @elastic/appex-sharedux

View file

@ -11,7 +11,12 @@
"id": "share",
"browser": true,
"server": true,
"requiredBundles": ["kibanaUtils"],
"optionalPlugins": ["licensing"]
"requiredBundles": [
"kibanaUtils"
],
"optionalPlugins": [
"licensing",
"taskManager"
]
}
}
}

View file

@ -8,6 +8,11 @@
*/
import { schema, TypeOf } from '@kbn/config-schema';
import {
DEFAULT_URL_LIMIT,
DEFAULT_URL_EXPIRATION_CHECK_INTERVAL,
DEFAULT_URL_EXPIRATION_DURATION,
} from './unused_urls_task';
export const configSchema = schema.object({
new_version: schema.object({
@ -15,6 +20,20 @@ export const configSchema = schema.object({
defaultValue: false,
}),
}),
url_expiration: schema.object({
enabled: schema.boolean({
defaultValue: false,
}),
duration: schema.duration({
defaultValue: DEFAULT_URL_EXPIRATION_DURATION,
}),
check_interval: schema.duration({
defaultValue: DEFAULT_URL_EXPIRATION_CHECK_INTERVAL,
}),
url_limit: schema.number({
defaultValue: DEFAULT_URL_LIMIT,
}),
}),
});
export type ConfigSchema = TypeOf<typeof configSchema>;

View file

@ -17,6 +17,23 @@ export type {
export { CSV_QUOTE_VALUES_SETTING, CSV_SEPARATOR_SETTING } from '../common/constants';
export {
TASK_ID,
SAVED_OBJECT_TYPE,
DEFAULT_URL_LIMIT,
DEFAULT_URL_EXPIRATION_CHECK_INTERVAL,
DEFAULT_URL_EXPIRATION_DURATION,
} from './unused_urls_task';
export {
durationToSeconds,
getDeleteUnusedUrlTaskInstance,
deleteUnusedUrls,
fetchUnusedUrlsFromFirstNamespace,
runDeleteUnusedUrlsTask,
scheduleUnusedUrlsCleanupTask,
} from './unused_urls_task';
export async function plugin(initializerContext: PluginInitializerContext) {
const { SharePlugin } = await import('./plugin');
return new SharePlugin(initializerContext);

View file

@ -9,7 +9,17 @@
import { i18n } from '@kbn/i18n';
import { schema } from '@kbn/config-schema';
import { CoreSetup, Plugin, PluginInitializerContext } from '@kbn/core/server';
import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server';
import {
TaskManagerSetupContract,
TaskManagerStartContract,
} from '@kbn/task-manager-plugin/server';
import { registerDeleteUnusedUrlsRoute } from './unused_urls_task/register_delete_unused_urls_route';
import {
TASK_ID,
runDeleteUnusedUrlsTask,
scheduleUnusedUrlsCleanupTask,
} from './unused_urls_task';
import { CSV_SEPARATOR_SETTING, CSV_QUOTE_VALUES_SETTING } from '../common/constants';
import { UrlService } from '../common/url_service';
import {
@ -20,6 +30,7 @@ import {
} from './url_service';
import { LegacyShortUrlLocatorDefinition } from '../common/url_service/locators/legacy_short_url_locator';
import { ShortUrlRedirectLocatorDefinition } from '../common/url_service/locators/short_url_redirect_locator';
import { ConfigSchema } from './config';
/** @public */
export interface SharePublicSetup {
@ -31,11 +42,13 @@ export interface SharePublicStart {
url: ServerUrlService;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface SharePublicSetupDependencies {}
export interface SharePublicSetupDependencies {
taskManager?: TaskManagerSetupContract;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface SharePublicStartDependencies {}
export interface SharePublicStartDependencies {
taskManager?: TaskManagerStartContract;
}
export class SharePlugin
implements
@ -47,13 +60,17 @@ export class SharePlugin
>
{
private url?: ServerUrlService;
private version: string;
private readonly version: string;
private readonly logger: Logger;
private readonly config: ConfigSchema;
constructor(private readonly initializerContext: PluginInitializerContext) {
this.version = initializerContext.env.packageInfo.version;
this.logger = initializerContext.logger.get();
this.config = initializerContext.config.get<ConfigSchema>();
}
public setup(core: CoreSetup) {
public setup(core: CoreSetup, { taskManager }: SharePublicSetupDependencies) {
this.url = new UrlService({
baseUrl: core.http.basePath.publicBaseUrl || core.http.basePath.serverBasePath,
version: this.initializerContext.env.packageInfo.version,
@ -75,6 +92,15 @@ export class SharePlugin
registerUrlServiceSavedObjectType(core.savedObjects, this.url);
registerUrlServiceRoutes(core, core.http.createRouter(), this.url);
registerDeleteUnusedUrlsRoute({
router: core.http.createRouter(),
core,
urlExpirationDuration: this.config.url_expiration.duration,
urlLimit: this.config.url_expiration.url_limit,
logger: this.logger,
isEnabled: this.config.url_expiration.enabled && Boolean(taskManager),
});
core.uiSettings.register({
[CSV_SEPARATOR_SETTING]: {
name: i18n.translate('share.advancedSettings.csv.separatorTitle', {
@ -98,13 +124,42 @@ export class SharePlugin
},
});
if (taskManager) {
taskManager.registerTaskDefinitions({
[TASK_ID]: {
title: 'Unused URLs Cleanup',
description: "Deletes unused saved objects of type 'url'",
maxAttempts: 5,
createTaskRunner: () => ({
run: async () => {
await runDeleteUnusedUrlsTask({
core,
urlExpirationDuration: this.config.url_expiration.duration,
logger: this.logger,
urlLimit: this.config.url_expiration.url_limit,
isEnabled: this.config.url_expiration.enabled,
});
},
}),
},
});
}
return {
url: this.url,
};
}
public start() {
this.initializerContext.logger.get().debug('Starting plugin');
public start(_core: CoreStart, { taskManager }: SharePublicStartDependencies) {
this.logger.debug('Starting plugin');
if (taskManager) {
void scheduleUnusedUrlsCleanupTask({
taskManager,
checkInterval: this.config.url_expiration.check_interval,
isEnabled: this.config.url_expiration.enabled,
});
}
return {
url: this.url!,
@ -112,6 +167,6 @@ export class SharePlugin
}
public stop() {
this.initializerContext.logger.get().debug('Stopping plugin');
this.logger.debug('Stopping plugin');
}
}

View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export const TASK_ID = 'unusedUrlsCleanupTask';
export const SAVED_OBJECT_TYPE = 'url';
export const DEFAULT_URL_LIMIT = 10000;
export const DEFAULT_URL_EXPIRATION_DURATION = '1y';
export const DEFAULT_URL_EXPIRATION_CHECK_INTERVAL = '7d';

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export * from './constants';
export * from './task';
export * from './register_unused_urls_task_routes';

View file

@ -0,0 +1,120 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import moment from 'moment';
import { KibanaRequest, ReservedPrivilegesSet } from '@kbn/core/server';
import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
import { coreMock, httpResourcesMock } from '@kbn/core/server/mocks';
import { mockRouter as router } from '@kbn/core-http-router-server-mocks';
import { registerDeleteUnusedUrlsRoute } from './register_delete_unused_urls_route';
import { runDeleteUnusedUrlsTask } from './task';
jest.mock('./task', () => ({
runDeleteUnusedUrlsTask: jest.fn().mockResolvedValue({ deletedCount: 5 }),
}));
describe('registerDeleteUnusedUrlsRoute', () => {
const mockRouter = router.create();
const mockCoreSetup = coreMock.createSetup();
const mockUrlExpirationDuration = moment.duration(1, 'year');
const mockUrlLimit = 1000;
const mockLogger = loggingSystemMock.create().get();
const mockResponseFactory = httpResourcesMock.createResponseFactory();
beforeEach(() => {
mockRouter.post.mockReset();
});
it('registers the POST route with correct path and options', () => {
registerDeleteUnusedUrlsRoute({
router: mockRouter,
core: mockCoreSetup,
urlExpirationDuration: mockUrlExpirationDuration,
urlLimit: mockUrlLimit,
logger: mockLogger,
isEnabled: true,
});
expect(mockRouter.post).toHaveBeenCalledTimes(1);
expect(mockRouter.post).toHaveBeenCalledWith(
expect.objectContaining({
path: '/internal/unused_urls_task/run',
security: {
authz: {
requiredPrivileges: [ReservedPrivilegesSet.superuser],
},
},
options: {
access: 'internal',
summary: 'Runs the unused URLs cleanup task',
},
validate: {},
}),
expect.any(Function)
);
});
it('route handler calls runDeleteUnusedUrlsTask and returns success response', async () => {
registerDeleteUnusedUrlsRoute({
router: mockRouter,
core: mockCoreSetup,
urlExpirationDuration: mockUrlExpirationDuration,
urlLimit: mockUrlLimit,
logger: mockLogger,
isEnabled: true,
});
const routeHandler = mockRouter.post.mock.calls[0][1];
const mockRequest = {} as KibanaRequest;
const mockContext = {} as any;
await routeHandler(mockContext, mockRequest, mockResponseFactory);
expect(runDeleteUnusedUrlsTask).toHaveBeenCalledTimes(1);
expect(runDeleteUnusedUrlsTask).toHaveBeenCalledWith({
core: mockCoreSetup,
urlExpirationDuration: mockUrlExpirationDuration,
urlLimit: mockUrlLimit,
logger: mockLogger,
isEnabled: true,
});
expect(mockResponseFactory.ok).toHaveBeenCalledTimes(1);
expect(mockResponseFactory.ok).toHaveBeenCalledWith({
body: {
message: 'Unused URLs cleanup task has finished.',
deletedCount: 5,
},
});
});
it('returns forbidden response if task is disabled', async () => {
registerDeleteUnusedUrlsRoute({
router: mockRouter,
core: mockCoreSetup,
urlExpirationDuration: mockUrlExpirationDuration,
urlLimit: mockUrlLimit,
logger: mockLogger,
isEnabled: false,
});
const routeHandler = mockRouter.post.mock.calls[0][1];
const mockRequest = {} as KibanaRequest;
const mockContext = {} as any;
await routeHandler(mockContext, mockRequest, mockResponseFactory);
expect(mockResponseFactory.forbidden).toHaveBeenCalledTimes(1);
expect(mockResponseFactory.forbidden).toHaveBeenCalledWith({
body: {
message: 'Unused URLs cleanup task is disabled. Enable it in the configuration.',
},
});
});
});

View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Duration } from 'moment';
import { IRouter, Logger, ReservedPrivilegesSet } from '@kbn/core/server';
import { CoreSetup } from '@kbn/core/server';
import { runDeleteUnusedUrlsTask } from './task';
export const registerDeleteUnusedUrlsRoute = ({
router,
core,
urlExpirationDuration,
urlLimit,
logger,
isEnabled,
}: {
router: IRouter;
core: CoreSetup;
urlExpirationDuration: Duration;
urlLimit: number;
logger: Logger;
isEnabled: boolean;
}) => {
router.post(
{
path: '/internal/unused_urls_task/run',
security: {
authz: {
requiredPrivileges: [ReservedPrivilegesSet.superuser],
},
},
options: {
access: 'internal',
summary: 'Runs the unused URLs cleanup task',
},
validate: {},
},
async (_ctx, _req, res) => {
if (!isEnabled) {
return res.forbidden({
body: {
message: 'Unused URLs cleanup task is disabled. Enable it in the configuration.',
},
});
}
const { deletedCount } = await runDeleteUnusedUrlsTask({
core,
urlExpirationDuration,
urlLimit,
logger,
isEnabled,
});
return res.ok({
body: {
message: 'Unused URLs cleanup task has finished.',
deletedCount,
},
});
}
);
};

View file

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Duration } from 'moment';
import { CoreSetup, IRouter, Logger } from '@kbn/core/server';
import { registerDeleteUnusedUrlsRoute } from './register_delete_unused_urls_route';
export const registerUrlServiceRoutes = ({
router,
core,
urlExpirationDuration,
urlLimit,
logger,
isEnabled,
}: {
router: IRouter;
core: CoreSetup;
urlExpirationDuration: Duration;
urlLimit: number;
logger: Logger;
isEnabled: boolean;
}) => {
registerDeleteUnusedUrlsRoute({
router,
core,
urlExpirationDuration,
urlLimit,
logger,
isEnabled,
});
};

View file

@ -0,0 +1,497 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import moment from 'moment';
import { TaskInstanceWithId } from '@kbn/task-manager-plugin/server/task';
import {
SavedObjectsBulkDeleteObject,
SavedObjectsBulkDeleteResponse,
SavedObjectsFindResult,
SavedObjectsServiceStart,
} from '@kbn/core/server';
import { coreMock, loggingSystemMock, savedObjectsRepositoryMock } from '@kbn/core/server/mocks';
import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks';
import { SAVED_OBJECT_TYPE, TASK_ID } from './constants';
import {
durationToSeconds,
getDeleteUnusedUrlTaskInstance,
deleteUnusedUrls,
fetchUnusedUrlsFromFirstNamespace,
runDeleteUnusedUrlsTask,
scheduleUnusedUrlsCleanupTask,
} from './task';
describe('unused_urls_task', () => {
const mockSavedObjectsRepository = savedObjectsRepositoryMock.create();
const mockLogger = loggingSystemMock.create().get();
const mockCoreSetup = coreMock.createSetup();
const mockCoreStart = coreMock.createStart();
const mockTaskManager = taskManagerMock.createStart();
const checkInterval = moment.duration(1, 'hour');
const urlExpirationDuration = moment.duration(30, 'days');
mockCoreSetup.getStartServices.mockResolvedValue([
{
...mockCoreStart,
savedObjects: {
createInternalRepository: jest.fn(() => mockSavedObjectsRepository),
} as unknown as SavedObjectsServiceStart,
},
{},
{},
]);
describe('durationToSeconds', () => {
it('should convert moment duration to seconds string', () => {
const duration = moment.duration(5, 'minutes');
expect(durationToSeconds(duration)).toBe('300s');
});
});
describe('getDeleteUnusedUrlTaskInstance', () => {
it('should return a valid TaskInstanceWithId', () => {
const interval = moment.duration(1, 'hour');
const taskInstance = getDeleteUnusedUrlTaskInstance(interval);
expect(taskInstance).toEqual({
id: TASK_ID,
taskType: TASK_ID,
params: {},
state: {},
schedule: {
interval: '3600s',
},
scope: ['share'],
});
});
});
describe('deleteUnusedUrls', () => {
it('should call bulkDelete', async () => {
const unusedUrls: SavedObjectsBulkDeleteObject[] = [{ type: 'url', id: '1' }];
const namespace = 'test-namespace';
mockSavedObjectsRepository.bulkDelete.mockResolvedValue({} as SavedObjectsBulkDeleteResponse);
await deleteUnusedUrls({
savedObjectsRepository: mockSavedObjectsRepository,
unusedUrls,
namespace,
logger: mockLogger,
});
expect(mockSavedObjectsRepository.bulkDelete).toHaveBeenCalledWith(unusedUrls, {
refresh: 'wait_for',
namespace,
});
});
it('should throw an error if bulkDelete fails', async () => {
const unusedUrls = [{ type: 'url', id: '1' }];
const namespace = 'test-namespace';
const errorMessage = 'Bulk delete failed';
mockSavedObjectsRepository.bulkDelete.mockRejectedValue(new Error(errorMessage));
await expect(
deleteUnusedUrls({
savedObjectsRepository: mockSavedObjectsRepository,
unusedUrls,
namespace,
logger: mockLogger,
})
).rejects.toThrow(
`Failed to delete unused URL(s) in namespace "${namespace}": ${errorMessage}`
);
});
});
describe('fetchUnusedUrls', () => {
it('should fetch unused URLs and determine hasMore correctly', async () => {
const urlLimit = 2;
const savedObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
namespaces: ['test-namespace'],
},
{
id: '2',
type: SAVED_OBJECT_TYPE,
namespaces: ['test-namespace'],
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: savedObjects,
total: 3,
per_page: urlLimit,
page: 1,
});
const result = await fetchUnusedUrlsFromFirstNamespace({
savedObjectsRepository: mockSavedObjectsRepository,
urlExpirationDuration,
urlLimit,
});
expect(mockSavedObjectsRepository.find).toHaveBeenCalledWith({
type: SAVED_OBJECT_TYPE,
filter: 'url.attributes.accessDate <= now-2592000s',
perPage: urlLimit,
namespaces: ['*'],
fields: ['type'],
});
const savedObjectsDeleteObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
},
{
id: '2',
type: SAVED_OBJECT_TYPE,
},
];
expect(result.unusedUrls).toEqual(savedObjectsDeleteObjects);
expect(result.hasMore).toBe(true);
expect(result.namespace).toBe('test-namespace');
});
it('should set hasMore to false if fewer items than urlLimit are returned', async () => {
const urlLimit = 2;
const savedObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
namespaces: ['test-namespace'],
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: savedObjects,
total: 1,
per_page: urlLimit,
page: 1,
});
const result = await fetchUnusedUrlsFromFirstNamespace({
savedObjectsRepository: mockSavedObjectsRepository,
urlExpirationDuration,
urlLimit,
});
const savedObjectsDeleteObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
},
];
expect(result.unusedUrls).toEqual(savedObjectsDeleteObjects);
expect(result.hasMore).toBe(false);
expect(result.namespace).toBe('test-namespace');
});
it('should return default namespace if first object has no namespaces', async () => {
const urlLimit = 10;
const savedObjects = [
{
id: `id-1`,
type: SAVED_OBJECT_TYPE,
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: savedObjects,
total: 1,
per_page: urlLimit,
page: 1,
});
const result = await fetchUnusedUrlsFromFirstNamespace({
savedObjectsRepository: mockSavedObjectsRepository,
urlExpirationDuration,
urlLimit,
});
expect(result.namespace).toBe('default');
});
});
describe('runDeleteUnusedUrlsTask', () => {
beforeEach(() => {
mockSavedObjectsRepository.find.mockReset();
mockSavedObjectsRepository.bulkDelete.mockReset();
});
it('should not call delete if there are no saved objects', async () => {
const urlLimit = 2;
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: [],
total: 0,
per_page: urlLimit,
page: 1,
});
await runDeleteUnusedUrlsTask({
core: mockCoreSetup,
urlExpirationDuration,
urlLimit,
logger: mockLogger,
isEnabled: true,
});
expect(mockSavedObjectsRepository.find).toHaveBeenCalledTimes(1);
expect(mockSavedObjectsRepository.bulkDelete).not.toHaveBeenCalled();
});
it('should delete unused URLs if found', async () => {
const savedObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
namespaces: ['my-space'],
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: savedObjects,
total: 1,
per_page: 100,
page: 1,
});
mockSavedObjectsRepository.bulkDelete.mockResolvedValue({} as SavedObjectsBulkDeleteResponse);
const response = await runDeleteUnusedUrlsTask({
core: mockCoreSetup,
urlExpirationDuration,
urlLimit: 100,
logger: mockLogger,
isEnabled: true,
});
expect(response).toEqual({
deletedCount: 1,
});
const savedObjectsDeleteObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
},
];
expect(mockSavedObjectsRepository.bulkDelete).toHaveBeenCalledWith(
savedObjectsDeleteObjects,
{
refresh: 'wait_for',
namespace: 'my-space',
}
);
});
it('should handle pagination and delete across multiple pages', async () => {
const page1 = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
namespaces: ['default'],
},
] as SavedObjectsFindResult[];
const page2 = [
{
id: '2',
type: SAVED_OBJECT_TYPE,
namespaces: ['default'],
},
] as SavedObjectsFindResult[];
const page3 = [
{
id: '3',
type: SAVED_OBJECT_TYPE,
namespaces: ['other-namespace'],
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find
.mockResolvedValueOnce({
saved_objects: page1,
total: 3,
per_page: 1,
page: 1,
})
.mockResolvedValueOnce({
saved_objects: page2,
total: 3,
per_page: 1,
page: 2,
})
.mockResolvedValueOnce({
saved_objects: page3,
total: 3,
per_page: 1,
page: 3,
});
mockSavedObjectsRepository.bulkDelete.mockResolvedValue({} as SavedObjectsBulkDeleteResponse);
const response = await runDeleteUnusedUrlsTask({
core: mockCoreSetup,
urlExpirationDuration,
urlLimit: 2,
logger: mockLogger,
isEnabled: true,
});
expect(response).toEqual({
deletedCount: 2,
});
expect(mockSavedObjectsRepository.bulkDelete).toHaveBeenCalledTimes(2);
const savedObjectsDeleteObjectsPage1 = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
},
];
const savedObjectsDeleteObjectsPage2 = [
{
id: '2',
type: SAVED_OBJECT_TYPE,
},
];
expect(mockSavedObjectsRepository.bulkDelete).toHaveBeenNthCalledWith(
1,
savedObjectsDeleteObjectsPage1,
{
refresh: 'wait_for',
namespace: 'default',
}
);
expect(mockSavedObjectsRepository.bulkDelete).toHaveBeenNthCalledWith(
2,
savedObjectsDeleteObjectsPage2,
{
refresh: 'wait_for',
namespace: 'default',
}
);
});
it('should throw if deleteUnusedUrls fails', async () => {
const savedObjects = [
{
id: '1',
type: SAVED_OBJECT_TYPE,
namespaces: ['default'],
},
] as SavedObjectsFindResult[];
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: savedObjects,
total: 1,
per_page: 100,
page: 1,
});
mockSavedObjectsRepository.bulkDelete.mockRejectedValue(new Error('bulkDelete failed'));
await expect(
runDeleteUnusedUrlsTask({
core: mockCoreSetup,
urlExpirationDuration,
urlLimit: 100,
logger: mockLogger,
isEnabled: true,
})
).rejects.toThrow('Failed to delete unused URL(s) in namespace "default": bulkDelete failed');
});
it('should skip execution if isEnabled is false', async () => {
mockSavedObjectsRepository.find.mockResolvedValue({
saved_objects: [],
total: 0,
per_page: 100,
page: 1,
});
const response = await runDeleteUnusedUrlsTask({
core: mockCoreSetup,
urlExpirationDuration,
urlLimit: 100,
logger: mockLogger,
isEnabled: false,
});
expect(response).toEqual({ deletedCount: 0 });
expect(mockSavedObjectsRepository.find).not.toHaveBeenCalled();
expect(mockSavedObjectsRepository.bulkDelete).not.toHaveBeenCalled();
});
});
describe('scheduleUnusedUrlsCleanupTask', () => {
it('should schedule the task successfully', async () => {
mockTaskManager.ensureScheduled.mockResolvedValue({} as TaskInstanceWithId);
const expectedTaskInstance = getDeleteUnusedUrlTaskInstance(checkInterval);
await scheduleUnusedUrlsCleanupTask({
taskManager: mockTaskManager,
checkInterval,
isEnabled: true,
});
expect(mockTaskManager.ensureScheduled).toHaveBeenCalledWith(expectedTaskInstance);
});
it('should throw an error if scheduling fails with a message', async () => {
const errorMessage = 'Scheduling failed';
mockTaskManager.ensureScheduled.mockRejectedValue(new Error(errorMessage));
await expect(
scheduleUnusedUrlsCleanupTask({
taskManager: mockTaskManager,
checkInterval,
isEnabled: true,
})
).rejects.toThrow(errorMessage);
});
it('should throw a generic error if scheduling fails without a message', async () => {
mockTaskManager.ensureScheduled.mockRejectedValue(new Error());
await expect(
scheduleUnusedUrlsCleanupTask({
taskManager: mockTaskManager,
checkInterval,
isEnabled: true,
})
).rejects.toThrow('Failed to schedule unused URLs cleanup task');
});
it('should remove the task if isEnabled is false and not run it', async () => {
mockTaskManager.ensureScheduled.mockClear();
await scheduleUnusedUrlsCleanupTask({
taskManager: mockTaskManager,
checkInterval,
isEnabled: false,
});
expect(mockTaskManager.ensureScheduled).not.toHaveBeenCalled();
expect(mockTaskManager.removeIfExists).toHaveBeenCalledWith(TASK_ID);
});
});
});

View file

@ -0,0 +1,182 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { Duration } from 'moment';
import { CoreSetup, ISavedObjectsRepository, SavedObjectsBulkDeleteObject } from '@kbn/core/server';
import { Logger } from '@kbn/logging';
import { TaskInstanceWithId } from '@kbn/task-manager-plugin/server/task';
import { TaskManagerStartContract } from '@kbn/task-manager-plugin/server';
import { SavedObjectsUtils } from '@kbn/core-saved-objects-utils-server';
import { SAVED_OBJECT_TYPE, TASK_ID } from './constants';
export const durationToSeconds = (duration: Duration) => `${duration.asSeconds()}s`;
export const getDeleteUnusedUrlTaskInstance = (interval: Duration): TaskInstanceWithId => ({
id: TASK_ID,
taskType: TASK_ID,
params: {},
state: {},
schedule: {
interval: durationToSeconds(interval),
},
scope: ['share'],
});
export const deleteUnusedUrls = async ({
savedObjectsRepository,
unusedUrls,
namespace,
logger,
}: {
savedObjectsRepository: ISavedObjectsRepository;
unusedUrls: SavedObjectsBulkDeleteObject[];
namespace: string;
logger: Logger;
}) => {
try {
logger.debug(`Deleting ${unusedUrls.length} unused URL(s) in namespace "${namespace}"`);
await savedObjectsRepository.bulkDelete(unusedUrls, {
refresh: 'wait_for',
namespace,
});
logger.debug(
`Succesfully deleted ${unusedUrls.length} unused URL(s) in namespace "${namespace}"`
);
} catch (e) {
throw new Error(`Failed to delete unused URL(s) in namespace "${namespace}": ${e.message}`);
}
};
export const fetchUnusedUrlsFromFirstNamespace = async ({
savedObjectsRepository,
urlExpirationDuration,
urlLimit,
}: {
savedObjectsRepository: ISavedObjectsRepository;
urlExpirationDuration: Duration;
urlLimit: number;
}) => {
const filter = `url.attributes.accessDate <= now-${durationToSeconds(urlExpirationDuration)}`;
const {
saved_objects: savedObjects,
total,
per_page: perPage,
page,
} = await savedObjectsRepository.find({
type: SAVED_OBJECT_TYPE,
filter,
perPage: urlLimit,
namespaces: ['*'],
fields: ['type'],
});
const firstNamespace = SavedObjectsUtils.namespaceIdToString(savedObjects[0]?.namespaces?.[0]);
const savedObjectsByNamespace = savedObjects.filter(
(so) => so.namespaces?.length && so.namespaces.includes(firstNamespace)
);
const unusedUrls = savedObjectsByNamespace.map((so) => ({
id: so.id,
type: so.type,
}));
return {
unusedUrls,
hasMore: page * perPage < total,
namespace: firstNamespace,
};
};
export const runDeleteUnusedUrlsTask = async ({
core,
urlExpirationDuration,
urlLimit,
logger,
isEnabled,
}: {
core: CoreSetup;
urlExpirationDuration: Duration;
urlLimit: number;
logger: Logger;
isEnabled: boolean;
}) => {
if (!isEnabled) {
logger.debug('Unused URLs cleanup task is disabled, skipping execution');
return { deletedCount: 0 };
}
logger.debug('Unused URLs cleanup started');
const [coreStart] = await core.getStartServices();
const savedObjectsRepository = coreStart.savedObjects.createInternalRepository();
let deletedCount = 0;
let { unusedUrls, hasMore, namespace } = await fetchUnusedUrlsFromFirstNamespace({
savedObjectsRepository,
urlExpirationDuration,
urlLimit,
});
while (unusedUrls.length > 0 && deletedCount < urlLimit) {
await deleteUnusedUrls({
savedObjectsRepository,
unusedUrls,
logger,
namespace,
});
deletedCount += unusedUrls.length;
if (hasMore && deletedCount < urlLimit) {
const nextPage = await fetchUnusedUrlsFromFirstNamespace({
savedObjectsRepository,
urlExpirationDuration,
urlLimit: urlLimit - deletedCount,
});
unusedUrls = nextPage.unusedUrls;
hasMore = nextPage.hasMore;
namespace = nextPage.namespace;
} else {
break;
}
}
logger.debug('Unused URLs cleanup finished');
return { deletedCount };
};
export const scheduleUnusedUrlsCleanupTask = async ({
taskManager,
checkInterval,
isEnabled,
}: {
taskManager: TaskManagerStartContract;
checkInterval: Duration;
isEnabled: boolean;
}) => {
try {
if (!isEnabled) {
await taskManager.removeIfExists(TASK_ID);
return;
}
const taskInstance = getDeleteUnusedUrlTaskInstance(checkInterval);
await taskManager.ensureScheduled(taskInstance);
} catch (e) {
throw new Error(e.message || 'Failed to schedule unused URLs cleanup task');
}
};

View file

@ -22,11 +22,15 @@
"@kbn/shared-ux-tabbed-modal",
"@kbn/core-user-profile-browser",
"@kbn/datemath",
"@kbn/task-manager-plugin",
"@kbn/logging",
"@kbn/licensing-plugin",
"@kbn/core-rendering-browser",
"@kbn/std",
"@kbn/core-http-server-internal",
"@kbn/core-test-helpers-test-utils",
"@kbn/core-logging-server-mocks",
"@kbn/core-http-router-server-mocks",
],
"exclude": ["target/**/*"]
}

View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const apiIntegrationConfig = await readConfigFile(require.resolve('../../config.js'));
return {
...apiIntegrationConfig.getAll(),
testFiles: [require.resolve('.')],
kbnTestServer: {
...apiIntegrationConfig.get('kbnTestServer'),
serverArgs: [
...apiIntegrationConfig.get('kbnTestServer.serverArgs'),
'--share.url_expiration.enabled=true',
'--share.url_expiration.url_limit=5',
],
},
};
}

View file

@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('unused_urls_task', () => {
loadTestFile(require.resolve('./run'));
});
}

View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('run', () => {
beforeEach(async () => {
await kibanaServer.importExport.load(
'src/platform/test/api_integration/fixtures/unused_urls_task/urls.ndjson'
);
});
afterEach(async () => {
await kibanaServer.savedObjects.cleanStandardList();
await kibanaServer.importExport.unload(
'src/platform/test/api_integration/fixtures/unused_urls_task/urls.ndjson'
);
});
it('runs unused URLs cleanup if its enabled', async () => {
const response1 = await supertest.post('/internal/unused_urls_task/run');
expect(response1.status).to.be(200);
// Deletes only 5 URLs because the limit is set to 5
expect(response1.body).to.eql({
message: 'Unused URLs cleanup task has finished.',
deletedCount: 5,
});
// Delete the remaining URL
const response2 = await supertest.post('/internal/unused_urls_task/run');
expect(response2.status).to.be(200);
expect(response2.body).to.eql({
message: 'Unused URLs cleanup task has finished.',
deletedCount: 1,
});
});
});
}

View file

@ -0,0 +1,132 @@
{
"id": "1",
"type": "url",
"namespaces": [
"default"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "2",
"type": "url",
"namespaces": [
"default"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "3",
"type": "url",
"namespaces": [
"default"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "4",
"type": "url",
"namespaces": [
"foo"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "5",
"type": "url",
"namespaces": [
"bar"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "non-expired-url",
"type": "url",
"namespaces": [
"different"
],
"updated_at": "2025-06-02T21:07:10.533Z",
"created_at": "2025-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1748898430533,
"createDate": 1748898430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}
{
"id": "url-over-limit",
"type": "url",
"namespaces": [
"bar"
],
"updated_at": "2023-06-02T21:07:10.533Z",
"created_at": "2023-06-02T21:07:10.533Z",
"attributes": {
"accessCount": 1,
"accessDate": 1685730430533,
"createDate": 1685730430533,
"slug": "",
"locatorJSON": "",
"url": ""
},
"references": []
}

View file

@ -190,6 +190,7 @@ export default function ({ getService }: FtrProviderContext) {
'slo:temp-summary-cleanup-task',
'task_manager:delete_inactive_background_task_nodes',
'task_manager:mark_removed_tasks_as_unrecognized',
'unusedUrlsCleanupTask',
]);
});
});