[Security Solution][Timeline] Pinned events migrations (#112360)

* Starting migration class

* Fleshing out migrator

* Adding migration tests

* Refactoring

* Adding migrator to each client

* gzipping file

* Fixing cypress tests

* Cleaning up types and adding additional test

* Starting notes migrations

* Finishing notes references migration

* gzipping data.json

* Fixing unit tests

* Updating the archive and fixing spelling

* Starting pinned events

* Fixing more conflicts

* Finishing pinned events

* fixing pinned events not showing bug

* Fixing lint errors

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Jonathan Buttner 2021-09-22 09:09:34 -04:00 committed by GitHub
parent e39a9d495b
commit 93cc4fcd9b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 312 additions and 204 deletions

View file

@ -30,6 +30,12 @@ export const SavedPinnedEventRuntimeType = runtimeTypes.intersection([
export interface SavedPinnedEvent extends runtimeTypes.TypeOf<typeof SavedPinnedEventRuntimeType> {}
/**
* This type represents a pinned event type stored in a saved object that does not include any fields that reference
* other saved objects.
*/
export type PinnedEventWithoutExternalRefs = Omit<SavedPinnedEvent, 'timelineId'>;
/**
* Note Saved object type with metadata
*/

View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TIMELINE_ID_REF_NAME } from '../../constants';
import { timelineSavedObjectType } from '../../saved_object_mappings';
import { FieldMigrator } from '../../utils/migrator';
/**
* A migrator to handle moving specific fields that reference the timeline saved object to the references field within a note saved
* object.
*/
export const pinnedEventFieldsMigrator = new FieldMigrator([
{ path: 'timelineId', type: timelineSavedObjectType, name: TIMELINE_ID_REF_NAME },
]);

View file

@ -19,13 +19,13 @@ import {
PinnedEventSavedObjectRuntimeType,
SavedPinnedEvent,
PinnedEvent as PinnedEventResponse,
PinnedEventWithoutExternalRefs,
} from '../../../../../common/types/timeline/pinned_event';
import { PageInfoNote, SortNote } from '../../../../../common/types/timeline/note';
import { FrameworkRequest } from '../../../framework';
import { pickSavedTimeline } from '../../saved_object/timelines';
import { convertSavedObjectToSavedTimeline } from '../timelines';
import { createTimeline } from '../../saved_object/timelines';
import { pinnedEventSavedObjectType } from '../../saved_object_mappings/pinned_events';
import { pinnedEventFieldsMigrator } from './field_migrator';
import { timelineSavedObjectType } from '../../saved_object_mappings';
export interface PinnedEvent {
@ -46,13 +46,6 @@ export interface PinnedEvent {
timelineId: string
) => Promise<PinnedEventSavedObject[]>;
getAllPinnedEvents: (
request: FrameworkRequest,
pageInfo: PageInfoNote | null,
search: string | null,
sort: SortNote | null
) => Promise<PinnedEventSavedObject[]>;
persistPinnedEventOnTimeline: (
request: FrameworkRequest,
pinnedEventId: string | null, // pinned event saved object id
@ -117,26 +110,7 @@ export const getAllPinnedEventsByTimelineId = async (
): Promise<PinnedEventSavedObject[]> => {
const options: SavedObjectsFindOptions = {
type: pinnedEventSavedObjectType,
search: timelineId,
searchFields: ['timelineId'],
};
return getAllSavedPinnedEvents(request, options);
};
export const getAllPinnedEvents = async (
request: FrameworkRequest,
pageInfo: PageInfoNote | null,
search: string | null,
sort: SortNote | null
): Promise<PinnedEventSavedObject[]> => {
const options: SavedObjectsFindOptions = {
type: pinnedEventSavedObjectType,
perPage: pageInfo != null ? pageInfo.pageSize : undefined,
page: pageInfo != null ? pageInfo.pageIndex : undefined,
search: search != null ? search : undefined,
searchFields: ['timelineId', 'eventId'],
sortField: sort != null ? sort.sortField : undefined,
sortOrder: sort != null ? sort.sortOrder : undefined,
hasReference: { type: timelineSavedObjectType, id: timelineId },
};
return getAllSavedPinnedEvents(request, options);
};
@ -147,51 +121,35 @@ export const persistPinnedEventOnTimeline = async (
eventId: string,
timelineId: string | null
): Promise<PinnedEventResponse | null> => {
const savedObjectsClient = request.context.core.savedObjects.client;
try {
if (pinnedEventId == null) {
const timelineVersionSavedObject =
timelineId == null
? await (async () => {
const timelineResult = convertSavedObjectToSavedTimeline(
await savedObjectsClient.create(
timelineSavedObjectType,
pickSavedTimeline(null, {}, request.user || null)
)
);
timelineId = timelineResult.savedObjectId; // eslint-disable-line no-param-reassign
return timelineResult.version;
})()
: null;
if (timelineId != null) {
const allPinnedEventId = await getAllPinnedEventsByTimelineId(request, timelineId);
const isPinnedAlreadyExisting = allPinnedEventId.filter(
(pinnedEvent) => pinnedEvent.eventId === eventId
);
if (isPinnedAlreadyExisting.length === 0) {
const savedPinnedEvent: SavedPinnedEvent = {
eventId,
timelineId,
};
// create Pinned Event on Timeline
return convertSavedObjectToSavedPinnedEvent(
await savedObjectsClient.create(
pinnedEventSavedObjectType,
pickSavedPinnedEvent(pinnedEventId, savedPinnedEvent, request.user || null)
),
timelineVersionSavedObject != null ? timelineVersionSavedObject : undefined
);
}
return isPinnedAlreadyExisting[0];
}
throw new Error('You can NOT pinned event without a timelineID');
if (pinnedEventId != null) {
// Delete Pinned Event on Timeline
await deletePinnedEventOnTimeline(request, [pinnedEventId]);
return null;
}
// Delete Pinned Event on Timeline
await deletePinnedEventOnTimeline(request, [pinnedEventId]);
return null;
const { timelineId: validatedTimelineId, timelineVersion } = await getValidTimelineIdAndVersion(
request,
timelineId
);
const pinnedEvents = await getPinnedEventsInTimelineWithEventId(
request,
validatedTimelineId,
eventId
);
// we already had this event pinned so let's just return the one we already had
if (pinnedEvents.length > 0) {
return pinnedEvents[0];
}
return await createPinnedEvent({
request,
eventId,
timelineId: validatedTimelineId,
timelineVersion,
});
} catch (err) {
if (getOr(null, 'output.statusCode', err) === 404) {
/*
@ -215,11 +173,91 @@ export const persistPinnedEventOnTimeline = async (
}
};
const getValidTimelineIdAndVersion = async (
request: FrameworkRequest,
timelineId: string | null
): Promise<{ timelineId: string; timelineVersion?: string }> => {
if (timelineId != null) {
return {
timelineId,
};
}
const savedObjectsClient = request.context.core.savedObjects.client;
// create timeline because it didn't exist
const { timeline: timelineResult } = await createTimeline({
timelineId: null,
timeline: {},
savedObjectsClient,
userInfo: request.user,
});
return {
timelineId: timelineResult.savedObjectId,
timelineVersion: timelineResult.version,
};
};
const getPinnedEventsInTimelineWithEventId = async (
request: FrameworkRequest,
timelineId: string,
eventId: string
): Promise<PinnedEventSavedObject[]> => {
const allPinnedEventId = await getAllPinnedEventsByTimelineId(request, timelineId);
const pinnedEvents = allPinnedEventId.filter((pinnedEvent) => pinnedEvent.eventId === eventId);
return pinnedEvents;
};
const createPinnedEvent = async ({
request,
eventId,
timelineId,
timelineVersion,
}: {
request: FrameworkRequest;
eventId: string;
timelineId: string;
timelineVersion?: string;
}) => {
const savedObjectsClient = request.context.core.savedObjects.client;
const savedPinnedEvent: SavedPinnedEvent = {
eventId,
timelineId,
};
const pinnedEventWithCreator = pickSavedPinnedEvent(null, savedPinnedEvent, request.user);
const { transformedFields: migratedAttributes, references } =
pinnedEventFieldsMigrator.extractFieldsToReferences<PinnedEventWithoutExternalRefs>({
data: pinnedEventWithCreator,
});
const createdPinnedEvent = await savedObjectsClient.create<PinnedEventWithoutExternalRefs>(
pinnedEventSavedObjectType,
migratedAttributes,
{ references }
);
const repopulatedSavedObject =
pinnedEventFieldsMigrator.populateFieldsFromReferences(createdPinnedEvent);
// create Pinned Event on Timeline
return convertSavedObjectToSavedPinnedEvent(repopulatedSavedObject, timelineVersion);
};
const getSavedPinnedEvent = async (request: FrameworkRequest, pinnedEventId: string) => {
const savedObjectsClient = request.context.core.savedObjects.client;
const savedObject = await savedObjectsClient.get(pinnedEventSavedObjectType, pinnedEventId);
const savedObject = await savedObjectsClient.get<PinnedEventWithoutExternalRefs>(
pinnedEventSavedObjectType,
pinnedEventId
);
return convertSavedObjectToSavedPinnedEvent(savedObject);
const populatedPinnedEvent = pinnedEventFieldsMigrator.populateFieldsFromReferences(savedObject);
return convertSavedObjectToSavedPinnedEvent(populatedPinnedEvent);
};
const getAllSavedPinnedEvents = async (
@ -227,11 +265,14 @@ const getAllSavedPinnedEvents = async (
options: SavedObjectsFindOptions
) => {
const savedObjectsClient = request.context.core.savedObjects.client;
const savedObjects = await savedObjectsClient.find(options);
const savedObjects = await savedObjectsClient.find<PinnedEventWithoutExternalRefs>(options);
return savedObjects.saved_objects.map((savedObject) =>
convertSavedObjectToSavedPinnedEvent(savedObject)
);
return savedObjects.saved_objects.map((savedObject) => {
const populatedPinnedEvent =
pinnedEventFieldsMigrator.populateFieldsFromReferences(savedObject);
return convertSavedObjectToSavedPinnedEvent(populatedPinnedEvent);
});
};
export const savePinnedEvents = (
@ -284,11 +325,10 @@ export const pickSavedPinnedEvent = (
if (pinnedEventId == null) {
savedPinnedEvent.created = dateNow;
savedPinnedEvent.createdBy = userInfo?.username ?? UNAUTHENTICATED_USER;
savedPinnedEvent.updated = dateNow;
savedPinnedEvent.updatedBy = userInfo?.username ?? UNAUTHENTICATED_USER;
} else if (pinnedEventId != null) {
savedPinnedEvent.updated = dateNow;
savedPinnedEvent.updatedBy = userInfo?.username ?? UNAUTHENTICATED_USER;
}
savedPinnedEvent.updated = dateNow;
savedPinnedEvent.updatedBy = userInfo?.username ?? UNAUTHENTICATED_USER;
return savedPinnedEvent;
};

View file

@ -1,40 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TIMELINE_ID_REF_NAME } from '../../constants';
import { migrateNoteTimelineIdToReferences, TimelineId } from './notes';
describe('notes migrations', () => {
describe('7.16.0 timelineId', () => {
it('removes the timelineId from the migrated document', () => {
const migratedDoc = migrateNoteTimelineIdToReferences({
id: '1',
type: 'awesome',
attributes: { timelineId: '123' },
});
expect(migratedDoc.attributes).toEqual({});
expect(migratedDoc.references).toEqual([
// importing the timeline saved object type from the timeline saved object causes a circular import and causes the jest tests to fail
{ id: '123', name: TIMELINE_ID_REF_NAME, type: 'siem-ui-timeline' },
]);
});
it('preserves additional fields when migrating timeline id', () => {
const migratedDoc = migrateNoteTimelineIdToReferences({
id: '1',
type: 'awesome',
attributes: { awesome: 'yes', timelineId: '123' } as unknown as TimelineId,
});
expect(migratedDoc.attributes).toEqual({ awesome: 'yes' });
expect(migratedDoc.references).toEqual([
{ id: '123', name: TIMELINE_ID_REF_NAME, type: 'siem-ui-timeline' },
]);
});
});
});

View file

@ -5,39 +5,9 @@
* 2.0.
*/
import {
SavedObjectMigrationMap,
SavedObjectSanitizedDoc,
SavedObjectUnsanitizedDoc,
} from 'kibana/server';
import { timelineSavedObjectType } from '..';
import { TIMELINE_ID_REF_NAME } from '../../constants';
import { createMigratedDoc, createReference } from './utils';
export interface TimelineId {
timelineId?: string | null;
}
export const migrateNoteTimelineIdToReferences = (
doc: SavedObjectUnsanitizedDoc<TimelineId>
): SavedObjectSanitizedDoc<unknown> => {
const { timelineId, ...restAttributes } = doc.attributes;
const { references: docReferences = [] } = doc;
const timelineIdReferences = createReference(
timelineId,
TIMELINE_ID_REF_NAME,
timelineSavedObjectType
);
return createMigratedDoc({
doc,
attributes: restAttributes,
docReferences,
migratedReferences: timelineIdReferences,
});
};
import { SavedObjectMigrationMap } from 'kibana/server';
import { migrateTimelineIdToReferences } from './utils';
export const notesMigrations: SavedObjectMigrationMap = {
'7.16.0': migrateNoteTimelineIdToReferences,
'7.16.0': migrateTimelineIdToReferences,
};

View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectMigrationMap } from 'kibana/server';
import { migrateTimelineIdToReferences } from './utils';
export const pinnedEventsMigrations: SavedObjectMigrationMap = {
'7.16.0': migrateTimelineIdToReferences,
};

View file

@ -5,5 +5,6 @@
* 2.0.
*/
export { timelinesMigrations } from './timelines';
export { notesMigrations } from './notes';
export interface TimelineId {
timelineId?: string | null;
}

View file

@ -5,9 +5,39 @@
* 2.0.
*/
import { createMigratedDoc, createReference } from './utils';
import { timelineSavedObjectType } from '../timelines';
import { TIMELINE_ID_REF_NAME } from '../../constants';
import { TimelineId } from './types';
import { createMigratedDoc, createReference, migrateTimelineIdToReferences } from './utils';
describe('migration utils', () => {
describe('migrateTimelineIdToReferences', () => {
it('removes the timelineId from the migrated document', () => {
const migratedDoc = migrateTimelineIdToReferences({
id: '1',
type: 'awesome',
attributes: { timelineId: '123' },
});
expect(migratedDoc.attributes).toEqual({});
expect(migratedDoc.references).toEqual([
{ id: '123', name: TIMELINE_ID_REF_NAME, type: timelineSavedObjectType },
]);
});
it('preserves additional fields when migrating timeline id', () => {
const migratedDoc = migrateTimelineIdToReferences({
id: '1',
type: 'awesome',
attributes: { awesome: 'yes', timelineId: '123' } as unknown as TimelineId,
});
expect(migratedDoc.attributes).toEqual({ awesome: 'yes' });
expect(migratedDoc.references).toEqual([
{ id: '123', name: TIMELINE_ID_REF_NAME, type: timelineSavedObjectType },
]);
});
});
describe('createReference', () => {
it('returns an array with a reference when the id is defined', () => {
expect(createReference('awesome', 'name', 'type')).toEqual([

View file

@ -10,6 +10,9 @@ import {
SavedObjectSanitizedDoc,
SavedObjectUnsanitizedDoc,
} from 'kibana/server';
import { timelineSavedObjectType } from '../timelines';
import { TIMELINE_ID_REF_NAME } from '../../constants';
import { TimelineId } from './types';
export function createReference(
id: string | null | undefined,
@ -19,6 +22,26 @@ export function createReference(
return id != null ? [{ id, name, type }] : [];
}
export const migrateTimelineIdToReferences = (
doc: SavedObjectUnsanitizedDoc<TimelineId>
): SavedObjectSanitizedDoc<unknown> => {
const { timelineId, ...restAttributes } = doc.attributes;
const { references: docReferences = [] } = doc;
const timelineIdReferences = createReference(
timelineId,
TIMELINE_ID_REF_NAME,
timelineSavedObjectType
);
return createMigratedDoc({
doc,
attributes: restAttributes,
docReferences,
migratedReferences: timelineIdReferences,
});
};
export const createMigratedDoc = <T>({
doc,
attributes,

View file

@ -6,7 +6,7 @@
*/
import { SavedObjectsType } from '../../../../../../../src/core/server';
import { notesMigrations } from './migrations';
import { notesMigrations } from './migrations/notes';
export const noteSavedObjectType = 'siem-ui-timeline-note';

View file

@ -6,14 +6,12 @@
*/
import { SavedObjectsType } from '../../../../../../../src/core/server';
import { pinnedEventsMigrations } from './migrations/pinned_events';
export const pinnedEventSavedObjectType = 'siem-ui-timeline-pinned-event';
export const pinnedEventSavedObjectMappings: SavedObjectsType['mappings'] = {
properties: {
timelineId: {
type: 'keyword',
},
eventId: {
type: 'keyword',
},
@ -37,4 +35,5 @@ export const pinnedEventType: SavedObjectsType = {
hidden: false,
namespaceType: 'single',
mappings: pinnedEventSavedObjectMappings,
migrations: pinnedEventsMigrations,
};

View file

@ -6,7 +6,7 @@
*/
import { SavedObjectsType } from '../../../../../../../src/core/server';
import { timelinesMigrations } from './migrations';
import { timelinesMigrations } from './migrations/timelines';
export const timelineSavedObjectType = 'siem-ui-timeline';

View file

@ -6,18 +6,28 @@
*/
import expect from '@kbn/expect';
import { SavedTimeline } from '../../../../plugins/security_solution/common/types/timeline';
import { SavedNote } from '../../../../plugins/security_solution/common/types/timeline/note';
import {
noteSavedObjectType,
pinnedEventSavedObjectType,
timelineSavedObjectType,
} from '../../../../plugins/security_solution/server/lib/timeline/saved_object_mappings';
import { TimelineWithoutExternalRefs } from '../../../../plugins/security_solution/common/types/timeline';
import { NoteWithoutExternalRefs } from '../../../../plugins/security_solution/common/types/timeline/note';
import { FtrProviderContext } from '../../ftr_provider_context';
import { getSavedObjectFromES } from './utils';
import { PinnedEventWithoutExternalRefs } from '../../../../plugins/security_solution/common/types/timeline/pinned_event';
interface TimelineWithoutSavedQueryId {
'siem-ui-timeline': Omit<SavedTimeline, 'savedQueryId'>;
[timelineSavedObjectType]: TimelineWithoutExternalRefs;
}
interface NoteWithoutTimelineId {
'siem-ui-timeline-note': Omit<SavedNote, 'timelineId'>;
[noteSavedObjectType]: NoteWithoutExternalRefs;
}
interface PinnedEventWithoutTimelineId {
[pinnedEventSavedObjectType]: PinnedEventWithoutExternalRefs;
}
export default function ({ getService }: FtrProviderContext) {
@ -28,23 +38,22 @@ export default function ({ getService }: FtrProviderContext) {
const es = getService('es');
describe('7.16.0', () => {
before(async () => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
after(async () => {
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
describe('notes timelineId', () => {
before(async () => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
after(async () => {
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
it('removes the timelineId in the saved object', async () => {
const timelines = await getSavedObjectFromES<NoteWithoutTimelineId>(
es,
'siem-ui-timeline-note',
noteSavedObjectType,
{
ids: {
values: [
@ -55,13 +64,13 @@ export default function ({ getService }: FtrProviderContext) {
}
);
expect(
timelines.body.hits.hits[0]._source?.['siem-ui-timeline-note']
).to.not.have.property('timelineId');
expect(timelines.body.hits.hits[0]._source?.[noteSavedObjectType]).to.not.have.property(
'timelineId'
);
expect(
timelines.body.hits.hits[1]._source?.['siem-ui-timeline-note']
).to.not.have.property('timelineId');
expect(timelines.body.hits.hits[1]._source?.[noteSavedObjectType]).to.not.have.property(
'timelineId'
);
});
it('preserves the eventId in the saved object after migration', async () => {
@ -87,30 +96,18 @@ export default function ({ getService }: FtrProviderContext) {
});
describe('savedQueryId', () => {
before(async () => {
await esArchiver.load(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
after(async () => {
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/timelines/7.15.0'
);
});
it('removes the savedQueryId', async () => {
const timelines = await getSavedObjectFromES<TimelineWithoutSavedQueryId>(
es,
'siem-ui-timeline',
timelineSavedObjectType,
{
ids: { values: ['siem-ui-timeline:8dc70950-1012-11ec-9ad3-2d7c6600c0f7'] },
}
);
expect(timelines.body.hits.hits[0]._source?.['siem-ui-timeline']).to.not.have.property(
'savedQueryId'
);
expect(
timelines.body.hits.hits[0]._source?.[timelineSavedObjectType]
).to.not.have.property('savedQueryId');
});
it('preserves the title in the saved object after migration', async () => {
@ -129,6 +126,57 @@ export default function ({ getService }: FtrProviderContext) {
expect(resp.body.data.getOneTimeline.savedQueryId).to.be("It's me");
});
});
describe('pinned events timelineId', () => {
it('removes the timelineId in the saved object', async () => {
const timelines = await getSavedObjectFromES<PinnedEventWithoutTimelineId>(
es,
pinnedEventSavedObjectType,
{
ids: {
values: [
'siem-ui-timeline-pinned-event:7a9a5540-126e-11ec-83d2-db1096c73738',
'siem-ui-timeline-pinned-event:98d919b0-126e-11ec-83d2-db1096c73738',
],
},
}
);
expect(
timelines.body.hits.hits[0]._source?.[pinnedEventSavedObjectType]
).to.not.have.property('timelineId');
expect(
timelines.body.hits.hits[1]._source?.[pinnedEventSavedObjectType]
).to.not.have.property('timelineId');
});
it('preserves the eventId in the saved object after migration', async () => {
const resp = await supertest
.get('/api/timeline')
.query({ id: '6484cc90-126e-11ec-83d2-db1096c73738' });
expect(resp.body.data.getOneTimeline.pinnedEventsSaveObject[0].eventId).to.be(
'DNo00XsBEVtyvU-8LGNe'
);
expect(resp.body.data.getOneTimeline.pinnedEventsSaveObject[1].eventId).to.be(
'Edo00XsBEVtyvU-8LGNe'
);
});
it('returns the timelineId in the response', async () => {
const resp = await supertest
.get('/api/timeline')
.query({ id: '6484cc90-126e-11ec-83d2-db1096c73738' });
expect(resp.body.data.getOneTimeline.pinnedEventsSaveObject[0].timelineId).to.be(
'6484cc90-126e-11ec-83d2-db1096c73738'
);
expect(resp.body.data.getOneTimeline.pinnedEventsSaveObject[1].timelineId).to.be(
'6484cc90-126e-11ec-83d2-db1096c73738'
);
});
});
});
});
}