Modify saved object export API to support exporting nested dependencies (#34225)

* Modify saved object export API to be able to export nested dependencies

* Apply verbal feedback

* Apply PR feedback
This commit is contained in:
Mike Côté 2019-04-03 15:51:07 -04:00 committed by GitHub
parent b94c68abde
commit 1e56a48fc7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 861 additions and 44 deletions

View file

@ -16,6 +16,8 @@ Note: You cannot access this endpoint via the Console in Kibana.
(array|string) The saved object type(s) that the export should be limited to
`objects` (optional)::
(array) A list of objects to export
`includeReferencesDeep`::
(boolean) This will make the exported objects include all the referenced objects needed
Note: At least `type` or `objects` must be passed in.

View file

@ -221,6 +221,95 @@ Array [
`);
});
test('includes nested dependencies when passed in', async () => {
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '2',
type: 'search',
references: [
{
type: 'index-pattern',
id: '1',
},
],
},
],
});
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
const response = await getSortedObjectsForExport({
exportSizeLimit: 10000,
savedObjectsClient,
types: ['index-pattern', 'search'],
objects: [
{
type: 'search',
id: '2',
},
],
includeReferencesDeep: true,
});
expect(response).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"id": "2",
"references": Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "2",
"type": "search",
},
],
],
Array [
Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('export selected objects throws error when exceeding exportSizeLimit', async () => {
const exportOpts = {
exportSizeLimit: 1,

View file

@ -18,7 +18,8 @@
*/
import Boom from 'boom';
import { SavedObject, SavedObjectsClient } from '../service/saved_objects_client';
import { SavedObjectsClient } from '../service/saved_objects_client';
import { injectNestedDependencies } from './inject_nested_depdendencies';
import { sortObjects } from './sort_objects';
interface ObjectToExport {
@ -31,21 +32,26 @@ interface ExportObjectsOptions {
objects?: ObjectToExport[];
savedObjectsClient: SavedObjectsClient;
exportSizeLimit: number;
includeReferencesDeep?: boolean;
}
export async function getSortedObjectsForExport({
types,
async function fetchObjectsToExport({
objects,
savedObjectsClient,
types,
exportSizeLimit,
}: ExportObjectsOptions) {
let objectsToExport: SavedObject[] = [];
savedObjectsClient,
}: {
objects?: ObjectToExport[];
types?: string[];
exportSizeLimit: number;
savedObjectsClient: SavedObjectsClient;
}) {
if (objects) {
if (objects.length > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
}
({ saved_objects: objectsToExport } = await savedObjectsClient.bulkGet(objects));
const erroredObjects = objectsToExport.filter(obj => !!obj.error);
const bulkGetResult = await savedObjectsClient.bulkGet(objects);
const erroredObjects = bulkGetResult.saved_objects.filter(obj => !!obj.error);
if (erroredObjects.length) {
const err = Boom.badRequest();
err.output.payload.attributes = {
@ -53,17 +59,36 @@ export async function getSortedObjectsForExport({
};
throw err;
}
} else {
const findResponse = await savedObjectsClient.find({
type: types,
sortField: '_id',
sortOrder: 'asc',
perPage: exportSizeLimit,
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
}
({ saved_objects: objectsToExport } = findResponse);
return bulkGetResult.saved_objects;
}
return sortObjects(objectsToExport);
const findResponse = await savedObjectsClient.find({
type: types,
sortField: '_id',
sortOrder: 'asc',
perPage: exportSizeLimit,
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
}
return findResponse.saved_objects;
}
export async function getSortedObjectsForExport({
types,
objects,
savedObjectsClient,
exportSizeLimit,
includeReferencesDeep = false,
}: ExportObjectsOptions) {
const objectsToExport = await fetchObjectsToExport({
types,
objects,
savedObjectsClient,
exportSizeLimit,
});
return sortObjects(
includeReferencesDeep
? await injectNestedDependencies(objectsToExport, savedObjectsClient)
: objectsToExport
);
}

View file

@ -0,0 +1,556 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { SavedObject } from '../service/saved_objects_client';
import {
getObjectReferencesToFetch,
injectNestedDependencies,
} from './inject_nested_depdendencies';
describe('getObjectReferencesToFetch()', () => {
test('works with no saved objects', () => {
const map = new Map<string, SavedObject>();
const result = getObjectReferencesToFetch(map);
expect(result).toEqual([]);
});
test('excludes already fetched objects', () => {
const map = new Map<string, SavedObject>();
map.set('index-pattern:1', {
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
});
map.set('visualization:2', {
id: '2',
type: 'visualization',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
});
const result = getObjectReferencesToFetch(map);
expect(result).toEqual([]);
});
test('returns objects that are missing', () => {
const map = new Map<string, SavedObject>();
map.set('visualization:2', {
id: '2',
type: 'visualization',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
});
const result = getObjectReferencesToFetch(map);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"id": "1",
"type": "index-pattern",
},
]
`);
});
test(`doesn't deal with circular dependencies`, () => {
const map = new Map<string, SavedObject>();
map.set('index-pattern:1', {
id: '1',
type: 'index-pattern',
attributes: {},
references: [
{
name: 'ref_0',
type: 'visualization',
id: '2',
},
],
});
map.set('visualization:2', {
id: '2',
type: 'visualization',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
});
const result = getObjectReferencesToFetch(map);
expect(result).toEqual([]);
});
});
describe('injectNestedDependencies', () => {
const savedObjectsClient = {
errors: {} as any,
find: jest.fn(),
bulkGet: jest.fn(),
create: jest.fn(),
bulkCreate: jest.fn(),
delete: jest.fn(),
get: jest.fn(),
update: jest.fn(),
};
afterEach(() => {
jest.resetAllMocks();
});
test(`doesn't fetch when no dependencies are missing`, async () => {
const savedObjects = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
];
const result = await injectNestedDependencies(savedObjects, savedObjectsClient);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
]
`);
});
test(`doesn't fetch references that are already fetched`, async () => {
const savedObjects = [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
];
const result = await injectNestedDependencies(savedObjects, savedObjectsClient);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
});
test('fetches dependencies at least one level deep', async () => {
const savedObjects = [
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
];
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
],
});
const result = await injectNestedDependencies(savedObjects, savedObjectsClient);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "2",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "search",
},
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('fetches dependencies multiple levels deep', async () => {
const savedObjects = [
{
id: '5',
type: 'dashboard',
attributes: {},
references: [
{
name: 'panel_0',
type: 'visualization',
id: '4',
},
{
name: 'panel_1',
type: 'visualization',
id: '3',
},
],
},
];
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '4',
type: 'visualization',
attributes: {},
references: [
{
name: 'ref_0',
type: 'search',
id: '2',
},
],
},
{
id: '3',
type: 'visualization',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
],
});
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
],
});
const result = await injectNestedDependencies(savedObjects, savedObjectsClient);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "5",
"references": Array [
Object {
"id": "4",
"name": "panel_0",
"type": "visualization",
},
Object {
"id": "3",
"name": "panel_1",
"type": "visualization",
},
],
"type": "dashboard",
},
Object {
"attributes": Object {},
"id": "4",
"references": Array [
Object {
"id": "2",
"name": "ref_0",
"type": "search",
},
],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "3",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "visualization",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "search",
},
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "4",
"type": "visualization",
},
Object {
"id": "3",
"type": "visualization",
},
],
],
Array [
Array [
Object {
"id": "2",
"type": "search",
},
Object {
"id": "1",
"type": "index-pattern",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
test('throws error when bulkGet returns an error', async () => {
const savedObjects = [
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
];
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '1',
type: 'index-pattern',
error: {
statusCode: 404,
message: 'Not found',
},
},
],
});
await expect(
injectNestedDependencies(savedObjects, savedObjectsClient)
).rejects.toThrowErrorMatchingInlineSnapshot(`"Bad Request"`);
});
test(`doesn't deal with circular dependencies`, async () => {
const savedObjects = [
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
];
savedObjectsClient.bulkGet.mockResolvedValueOnce({
saved_objects: [
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [
{
name: 'ref_0',
type: 'search',
id: '2',
},
],
},
],
});
const result = await injectNestedDependencies(savedObjects, savedObjectsClient);
expect(result).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "2",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "search",
},
Object {
"attributes": Object {},
"id": "1",
"references": Array [
Object {
"id": "2",
"name": "ref_0",
"type": "search",
},
],
"type": "index-pattern",
},
]
`);
expect(savedObjectsClient.bulkGet).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Array [
Object {
"id": "1",
"type": "index-pattern",
},
],
],
],
"results": Array [
Object {
"type": "return",
"value": Promise {},
},
],
}
`);
});
});

View file

@ -0,0 +1,62 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import Boom from 'boom';
import { SavedObject, SavedObjectsClient } from '../service/saved_objects_client';
export function getObjectReferencesToFetch(savedObjectsMap: Map<string, SavedObject>) {
const objectsToFetch = new Map<string, { type: string; id: string }>();
for (const savedObject of savedObjectsMap.values()) {
for (const { type, id } of savedObject.references || []) {
if (!savedObjectsMap.has(`${type}:${id}`)) {
objectsToFetch.set(`${type}:${id}`, { type, id });
}
}
}
return [...objectsToFetch.values()];
}
export async function injectNestedDependencies(
savedObjects: SavedObject[],
savedObjectsClient: SavedObjectsClient
) {
const savedObjectsMap = new Map<string, SavedObject>();
for (const savedObject of savedObjects) {
savedObjectsMap.set(`${savedObject.type}:${savedObject.id}`, savedObject);
}
let objectsToFetch = getObjectReferencesToFetch(savedObjectsMap);
while (objectsToFetch.length > 0) {
const bulkGetResponse = await savedObjectsClient.bulkGet(objectsToFetch);
// Check for errors
const erroredObjects = bulkGetResponse.saved_objects.filter(obj => !!obj.error);
if (erroredObjects.length) {
const err = Boom.badRequest();
err.output.payload.attributes = {
objects: erroredObjects,
};
throw err;
}
// Push to array result
for (const savedObject of bulkGetResponse.saved_objects) {
savedObjectsMap.set(`${savedObject.type}:${savedObject.id}`, savedObject);
}
objectsToFetch = getObjectReferencesToFetch(savedObjectsMap);
}
return [...savedObjectsMap.values()];
}

View file

@ -18,9 +18,16 @@
*/
import Hapi from 'hapi';
import * as exportMock from '../export';
import { createMockServer } from './_mock_server';
import { createExportRoute } from './export';
const getSortedObjectsForExport = exportMock.getSortedObjectsForExport as jest.Mock;
jest.mock('../export', () => ({
getSortedObjectsForExport: jest.fn(),
}));
describe('POST /api/saved_objects/_export', () => {
let server: Hapi.Server;
const savedObjectsClient = {
@ -49,13 +56,7 @@ describe('POST /api/saved_objects/_export', () => {
});
afterEach(() => {
savedObjectsClient.bulkCreate.mockReset();
savedObjectsClient.bulkGet.mockReset();
savedObjectsClient.create.mockReset();
savedObjectsClient.delete.mockReset();
savedObjectsClient.find.mockReset();
savedObjectsClient.get.mockReset();
savedObjectsClient.update.mockReset();
jest.resetAllMocks();
});
test('formats successful response', async () => {
@ -63,19 +64,30 @@ describe('POST /api/saved_objects/_export', () => {
method: 'POST',
url: '/api/saved_objects/_export',
payload: {
type: 'index-pattern',
type: 'search',
includeReferencesDeep: true,
},
};
savedObjectsClient.find.mockResolvedValueOnce({
total: 1,
saved_objects: [
{
id: '1',
type: 'index-pattern',
references: [],
},
],
});
getSortedObjectsForExport.mockResolvedValueOnce([
{
id: '1',
type: 'index-pattern',
attributes: {},
references: [],
},
{
id: '2',
type: 'search',
attributes: {},
references: [
{
name: 'ref_0',
type: 'index-pattern',
id: '1',
},
],
},
]);
const { payload, statusCode, headers } = await server.inject(request);
const objects = payload.split('\n').map(row => JSON.parse(row));
@ -86,22 +98,45 @@ describe('POST /api/saved_objects/_export', () => {
expect(objects).toMatchInlineSnapshot(`
Array [
Object {
"attributes": Object {},
"id": "1",
"references": Array [],
"type": "index-pattern",
},
Object {
"attributes": Object {},
"id": "2",
"references": Array [
Object {
"id": "1",
"name": "ref_0",
"type": "index-pattern",
},
],
"type": "search",
},
]
`);
expect(savedObjectsClient.find).toMatchInlineSnapshot(`
expect(getSortedObjectsForExport).toMatchInlineSnapshot(`
[MockFunction] {
"calls": Array [
Array [
Object {
"perPage": 10000,
"sortField": "_id",
"sortOrder": "asc",
"type": Array [
"index-pattern",
"exportSizeLimit": 10000,
"includeReferencesDeep": true,
"objects": undefined,
"savedObjectsClient": Object {
"bulkCreate": [MockFunction],
"bulkGet": [MockFunction],
"create": [MockFunction],
"delete": [MockFunction],
"errors": Object {},
"find": [MockFunction],
"get": [MockFunction],
"update": [MockFunction],
},
"types": Array [
"search",
],
},
],

View file

@ -36,6 +36,7 @@ interface ExportRequest extends Hapi.Request {
type: string;
id: string;
}>;
includeReferencesDeep: boolean;
};
}
@ -60,6 +61,7 @@ export const createExportRoute = (prereqs: Prerequisites, server: Hapi.Server) =
})
.max(server.config().get('savedObjects.maxImportExportSize'))
.optional(),
includeReferencesDeep: Joi.boolean().default(false),
})
.xor('type', 'objects')
.default(),
@ -71,6 +73,7 @@ export const createExportRoute = (prereqs: Prerequisites, server: Hapi.Server) =
types: request.payload.type,
objects: request.payload.objects,
exportSizeLimit: server.config().get('savedObjects.maxImportExportSize'),
includeReferencesDeep: request.payload.includeReferencesDeep,
});
return h
.response(docsToExport.map(doc => stringify(doc)).join('\n'))

View file

@ -90,6 +90,51 @@ export default function ({ getService }) {
});
});
it('should support including dependencies when exporting selected objects', async () => {
await supertest
.post('/api/saved_objects/_export')
.send({
includeReferencesDeep: true,
objects: [
{
type: 'dashboard',
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
},
],
})
.expect(200)
.then((resp) => {
const objects = resp.text.split('\n').map(JSON.parse);
expect(objects).to.have.length(3);
expect(objects[0]).to.have.property('id', '91200a00-9efd-11e7-acb3-3dab96693fab');
expect(objects[0]).to.have.property('type', 'index-pattern');
expect(objects[1]).to.have.property('id', 'dd7caf20-9efd-11e7-acb3-3dab96693fab');
expect(objects[1]).to.have.property('type', 'visualization');
expect(objects[2]).to.have.property('id', 'be3733a0-9efe-11e7-acb3-3dab96693fab');
expect(objects[2]).to.have.property('type', 'dashboard');
});
});
it('should support including dependencies when exporting by type', async () => {
await supertest
.post('/api/saved_objects/_export')
.send({
includeReferencesDeep: true,
type: ['dashboard'],
})
.expect(200)
.then((resp) => {
const objects = resp.text.split('\n').map(JSON.parse);
expect(objects).to.have.length(3);
expect(objects[0]).to.have.property('id', '91200a00-9efd-11e7-acb3-3dab96693fab');
expect(objects[0]).to.have.property('type', 'index-pattern');
expect(objects[1]).to.have.property('id', 'dd7caf20-9efd-11e7-acb3-3dab96693fab');
expect(objects[1]).to.have.property('type', 'visualization');
expect(objects[2]).to.have.property('id', 'be3733a0-9efe-11e7-acb3-3dab96693fab');
expect(objects[2]).to.have.property('type', 'dashboard');
});
});
it(`should throw error when object doesn't exist`, async () => {
await supertest
.post('/api/saved_objects/_export')