mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[streams][content pack] check decompressed file size (#220486)
## Summary Verifies that the uncompressed size of a single object included in the content pack do not exceed 1MB as a safety measure. The 1MB limit is based on MAX(`elastic/integrations` dashboard size) * 2. The change also includes a constraint on the archive internal structure and expects a single root directory as entry. Example: ``` content_pack-1.0.0/manifest.yml content_pack-1.0.0/kibana/dashboard/123-..json content_pack-1.0.0/kibana/index-pattern/123-..json ... ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Kevin Lacabane
GitHub
parent
f635e2a3b0
commit
2e78b00114
12 changed files with 625 additions and 156 deletions
File diff suppressed because one or more lines are too long
|
|
@ -1,62 +0,0 @@
|
|||
{
|
||||
"attributes": {
|
||||
"allowHidden": false,
|
||||
"fieldAttrs": "{}",
|
||||
"fieldFormatMap": "{}",
|
||||
"fields": "[]",
|
||||
"name": "logs",
|
||||
"runtimeFieldMap": "{}",
|
||||
"sourceFilters": "[]",
|
||||
"timeFieldName": "@timestamp",
|
||||
"title": "logs"
|
||||
},
|
||||
"coreMigrationVersion": "8.8.0",
|
||||
"created_at": "2025-04-09T17:56:37.310Z",
|
||||
"created_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
|
||||
"id": "a7e3b27d-c804-474a-b3cd-141b9b07fd04",
|
||||
"managed": false,
|
||||
"references": [],
|
||||
"type": "index-pattern",
|
||||
"typeMigrationVersion": "8.0.0",
|
||||
"updated_at": "2025-04-09T17:56:37.310Z",
|
||||
"updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
|
||||
"version": "WzExLDFd"
|
||||
}
|
||||
|
||||
{
|
||||
"attributes": {
|
||||
"controlGroupInput": {
|
||||
"chainingSystem": "HIERARCHICAL",
|
||||
"controlStyle": "oneLine",
|
||||
"ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}",
|
||||
"panelsJSON": "{}",
|
||||
"showApplySelections": false
|
||||
},
|
||||
"description": "",
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"kuery\"}}"
|
||||
},
|
||||
"optionsJSON": "{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}",
|
||||
"panelsJSON": "[{\"type\":\"lens\",\"embeddableConfig\":{\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"filters\":[],\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"a7e3b27d-c804-474a-b3cd-141b9b07fd04\",\"name\":\"indexpattern-datasource-layer-83bebabf-6399-45c3-962b-2b33aaea84fd\"}],\"state\":{\"visualization\":{\"title\":\"Empty XY chart\",\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"83bebabf-6399-45c3-962b-2b33aaea84fd\",\"accessors\":[\"59fe648e-fc10-4301-92ec-c8f0db37b9cd\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"default\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"111134dc-9b12-4506-a22a-3c214a61101c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"83bebabf-6399-45c3-962b-2b33aaea84fd\":{\"columns\":{\"111134dc-9b12-4506-a22a-3c214a61101c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"59fe648e-fc10-4301-92ec-c8f0db37b9cd\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"111134dc-9b12-4506-a22a-3c214a61101c\",\"59fe648e-fc10-4301-92ec-c8f0db37b9cd\"],\"sampling\":1,\"ignoreGlobalFilters\":false,\"incompleteColumns\":{}}}},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}}},\"panelIndex\":\"1b439b9a-c7b9-4e0a-a44f-3a7d1c19dec0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1b439b9a-c7b9-4e0a-a44f-3a7d1c19dec0\"}},{\"type\":\"lens\",\"embeddableConfig\":{\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"filters\":[],\"query\":{\"esql\":\"FROM logs | LIMIT 10\"},\"attributes\":{\"title\":\"Bar vertical stacked\",\"references\":[],\"state\":{\"datasourceStates\":{\"textBased\":{\"layers\":{\"4ea455e2-6c30-4a4b-8789-7d6bb25ede5f\":{\"index\":\"07aa17244c8e39c6db4cf18d211fde421969410cf83140590f66f41f734daea5\",\"query\":{\"esql\":\"FROM logs | STATS count=count(*) by message | sort count desc | LIMIT 10\"},\"columns\":[{\"columnId\":\"count\",\"fieldName\":\"count\",\"label\":\"count\",\"customLabel\":false,\"meta\":{\"type\":\"number\",\"esType\":\"long\"},\"inMetricDimension\":true},{\"columnId\":\"message\",\"fieldName\":\"message\",\"label\":\"message\",\"customLabel\":false,\"meta\":{\"type\":\"string\",\"esType\":\"text\"}}],\"timeField\":\"@timestamp\"}},\"indexPatternRefs\":[{\"id\":\"07aa17244c8e39c6db4cf18d211fde421969410cf83140590f66f41f734daea5\",\"title\":\"logs\",\"timeField\":\"@timestamp\"}]}},\"filters\":[],\"query\":{\"esql\":\"FROM logs | STATS count=count(*) by message | sort count desc | LIMIT 10\"},\"visualization\":{\"layerId\":\"4ea455e2-6c30-4a4b-8789-7d6bb25ede5f\",\"layerType\":\"data\",\"columns\":[{\"columnId\":\"count\"},{\"columnId\":\"message\"}]},\"adHocDataViews\":{\"07aa17244c8e39c6db4cf18d211fde421969410cf83140590f66f41f734daea5\":{\"id\":\"07aa17244c8e39c6db4cf18d211fde421969410cf83140590f66f41f734daea5\",\"title\":\"logs\",\"timeFieldName\":\"@timestamp\",\"sourceFilters\":[],\"type\":\"esql\",\"fieldFormats\":{},\"runtimeFieldMap\":{},\"allowNoIndex\":false,\"name\":\"logs\",\"allowHidden\":false}},\"needsRefresh\":false},\"visualizationType\":\"lnsDatatable\"}},\"panelIndex\":\"5dcf8705-378a-4b1d-a9da-9b2025cdf7fe\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"5dcf8705-378a-4b1d-a9da-9b2025cdf7fe\"}}]",
|
||||
"timeRestore": false,
|
||||
"title": "Logs count and top 10 messages",
|
||||
"version": 3
|
||||
},
|
||||
"coreMigrationVersion": "8.8.0",
|
||||
"created_at": "2025-04-09T17:58:56.268Z",
|
||||
"created_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
|
||||
"id": "c22ba8ed-fd4b-4864-a98c-3cba1d11cfb2",
|
||||
"managed": false,
|
||||
"references": [
|
||||
{
|
||||
"id": "a7e3b27d-c804-474a-b3cd-141b9b07fd04",
|
||||
"name": "1b439b9a-c7b9-4e0a-a44f-3a7d1c19dec0:indexpattern-datasource-layer-83bebabf-6399-45c3-962b-2b33aaea84fd",
|
||||
"type": "index-pattern"
|
||||
}
|
||||
],
|
||||
"type": "dashboard",
|
||||
"typeMigrationVersion": "10.2.0",
|
||||
"updated_at": "2025-04-09T17:58:56.268Z",
|
||||
"updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0",
|
||||
"version": "WzE1LDFd"
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ export const isIndexPlaceholder = (index: string) => index.startsWith(INDEX_PLAC
|
|||
|
||||
interface TraverseOptions {
|
||||
esqlQuery(query: string): string;
|
||||
indexPattern<T extends { name?: string; title?: string }>(pattern: T): T;
|
||||
indexPattern<T extends { title?: string }>(pattern: T): T;
|
||||
field<T extends GenericIndexPatternColumn | TextBasedLayerColumn>(field: T): T;
|
||||
}
|
||||
|
||||
|
|
@ -82,10 +82,17 @@ export function replaceIndexPatterns(
|
|||
.map((index) => patternReplacements[index] ?? index)
|
||||
.join(',');
|
||||
|
||||
// data view references may be named after the index patterns they represent,
|
||||
// so we attempt to replace index patterns to avoid wrongly named data views
|
||||
const updatedName = pattern.name
|
||||
?.split(',')
|
||||
.map((index) => patternReplacements[index] ?? index)
|
||||
.join(',');
|
||||
|
||||
return {
|
||||
...pattern,
|
||||
name: updatedPattern,
|
||||
title: updatedPattern,
|
||||
name: updatedName,
|
||||
};
|
||||
},
|
||||
field(field: any) {
|
||||
|
|
|
|||
|
|
@ -5,12 +5,41 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import { z } from '@kbn/zod';
|
||||
import { ContentPackSavedObject } from './saved_object';
|
||||
import { ContentPackSavedObject, SUPPORTED_SAVED_OBJECT_TYPE } from './saved_object';
|
||||
|
||||
export * from './api';
|
||||
export * from './saved_object';
|
||||
|
||||
export const SUPPORTED_ENTRY_TYPE: Record<ContentPackEntry['type'], string> = {
|
||||
...SUPPORTED_SAVED_OBJECT_TYPE,
|
||||
};
|
||||
|
||||
export type SupportedEntryType = keyof typeof SUPPORTED_ENTRY_TYPE;
|
||||
|
||||
export const isSupportedFile = (rootDir: string, filepath: string) => {
|
||||
return Object.values(SUPPORTED_ENTRY_TYPE).some(
|
||||
(dir) => path.dirname(filepath) === path.join(rootDir, 'kibana', dir)
|
||||
);
|
||||
};
|
||||
|
||||
export const getEntryTypeByFile = (rootDir: string, filepath: string): SupportedEntryType => {
|
||||
const entry = Object.entries(SUPPORTED_ENTRY_TYPE).find(
|
||||
([t, dir]) => path.dirname(filepath) === path.join(rootDir, 'kibana', dir)
|
||||
) as [SupportedEntryType, string] | undefined;
|
||||
|
||||
if (!entry) {
|
||||
throw new Error(`Unknown entry type for filepath [${filepath}]`);
|
||||
}
|
||||
|
||||
return entry[0];
|
||||
};
|
||||
|
||||
export const isSupportedEntryType = (type: string) => {
|
||||
return type in SUPPORTED_ENTRY_TYPE;
|
||||
};
|
||||
|
||||
export interface ContentPackManifest {
|
||||
name: string;
|
||||
description: string;
|
||||
|
|
@ -23,10 +52,6 @@ export const contentPackManifestSchema: z.Schema<ContentPackManifest> = z.object
|
|||
version: z.string(),
|
||||
});
|
||||
|
||||
export function isContentPackSavedObject(entry: ContentPackEntry): entry is ContentPackSavedObject {
|
||||
return ['dashboard', 'index-pattern'].includes(entry.type);
|
||||
}
|
||||
|
||||
export type ContentPackEntry = ContentPackSavedObject;
|
||||
|
||||
export interface ContentPack extends ContentPackManifest {
|
||||
|
|
|
|||
|
|
@ -10,27 +10,34 @@ import type { SavedObject } from '@kbn/core/server';
|
|||
import type { DashboardAttributes } from '@kbn/dashboard-plugin/common/content_management/v2';
|
||||
import type { DataViewSavedObjectAttrs } from '@kbn/data-views-plugin/common/data_views';
|
||||
import type { LensAttributes } from '@kbn/lens-embeddable-utils';
|
||||
import type { ContentPackEntry } from '.';
|
||||
|
||||
export const SUPPORTED_SAVED_OBJECT_TYPE: Record<ContentPackSavedObject['type'], string> = {
|
||||
dashboard: 'dashboard',
|
||||
'index-pattern': 'index_pattern',
|
||||
lens: 'lens',
|
||||
};
|
||||
|
||||
export const SUPPORTED_SAVED_OBJECT_TYPES = [
|
||||
{ type: 'dashboard', dir: 'dashboard' },
|
||||
{ type: 'index-pattern', dir: 'index_pattern' },
|
||||
{ type: 'lens', dir: 'lens' },
|
||||
];
|
||||
export const isSupportedSavedObjectType = (
|
||||
entry: SavedObject<unknown>
|
||||
entry: SavedObject<unknown> | ContentPackEntry
|
||||
): entry is ContentPackSavedObject => {
|
||||
return SUPPORTED_SAVED_OBJECT_TYPES.some(({ type }) => type === entry.type);
|
||||
return entry.type in SUPPORTED_SAVED_OBJECT_TYPE;
|
||||
};
|
||||
|
||||
export const isSupportedSavedObjectFile = (filepath: string) => {
|
||||
return SUPPORTED_SAVED_OBJECT_TYPES.some(
|
||||
({ dir }) => path.dirname(filepath) === path.join('kibana', dir)
|
||||
);
|
||||
export const isDashboardFile = (rootDir: string, filepath: string) => {
|
||||
const subDir = SUPPORTED_SAVED_OBJECT_TYPE.dashboard;
|
||||
return path.dirname(filepath) === path.join(rootDir, 'kibana', subDir);
|
||||
};
|
||||
|
||||
type ContentPackDashboard = SavedObject<DashboardAttributes> & { type: 'dashboard' };
|
||||
type ContentPackDataView = SavedObject<DataViewSavedObjectAttrs> & { type: 'index-pattern' };
|
||||
type ContentPackLens = SavedObject<LensAttributes> & { type: 'lens' };
|
||||
export const isSupportedReferenceType = (type: string) => {
|
||||
const referenceTypes: Array<ContentPackSavedObject['type']> = ['index-pattern', 'lens'];
|
||||
return referenceTypes.some((refType) => refType === type);
|
||||
};
|
||||
|
||||
export type ContentPackDashboard = SavedObject<DashboardAttributes> & { type: 'dashboard' };
|
||||
export type ContentPackDataView = SavedObject<DataViewSavedObjectAttrs> & { type: 'index-pattern' };
|
||||
export type ContentPackLens = SavedObject<LensAttributes> & { type: 'lens' };
|
||||
|
||||
export type ContentPackSavedObject = ContentPackDashboard | ContentPackDataView | ContentPackLens;
|
||||
|
||||
export interface SavedObjectLink {
|
||||
|
|
|
|||
|
|
@ -8,17 +8,24 @@
|
|||
import YAML from 'yaml';
|
||||
import {
|
||||
ContentPack,
|
||||
ContentPackDashboard,
|
||||
ContentPackEntry,
|
||||
ContentPackManifest,
|
||||
SUPPORTED_SAVED_OBJECT_TYPES,
|
||||
ContentPackSavedObject,
|
||||
SUPPORTED_SAVED_OBJECT_TYPE,
|
||||
contentPackManifestSchema,
|
||||
isSupportedSavedObjectFile,
|
||||
isSupportedSavedObjectType,
|
||||
getEntryTypeByFile,
|
||||
isSupportedEntryType,
|
||||
isSupportedFile,
|
||||
isSupportedReferenceType,
|
||||
} from '@kbn/content-packs-schema';
|
||||
import AdmZip from 'adm-zip';
|
||||
import path from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import { pick } from 'lodash';
|
||||
import { compact, pick, uniqBy } from 'lodash';
|
||||
import { InvalidContentPackError } from './error';
|
||||
|
||||
const ARCHIVE_ENTRY_MAX_SIZE_BYTES = 1 * 1024 * 1024; // 1MB
|
||||
|
||||
export async function parseArchive(archive: Readable): Promise<ContentPack> {
|
||||
const zip: AdmZip = await new Promise((resolve, reject) => {
|
||||
|
|
@ -28,50 +35,42 @@ export async function parseArchive(archive: Readable): Promise<ContentPack> {
|
|||
try {
|
||||
resolve(new AdmZip(Buffer.concat(bufs)));
|
||||
} catch (err) {
|
||||
reject(new Error('Invalid content pack format'));
|
||||
reject(new InvalidContentPackError('Invalid content pack format'));
|
||||
}
|
||||
});
|
||||
archive.on('error', (error) => reject(error));
|
||||
});
|
||||
|
||||
let manifestEntry: AdmZip.IZipEntry | undefined;
|
||||
const entries: ContentPackEntry[] = [];
|
||||
zip.forEach((entry) => {
|
||||
const filepath = path.join(...entry.entryName.split(path.sep).slice(1));
|
||||
if (filepath === 'manifest.yml') {
|
||||
manifestEntry = entry;
|
||||
} else if (isSupportedSavedObjectFile(filepath)) {
|
||||
entries.push(JSON.parse(entry.getData().toString()));
|
||||
}
|
||||
});
|
||||
const rootDir = getRootDir(zip.getEntries());
|
||||
const manifest = await extractManifest(rootDir, zip);
|
||||
const entries = await extractEntries(rootDir, zip);
|
||||
|
||||
if (!manifestEntry) {
|
||||
throw new Error('Missing content pack manifest');
|
||||
}
|
||||
|
||||
const { data: manifestData, success } = contentPackManifestSchema.safeParse(
|
||||
YAML.parse(manifestEntry.getData().toString())
|
||||
);
|
||||
if (!success) {
|
||||
throw new Error('Invalid content pack manifest format');
|
||||
}
|
||||
|
||||
return { ...manifestData, entries };
|
||||
return { ...manifest, entries };
|
||||
}
|
||||
|
||||
export async function generateArchive(manifest: ContentPackManifest, objects: ContentPackEntry[]) {
|
||||
const zip = new AdmZip();
|
||||
const rootDir = `${manifest.name}-${manifest.version}`;
|
||||
|
||||
objects.forEach((object: ContentPackEntry) => {
|
||||
if (isSupportedSavedObjectType(object)) {
|
||||
const dir = SUPPORTED_SAVED_OBJECT_TYPES.find(({ type }) => type === object.type)!.dir;
|
||||
zip.addFile(
|
||||
path.join(rootDir, 'kibana', dir, `${object.id}.json`),
|
||||
Buffer.from(JSON.stringify(object, null, 2))
|
||||
);
|
||||
}
|
||||
});
|
||||
objects
|
||||
.filter((object) => isSupportedEntryType(object.type))
|
||||
.forEach((object: ContentPackEntry) => {
|
||||
const type = object.type;
|
||||
switch (type) {
|
||||
case 'dashboard':
|
||||
case 'index-pattern':
|
||||
case 'lens':
|
||||
const subDir = SUPPORTED_SAVED_OBJECT_TYPE[object.type];
|
||||
zip.addFile(
|
||||
path.join(rootDir, 'kibana', subDir, `${object.id}.json`),
|
||||
Buffer.from(JSON.stringify(object, null, 2))
|
||||
);
|
||||
return;
|
||||
|
||||
default:
|
||||
missingEntryTypeImpl(type);
|
||||
}
|
||||
});
|
||||
|
||||
zip.addFile(
|
||||
path.join(rootDir, 'manifest.yml'),
|
||||
|
|
@ -80,3 +79,126 @@ export async function generateArchive(manifest: ContentPackManifest, objects: Co
|
|||
|
||||
return zip.toBufferPromise();
|
||||
}
|
||||
|
||||
async function readEntry(entry: AdmZip.IZipEntry): Promise<Buffer> {
|
||||
const buf = await new Promise<Buffer>((resolve, reject) => {
|
||||
entry.getDataAsync((data, err) => {
|
||||
if (err) return reject(new Error(err));
|
||||
resolve(data);
|
||||
});
|
||||
});
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
async function extractManifest(rootDir: string, zip: AdmZip): Promise<ContentPackManifest> {
|
||||
const manifestPath = `${rootDir}/manifest.yml`;
|
||||
const entry = zip.getEntry(manifestPath);
|
||||
if (!entry) {
|
||||
throw new InvalidContentPackError(`Expected manifest at [${manifestPath}]`);
|
||||
}
|
||||
|
||||
assertUncompressedSize(entry);
|
||||
|
||||
const { data: manifest, success } = contentPackManifestSchema.safeParse(
|
||||
YAML.parse((await readEntry(entry)).toString())
|
||||
);
|
||||
if (!success) {
|
||||
throw new InvalidContentPackError('Invalid content pack manifest format');
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
async function extractEntries(rootDir: string, zip: AdmZip): Promise<ContentPackEntry[]> {
|
||||
const supportedEntries = zip
|
||||
.getEntries()
|
||||
.filter((entry) => isSupportedFile(rootDir, entry.entryName));
|
||||
|
||||
supportedEntries.forEach((entry) => assertUncompressedSize(entry));
|
||||
|
||||
const entries = await Promise.all(
|
||||
supportedEntries.map((entry) => {
|
||||
const type = getEntryTypeByFile(rootDir, entry.entryName);
|
||||
switch (type) {
|
||||
case 'lens':
|
||||
case 'index-pattern':
|
||||
// these are handled by their parent dashboard
|
||||
return [];
|
||||
|
||||
case 'dashboard':
|
||||
return resolveDashboard(rootDir, zip, entry);
|
||||
|
||||
default:
|
||||
missingEntryTypeImpl(type);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
return entries.flat();
|
||||
}
|
||||
|
||||
async function resolveDashboard(
|
||||
rootDir: string,
|
||||
zip: AdmZip,
|
||||
dashboardEntry: AdmZip.IZipEntry
|
||||
): Promise<ContentPackSavedObject[]> {
|
||||
const dashboard = JSON.parse(
|
||||
(await readEntry(dashboardEntry)).toString()
|
||||
) as ContentPackDashboard;
|
||||
|
||||
const uniqReferences = uniqBy(dashboard.references, (ref) => ref.id);
|
||||
if (uniqReferences.some(({ type }) => !isSupportedReferenceType(type))) {
|
||||
throw new InvalidContentPackError(
|
||||
`Dashboard [${
|
||||
dashboard.id
|
||||
}] references saved object types not supported by content packs: ${uniqReferences.filter(
|
||||
({ type }) => !isSupportedReferenceType(type)
|
||||
)}`
|
||||
);
|
||||
}
|
||||
|
||||
const includedReferences = compact(
|
||||
(uniqReferences as Array<{ type: ContentPackSavedObject['type']; id: string }>).map((ref) =>
|
||||
zip.getEntry(
|
||||
path.join(rootDir, 'kibana', SUPPORTED_SAVED_OBJECT_TYPE[ref.type], `${ref.id}.json`)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
const resolvedReferences = await Promise.all(
|
||||
includedReferences.map(
|
||||
async (entry) => JSON.parse((await readEntry(entry)).toString()) as ContentPackSavedObject
|
||||
)
|
||||
);
|
||||
|
||||
return [dashboard, ...resolvedReferences];
|
||||
}
|
||||
|
||||
function getRootDir(entries: AdmZip.IZipEntry[]) {
|
||||
const rootDirs = new Set<string>();
|
||||
for (const entry of entries) {
|
||||
const rootDir = entry.entryName.split(path.sep)[0];
|
||||
rootDirs.add(rootDir);
|
||||
}
|
||||
|
||||
if (rootDirs.size !== 1) {
|
||||
throw new InvalidContentPackError(
|
||||
`Expected a single root directory but got [${Array.from(rootDirs)}]`
|
||||
);
|
||||
}
|
||||
|
||||
return rootDirs.keys().next().value;
|
||||
}
|
||||
|
||||
function assertUncompressedSize(entry: AdmZip.IZipEntry) {
|
||||
if (entry.header.size > ARCHIVE_ENTRY_MAX_SIZE_BYTES) {
|
||||
throw new InvalidContentPackError(
|
||||
`Object [${entry.entryName}] exceeds the limit of ${ARCHIVE_ENTRY_MAX_SIZE_BYTES} bytes`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function missingEntryTypeImpl(type: never): never {
|
||||
throw new Error(`Content pack entry type [${type}] is not implemented`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { StatusError } from '../streams/errors/status_error';
|
||||
|
||||
export class InvalidContentPackError extends StatusError {
|
||||
constructor(message: string) {
|
||||
super(message, 400);
|
||||
this.name = 'InvalidContentPackError';
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ import {
|
|||
ContentPack,
|
||||
contentPackIncludedObjectsSchema,
|
||||
isIncludeAll,
|
||||
isSupportedSavedObjectType,
|
||||
} from '@kbn/content-packs-schema';
|
||||
import type { SavedObject } from '@kbn/core/server';
|
||||
import { STREAMS_API_PRIVILEGES } from '../../../common/constants';
|
||||
|
|
@ -107,7 +108,7 @@ const exportContentRoute = createServerRoute({
|
|||
return response.ok({
|
||||
body: archive,
|
||||
headers: {
|
||||
'Content-Disposition': `attachment; filename="${params.body.name}.zip"`,
|
||||
'Content-Disposition': `attachment; filename="${params.body.name}-${params.body.version}.zip"`,
|
||||
'Content-Type': 'application/zip',
|
||||
},
|
||||
});
|
||||
|
|
@ -165,11 +166,12 @@ const importContentRoute = createServerRoute({
|
|||
throw err;
|
||||
});
|
||||
|
||||
const links = savedObjectLinks(contentPack.entries, storedContentPack);
|
||||
const savedObjectEntries = contentPack.entries.filter(isSupportedSavedObjectType);
|
||||
const links = savedObjectLinks(savedObjectEntries, storedContentPack);
|
||||
const savedObjects = prepareForImport({
|
||||
target: params.path.name,
|
||||
include: params.body.include,
|
||||
savedObjects: contentPack.entries,
|
||||
savedObjects: savedObjectEntries,
|
||||
links,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -84,7 +84,9 @@ export function ExportContentPackFlyout({
|
|||
const contentPack = await previewContent({
|
||||
http,
|
||||
definition,
|
||||
file: new File([contentPackRaw], 'archive.zip', { type: 'application/zip' }),
|
||||
file: new File([contentPackRaw], `${definition.stream.name}-1.0.0.zip`, {
|
||||
type: 'application/zip',
|
||||
}),
|
||||
});
|
||||
|
||||
const indexPatterns = uniq(
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import { useKibana } from '../../hooks/use_kibana';
|
|||
import { ContentPackObjectsList } from './content_pack_objects_list';
|
||||
import { importContent, previewContent } from './content/requests';
|
||||
import { ContentPackMetadata } from './content_pack_manifest';
|
||||
import { getFormattedError } from '../../util/errors';
|
||||
|
||||
export function ImportContentPackFlyout({
|
||||
definition,
|
||||
|
|
@ -93,6 +94,7 @@ export function ImportContentPackFlyout({
|
|||
title: i18n.translate('xpack.streams.failedToPreviewContentError', {
|
||||
defaultMessage: 'Failed to preview content pack',
|
||||
}),
|
||||
toastMessage: getFormattedError(err).message,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
|
|
@ -154,6 +156,7 @@ export function ImportContentPackFlyout({
|
|||
title: i18n.translate('xpack.streams.failedToImportContentError', {
|
||||
defaultMessage: 'Failed to import content pack',
|
||||
}),
|
||||
toastMessage: getFormattedError(err).message,
|
||||
});
|
||||
}
|
||||
}}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { uniqBy } from 'lodash';
|
||||
import { generateArchive, parseArchive } from '@kbn/streams-plugin/server/lib/content';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
|
|
@ -13,6 +14,7 @@ import {
|
|||
ContentPackSavedObject,
|
||||
INDEX_PLACEHOLDER,
|
||||
findConfiguration,
|
||||
isSupportedSavedObjectType,
|
||||
} from '@kbn/content-packs-schema';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
|
||||
import {
|
||||
|
|
@ -37,11 +39,43 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
|
||||
const SPACE_ID = 'default';
|
||||
const ARCHIVES = [
|
||||
// this archive contains a dashboard with esql panel and a lens panel referencing a data view
|
||||
// both read from `logs`
|
||||
'src/platform/test/api_integration/fixtures/kbn_archiver/saved_objects/content_pack_two_panels.json',
|
||||
// this archive contains a dashboard with 4 types of panel all reading from `logs`
|
||||
'src/platform/test/api_integration/fixtures/kbn_archiver/saved_objects/content_pack_four_panels.json',
|
||||
];
|
||||
const TWO_PANELS_DASHBOARD_ID = 'c22ba8ed-fd4b-4864-a98c-3cba1d11cfb2';
|
||||
const ARCHIVE_DASHBOARD_ID = '9230e631-1f1a-476d-b613-4b074c6cfdd0';
|
||||
|
||||
function expectIndexPatternsFromEntries(
|
||||
entries: ContentPackSavedObject[],
|
||||
expectedPatterns: string[]
|
||||
) {
|
||||
entries.forEach((entry) => {
|
||||
const { patterns } = findConfiguration(entry);
|
||||
if (entry.attributes.title === 'lens-reference-with-index-pattern-ref') {
|
||||
expect(patterns).to.eql([]);
|
||||
return;
|
||||
}
|
||||
expect(patterns).to.eql(expectedPatterns);
|
||||
});
|
||||
}
|
||||
|
||||
async function expectIndexPatternsFromDashboard(dashboardId: string, expectedPatterns: string[]) {
|
||||
const dashboard = await kibanaServer.savedObjects.get({
|
||||
type: 'dashboard',
|
||||
id: dashboardId,
|
||||
});
|
||||
expect(uniqBy(dashboard.references, (ref) => ref.id).length).to.eql(3);
|
||||
|
||||
const resolvedReferences = await Promise.all(
|
||||
dashboard.references.map((ref) =>
|
||||
kibanaServer.savedObjects.get({ type: ref.type, id: ref.id })
|
||||
)
|
||||
);
|
||||
|
||||
expectIndexPatternsFromEntries(
|
||||
[dashboard, ...resolvedReferences] as ContentPackSavedObject[],
|
||||
expectedPatterns
|
||||
);
|
||||
}
|
||||
|
||||
describe('Content packs', () => {
|
||||
let contentPack: ContentPack;
|
||||
|
|
@ -49,22 +83,18 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
before(async () => {
|
||||
apiClient = await createStreamsRepositoryAdminClient(roleScopedSupertest);
|
||||
await enableStreams(apiClient);
|
||||
|
||||
await loadDashboards(kibanaServer, ARCHIVES, SPACE_ID);
|
||||
await linkDashboard(apiClient, 'logs', ARCHIVE_DASHBOARD_ID);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await disableStreams(apiClient);
|
||||
|
||||
await unloadDashboards(kibanaServer, ARCHIVES, SPACE_ID);
|
||||
});
|
||||
|
||||
describe('Export', () => {
|
||||
before(async () => {
|
||||
await loadDashboards(kibanaServer, ARCHIVES, SPACE_ID);
|
||||
await linkDashboard(apiClient, 'logs', TWO_PANELS_DASHBOARD_ID);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await unloadDashboards(kibanaServer, ARCHIVES, SPACE_ID);
|
||||
});
|
||||
|
||||
it('creates a content pack', async () => {
|
||||
const response = await exportContent(apiClient, 'logs', {
|
||||
name: 'logs-content_pack',
|
||||
|
|
@ -78,19 +108,20 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
expect(contentPack.name).to.be('logs-content_pack');
|
||||
expect(contentPack.version).to.be('1.0.0');
|
||||
expect(contentPack.description).to.be('my content pack');
|
||||
expect(contentPack.entries.length).to.eql(2);
|
||||
expect(contentPack.entries.length).to.eql(4);
|
||||
expect(contentPack.entries.filter((entry) => entry.type === 'dashboard').length).to.be(1);
|
||||
expect(contentPack.entries.filter((entry) => entry.type === 'lens').length).to.be(2);
|
||||
expect(contentPack.entries.filter((entry) => entry.type === 'index-pattern').length).to.be(
|
||||
1
|
||||
);
|
||||
});
|
||||
|
||||
it('puts placeholders for patterns matching the source stream', async () => {
|
||||
expect(contentPack.entries.length).to.eql(2);
|
||||
contentPack.entries.forEach((entry) => {
|
||||
const { patterns } = findConfiguration(entry);
|
||||
expect(patterns).to.eql([INDEX_PLACEHOLDER]);
|
||||
});
|
||||
// all saved objects only read from `logs`. since we exported the dashboard from
|
||||
// the root stream, the replacement logic should only leave placeholders
|
||||
const savedObjects = contentPack.entries.filter(isSupportedSavedObjectType);
|
||||
expect(savedObjects.length).to.eql(4);
|
||||
expectIndexPatternsFromEntries(savedObjects, [INDEX_PLACEHOLDER]);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -115,6 +146,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
const response = await importContent(apiClient, 'logs.importstream', {
|
||||
include: { all: {} },
|
||||
content: Readable.from(archive),
|
||||
filename: 'logs-content_pack-1.0.0.zip',
|
||||
});
|
||||
|
||||
expect(response.errors.length).to.be(0);
|
||||
|
|
@ -129,6 +161,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
const response = await importContent(apiClient, 'logs.importstream', {
|
||||
include: { all: {} },
|
||||
content: Readable.from(archive),
|
||||
filename: 'logs-content_pack-1.0.0.zip',
|
||||
});
|
||||
|
||||
expect(response.errors.length).to.be(0);
|
||||
|
|
@ -140,20 +173,58 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
|
||||
it('replaces placeholders with target stream pattern', async () => {
|
||||
const stream = await getStream(apiClient, 'logs.importstream');
|
||||
const dashboard = await kibanaServer.savedObjects.get({
|
||||
type: 'dashboard',
|
||||
id: stream.dashboards[0],
|
||||
});
|
||||
expect(dashboard.references.length).to.eql(1);
|
||||
const indexPattern = await kibanaServer.savedObjects.get({
|
||||
type: 'index-pattern',
|
||||
id: dashboard.references[0].id,
|
||||
});
|
||||
await expectIndexPatternsFromDashboard(stream.dashboards[0], ['logs.importstream']);
|
||||
});
|
||||
|
||||
[dashboard, indexPattern].forEach((object) => {
|
||||
const { patterns } = findConfiguration(object as ContentPackSavedObject);
|
||||
expect(patterns).to.eql(['logs.importstream']);
|
||||
});
|
||||
it('does not mutate the source saved objects', async () => {
|
||||
const stream = await getStream(apiClient, 'logs');
|
||||
await expectIndexPatternsFromDashboard(stream.dashboards[0], ['logs']);
|
||||
});
|
||||
|
||||
it('fails if an object is too large', async () => {
|
||||
const twoMB = 2 * 1024 * 1024;
|
||||
const archive = await generateArchive(
|
||||
{
|
||||
name: 'content_pack',
|
||||
description: 'with objects too big',
|
||||
version: '1.0.0',
|
||||
},
|
||||
[
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: 'regular_data_view',
|
||||
references: [],
|
||||
attributes: {
|
||||
title: 'logs*',
|
||||
name: 'logs*',
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'index-pattern',
|
||||
id: 'big_data_view',
|
||||
references: [],
|
||||
attributes: {
|
||||
title: 'a'.repeat(twoMB),
|
||||
name: 'big data view',
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
|
||||
const response = await importContent(
|
||||
apiClient,
|
||||
'logs.importstream',
|
||||
{
|
||||
include: { all: {} },
|
||||
content: Readable.from(archive),
|
||||
filename: 'content_pack-1.0.0.zip',
|
||||
},
|
||||
400
|
||||
);
|
||||
|
||||
expect((response as unknown as { message: string }).message).to.match(
|
||||
/^Object \[content_pack-1.0.0\/kibana\/index_pattern\/big_data_view.json\] exceeds the limit of \d+ bytes/
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ export async function importContent(
|
|||
body: {
|
||||
include: ContentPackIncludedObjects;
|
||||
content: Readable;
|
||||
filename: string;
|
||||
},
|
||||
expectStatusCode: number = 200
|
||||
) {
|
||||
|
|
@ -193,7 +194,7 @@ export async function importContent(
|
|||
content: body.content,
|
||||
},
|
||||
},
|
||||
file: { key: 'content', filename: 'content_pack.zip' },
|
||||
file: { key: 'content', filename: body.filename },
|
||||
})
|
||||
.expect(expectStatusCode)
|
||||
.then((response) => response.body);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue