[EPM] Change PACKAGES_SAVED_OBJECT_TYPE id (#62818)

* changed PACKAGES_SAVED_OBJECT_TYPE id from packageName-version to packageName

* change references to keys to package and version

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Sandra Gonzales 2020-04-09 11:42:10 -04:00 committed by GitHub
parent 6985478a32
commit a0c247b9cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 118 additions and 139 deletions

View file

@ -102,7 +102,9 @@ export const getInfoHandler: RequestHandler<TypeOf<typeof GetInfoRequestSchema.p
try {
const { pkgkey } = request.params;
const savedObjectsClient = context.core.savedObjects.client;
const res = await getPackageInfo({ savedObjectsClient, pkgkey });
// TODO: change epm API to /packageName/version so we don't need to do this
const [pkgName, pkgVersion] = pkgkey.split('-');
const res = await getPackageInfo({ savedObjectsClient, pkgName, pkgVersion });
const body: GetInfoResponse = {
response: res,
success: true,

View file

@ -9,7 +9,7 @@ import { DeleteDatasourcesResponse, packageToConfigDatasource } from '../../comm
import { DATASOURCE_SAVED_OBJECT_TYPE } from '../constants';
import { NewDatasource, Datasource, ListWithKuery } from '../types';
import { agentConfigService } from './agent_config';
import { findInstalledPackageByName, getPackageInfo } from './epm/packages';
import { getPackageInfo, getInstallation } from './epm/packages';
import { outputService } from './output';
const SAVED_OBJECT_TYPE = DATASOURCE_SAVED_OBJECT_TYPE;
@ -172,15 +172,13 @@ class DatasourceService {
soClient: SavedObjectsClientContract,
pkgName: string
): Promise<NewDatasource | undefined> {
const pkgInstall = await findInstalledPackageByName({
savedObjectsClient: soClient,
pkgName,
});
const pkgInstall = await getInstallation({ savedObjectsClient: soClient, pkgName });
if (pkgInstall) {
const [pkgInfo, defaultOutputId] = await Promise.all([
getPackageInfo({
savedObjectsClient: soClient,
pkgkey: `${pkgInstall.name}-${pkgInstall.version}`,
pkgName: pkgInstall.name,
pkgVersion: pkgInstall.version,
}),
outputService.getDefaultOutputId(soClient),
]);

View file

@ -7,9 +7,15 @@
import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types';
import * as Registry from '../../registry';
export async function installILMPolicy(pkgkey: string, callCluster: CallESAsCurrentUser) {
const ilmPaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
isILMPolicy(entry)
export async function installILMPolicy(
pkgName: string,
pkgVersion: string,
callCluster: CallESAsCurrentUser
) {
const ilmPaths = await Registry.getArchiveInfo(
pkgName,
pkgVersion,
(entry: Registry.ArchiveEntry) => isILMPolicy(entry)
);
if (!ilmPaths.length) return;
await Promise.all(

View file

@ -30,11 +30,10 @@ export const installPipelines = async (
if (dataset.ingest_pipeline) {
acc.push(
installPipelinesForDataset({
pkgkey: Registry.pkgToPkgKey(registryPackage),
dataset,
callCluster,
packageName: registryPackage.name,
packageVersion: registryPackage.version,
pkgName: registryPackage.name,
pkgVersion: registryPackage.version,
})
);
}
@ -68,19 +67,19 @@ export function rewriteIngestPipeline(
export async function installPipelinesForDataset({
callCluster,
pkgkey,
pkgName,
pkgVersion,
dataset,
packageName,
packageVersion,
}: {
callCluster: CallESAsCurrentUser;
pkgkey: string;
pkgName: string;
pkgVersion: string;
dataset: Dataset;
packageName: string;
packageVersion: string;
}): Promise<AssetReference[]> {
const pipelinePaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
isDatasetPipeline(entry, dataset.path)
const pipelinePaths = await Registry.getArchiveInfo(
pkgName,
pkgVersion,
(entry: Registry.ArchiveEntry) => isDatasetPipeline(entry, dataset.path)
);
let pipelines: any[] = [];
const substitutions: RewriteSubstitution[] = [];
@ -90,7 +89,7 @@ export async function installPipelinesForDataset({
const nameForInstallation = getPipelineNameForInstallation({
pipelineName: name,
dataset,
packageVersion,
packageVersion: pkgVersion,
});
const content = Registry.getAsset(path).toString('utf-8');
pipelines.push({

View file

@ -20,11 +20,12 @@ import * as Registry from '../../registry';
export const installTemplates = async (
registryPackage: RegistryPackage,
callCluster: CallESAsCurrentUser,
pkgkey: string
pkgName: string,
pkgVersion: string
) => {
// install any pre-built index template assets,
// atm, this is only the base package's global template
installPreBuiltTemplates(pkgkey, callCluster);
installPreBuiltTemplates(pkgName, pkgVersion, callCluster);
// build templates per dataset from yml files
const datasets = registryPackage.datasets;
@ -45,9 +46,15 @@ export const installTemplates = async (
};
// this is temporary until we update the registry to use index templates v2 structure
const installPreBuiltTemplates = async (pkgkey: string, callCluster: CallESAsCurrentUser) => {
const templatePaths = await Registry.getArchiveInfo(pkgkey, (entry: Registry.ArchiveEntry) =>
isTemplate(entry)
const installPreBuiltTemplates = async (
pkgName: string,
pkgVersion: string,
callCluster: CallESAsCurrentUser
) => {
const templatePaths = await Registry.getArchiveInfo(
pkgName,
pkgVersion,
(entry: Registry.ArchiveEntry) => isTemplate(entry)
);
templatePaths.forEach(async path => {
const { file } = Registry.pathParts(path);

View file

@ -68,25 +68,32 @@ export enum IndexPatternType {
metrics = 'metrics',
events = 'events',
}
// TODO: use a function overload and make pkgName and pkgVersion required for install/update
// and not for an update removal. or separate out the functions
export async function installIndexPatterns(
savedObjectsClient: SavedObjectsClientContract,
pkgkey?: string
pkgName?: string,
pkgVersion?: string
) {
// get all user installed packages
const installedPackages = await getPackageKeysByStatus(
savedObjectsClient,
InstallationStatus.installed
);
// add this package to the array if it doesn't already exist
// this should not happen because a user can't "reinstall" a package
// if it does because the install endpoint is called directly, the install continues
if (pkgkey && !installedPackages.includes(pkgkey)) {
installedPackages.push(pkgkey);
if (pkgName && pkgVersion) {
// add this package to the array if it doesn't already exist
const foundPkg = installedPackages.find(pkg => pkg.pkgName === pkgName);
// this may be removed if we add the packged to saved objects before installing index patterns
// otherwise this is a first time install
// TODO: handle update case when versions are different
if (!foundPkg) {
installedPackages.push({ pkgName, pkgVersion });
}
}
// get each package's registry info
const installedPackagesFetchInfoPromise = installedPackages.map(pkg => Registry.fetchInfo(pkg));
const installedPackagesFetchInfoPromise = installedPackages.map(pkg =>
Registry.fetchInfo(pkg.pkgName, pkg.pkgVersion)
);
const installedPackagesInfo = await Promise.all(installedPackagesFetchInfoPromise);
// for each index pattern type, create an index pattern
@ -97,7 +104,7 @@ export async function installIndexPatterns(
];
indexPatternTypes.forEach(async indexPatternType => {
// if this is an update because a package is being unisntalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
if (!pkgkey && installedPackages.length === 0) {
if (!pkgName && installedPackages.length === 0) {
try {
await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, `${indexPatternType}-*`);
} catch (err) {

View file

@ -58,7 +58,7 @@ export async function getAssetsData(
): Promise<Registry.ArchiveEntry[]> {
// TODO: Needs to be called to fill the cache but should not be required
const pkgkey = packageInfo.name + '-' + packageInfo.version;
if (!cacheHas(pkgkey)) await Registry.getArchiveInfo(pkgkey);
if (!cacheHas(pkgkey)) await Registry.getArchiveInfo(packageInfo.name, packageInfo.version);
// Gather all asset data
const assets = getAssets(packageInfo, filter, datasetName);

View file

@ -41,7 +41,7 @@ export async function getPackages(
.map(item =>
createInstallableFrom(
item,
savedObjectsVisible.find(({ attributes }) => attributes.name === item.name)
savedObjectsVisible.find(({ id }) => id === item.name)
)
)
.sort(sortByName);
@ -53,9 +53,9 @@ export async function getPackageKeysByStatus(
status: InstallationStatus
) {
const allPackages = await getPackages({ savedObjectsClient });
return allPackages.reduce<string[]>((acc, pkg) => {
return allPackages.reduce<Array<{ pkgName: string; pkgVersion: string }>>((acc, pkg) => {
if (pkg.status === status) {
acc.push(`${pkg.name}-${pkg.version}`);
acc.push({ pkgName: pkg.name, pkgVersion: pkg.version });
}
return acc;
}, []);
@ -63,13 +63,14 @@ export async function getPackageKeysByStatus(
export async function getPackageInfo(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
pkgName: string;
pkgVersion: string;
}): Promise<PackageInfo> {
const { savedObjectsClient, pkgkey } = options;
const { savedObjectsClient, pkgName, pkgVersion } = options;
const [item, savedObject] = await Promise.all([
Registry.fetchInfo(pkgkey),
getInstallationObject({ savedObjectsClient, pkgkey }),
Registry.getArchiveInfo(pkgkey),
Registry.fetchInfo(pkgName, pkgVersion),
getInstallationObject({ savedObjectsClient, pkgName }),
Registry.getArchiveInfo(pkgName, pkgVersion),
] as const);
// adding `as const` due to regression in TS 3.7.2
// see https://github.com/microsoft/TypeScript/issues/34925#issuecomment-550021453
@ -86,37 +87,22 @@ export async function getPackageInfo(options: {
export async function getInstallationObject(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
pkgName: string;
}) {
const { savedObjectsClient, pkgkey } = options;
const { savedObjectsClient, pkgName } = options;
return savedObjectsClient
.get<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgkey)
.get<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName)
.catch(e => undefined);
}
export async function getInstallation(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
pkgName: string;
}) {
const savedObject = await getInstallationObject(options);
return savedObject?.attributes;
}
export async function findInstalledPackageByName(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
}): Promise<Installation | undefined> {
const { savedObjectsClient, pkgName } = options;
const res = await savedObjectsClient.find<Installation>({
type: PACKAGES_SAVED_OBJECT_TYPE,
search: pkgName,
searchFields: ['name'],
});
if (res.saved_objects.length) return res.saved_objects[0].attributes;
return undefined;
}
function sortByName(a: { name: string }, b: { name: string }) {
if (a.name > b.name) {
return 1;

View file

@ -11,46 +11,6 @@ import * as Registry from '../registry';
type ArchiveAsset = Pick<SavedObject, 'attributes' | 'migrationVersion' | 'references'>;
type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
export async function getObjects(
pkgkey: string,
filter = (entry: Registry.ArchiveEntry): boolean => true
): Promise<SavedObjectToBe[]> {
// Create a Map b/c some values, especially index-patterns, are referenced multiple times
const objects: Map<string, SavedObjectToBe> = new Map();
// Get paths which match the given filter
const paths = await Registry.getArchiveInfo(pkgkey, filter);
// Get all objects which matched filter. Add them to the Map
const rootObjects = await Promise.all(paths.map(getObject));
rootObjects.forEach(obj => objects.set(obj.id, obj));
// Each of those objects might have `references` property like [{id, type, name}]
for (const object of rootObjects) {
// For each of those objects, if they have references
for (const reference of object.references) {
// Get the referenced objects. Call same function with a new filter
const referencedObjects = await getObjects(pkgkey, (entry: Registry.ArchiveEntry) => {
// Skip anything we've already stored
if (objects.has(reference.id)) return false;
// Is the archive entry the reference we want?
const { type, file } = Registry.pathParts(entry.path);
const isType = type === reference.type;
const isJson = file === `${reference.id}.json`;
return isType && isJson;
});
// Add referenced objects to the Map
referencedObjects.forEach(ro => objects.set(ro.id, ro));
}
}
// return the array of unique objects
return Array.from(objects.values());
}
export async function getObject(key: string) {
const buffer = Registry.getAsset(key);

View file

@ -21,7 +21,6 @@ export {
getPackageInfo,
getPackages,
SearchParams,
findInstalledPackageByName,
} from './get';
export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install';

View file

@ -16,7 +16,7 @@ import {
import { installIndexPatterns } from '../kibana/index_pattern/install';
import * as Registry from '../registry';
import { getObject } from './get_objects';
import { getInstallation, findInstalledPackageByName } from './index';
import { getInstallation } from './index';
import { installTemplates } from '../elasticsearch/template/install';
import { installPipelines } from '../elasticsearch/ingest_pipeline/install';
import { installILMPolicy } from '../elasticsearch/ilm/install';
@ -63,7 +63,7 @@ export async function ensureInstalledPackage(options: {
callCluster: CallESAsCurrentUser;
}): Promise<Installation | undefined> {
const { savedObjectsClient, pkgName, callCluster } = options;
const installedPackage = await findInstalledPackageByName({ savedObjectsClient, pkgName });
const installedPackage = await getInstallation({ savedObjectsClient, pkgName });
if (installedPackage) {
return installedPackage;
}
@ -74,7 +74,7 @@ export async function ensureInstalledPackage(options: {
pkgName,
callCluster,
});
return await findInstalledPackageByName({ savedObjectsClient, pkgName });
return await getInstallation({ savedObjectsClient, pkgName });
} catch (err) {
throw new Error(err.message);
}
@ -86,22 +86,30 @@ export async function installPackage(options: {
callCluster: CallESAsCurrentUser;
}): Promise<AssetReference[]> {
const { savedObjectsClient, pkgkey, callCluster } = options;
const registryPackageInfo = await Registry.fetchInfo(pkgkey);
const { name: pkgName, version: pkgVersion, internal = false } = registryPackageInfo;
// TODO: change epm API to /packageName/version so we don't need to do this
const [pkgName, pkgVersion] = pkgkey.split('-');
const registryPackageInfo = await Registry.fetchInfo(pkgName, pkgVersion);
const { internal = false } = registryPackageInfo;
const installKibanaAssetsPromise = installKibanaAssets({
savedObjectsClient,
pkgkey,
pkgName,
pkgVersion,
});
const installPipelinePromises = installPipelines(registryPackageInfo, callCluster);
const installTemplatePromises = installTemplates(registryPackageInfo, callCluster, pkgkey);
const installTemplatePromises = installTemplates(
registryPackageInfo,
callCluster,
pkgName,
pkgVersion
);
// index patterns and ilm policies are not currently associated with a particular package
// so we do not save them in the package saved object state. at some point ILM policies can be installed/modified
// per dataset and we should then save them
await installIndexPatterns(savedObjectsClient, pkgkey);
await installIndexPatterns(savedObjectsClient, pkgName, pkgVersion);
// currenly only the base package has an ILM policy
await installILMPolicy(pkgkey, callCluster);
await installILMPolicy(pkgName, pkgVersion, callCluster);
const res = await Promise.all([
installKibanaAssetsPromise,
@ -126,14 +134,15 @@ export async function installPackage(options: {
// e.g. switch statement with cases for each enum key returning `never` for default case
export async function installKibanaAssets(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
pkgName: string;
pkgVersion: string;
}) {
const { savedObjectsClient, pkgkey } = options;
const { savedObjectsClient, pkgName, pkgVersion } = options;
// Only install Kibana assets during package installation.
const kibanaAssetTypes = Object.values(KibanaAssetType);
const installationPromises = kibanaAssetTypes.map(async assetType =>
installKibanaSavedObjects({ savedObjectsClient, pkgkey, assetType })
installKibanaSavedObjects({ savedObjectsClient, pkgName, pkgVersion, assetType })
);
// installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][]
@ -149,8 +158,8 @@ export async function saveInstallationReferences(options: {
internal: boolean;
toSave: AssetReference[];
}) {
const { savedObjectsClient, pkgkey, pkgName, pkgVersion, internal, toSave } = options;
const installation = await getInstallation({ savedObjectsClient, pkgkey });
const { savedObjectsClient, pkgName, pkgVersion, internal, toSave } = options;
const installation = await getInstallation({ savedObjectsClient, pkgName });
const savedRefs = installation?.installed || [];
const mergeRefsReducer = (current: AssetReference[], pending: AssetReference) => {
const hasRef = current.find(c => c.id === pending.id && c.type === pending.type);
@ -162,7 +171,7 @@ export async function saveInstallationReferences(options: {
await savedObjectsClient.create<Installation>(
PACKAGES_SAVED_OBJECT_TYPE,
{ installed: toInstall, name: pkgName, version: pkgVersion, internal },
{ id: pkgkey, overwrite: true }
{ id: pkgName, overwrite: true }
);
return toInstall;
@ -170,16 +179,18 @@ export async function saveInstallationReferences(options: {
async function installKibanaSavedObjects({
savedObjectsClient,
pkgkey,
pkgName,
pkgVersion,
assetType,
}: {
savedObjectsClient: SavedObjectsClientContract;
pkgkey: string;
pkgName: string;
pkgVersion: string;
assetType: KibanaAssetType;
}) {
const isSameType = ({ path }: Registry.ArchiveEntry) =>
assetType === Registry.pathParts(path).type;
const paths = await Registry.getArchiveInfo(pkgkey, isSameType);
const paths = await Registry.getArchiveInfo(pkgName, pkgVersion, isSameType);
const toBeSavedObjects = await Promise.all(paths.map(getObject));
if (toBeSavedObjects.length === 0) {

View file

@ -17,12 +17,14 @@ export async function removeInstallation(options: {
callCluster: CallESAsCurrentUser;
}): Promise<AssetReference[]> {
const { savedObjectsClient, pkgkey, callCluster } = options;
const installation = await getInstallation({ savedObjectsClient, pkgkey });
// TODO: the epm api should change to /name/version so we don't need to do this
const [pkgName] = pkgkey.split('-');
const installation = await getInstallation({ savedObjectsClient, pkgName });
const installedObjects = installation?.installed || [];
// Delete the manager saved object with references to the asset objects
// could also update with [] or some other state
await savedObjectsClient.delete(PACKAGES_SAVED_OBJECT_TYPE, pkgkey);
await savedObjectsClient.delete(PACKAGES_SAVED_OBJECT_TYPE, pkgName);
// recreate or delete index patterns when a package is uninstalled
await installIndexPatterns(savedObjectsClient);

View file

@ -56,10 +56,9 @@ export async function fetchFindLatestPackage(
}
}
export async function fetchInfo(pkgkey: string): Promise<RegistryPackage> {
export async function fetchInfo(pkgName: string, pkgVersion: string): Promise<RegistryPackage> {
const registryUrl = appContextService.getConfig()?.epm.registryUrl;
// change pkg-version to pkg/version
return fetchUrl(`${registryUrl}/package/${pkgkey.replace('-', '/')}`).then(JSON.parse);
return fetchUrl(`${registryUrl}/package/${pkgName}/${pkgVersion}`).then(JSON.parse);
}
export async function fetchFile(filePath: string): Promise<Response> {
@ -73,7 +72,8 @@ export async function fetchCategories(): Promise<CategorySummaryList> {
}
export async function getArchiveInfo(
pkgkey: string,
pkgName: string,
pkgVersion: string,
filter = (entry: ArchiveEntry): boolean => true
): Promise<string[]> {
const paths: string[] = [];
@ -87,7 +87,7 @@ export async function getArchiveInfo(
}
};
await extract(pkgkey, filter, onEntry);
await extract(pkgName, pkgVersion, filter, onEntry);
return paths;
}
@ -123,21 +123,22 @@ export function pathParts(path: string): AssetParts {
}
async function extract(
pkgkey: string,
pkgName: string,
pkgVersion: string,
filter = (entry: ArchiveEntry): boolean => true,
onEntry: (entry: ArchiveEntry) => void
) {
const archiveBuffer = await getOrFetchArchiveBuffer(pkgkey);
const archiveBuffer = await getOrFetchArchiveBuffer(pkgName, pkgVersion);
return untarBuffer(archiveBuffer, filter, onEntry);
}
async function getOrFetchArchiveBuffer(pkgkey: string): Promise<Buffer> {
async function getOrFetchArchiveBuffer(pkgName: string, pkgVersion: string): Promise<Buffer> {
// assume .tar.gz for now. add support for .zip if/when we need it
const key = `${pkgkey}.tar.gz`;
const key = `${pkgName}-${pkgVersion}.tar.gz`;
let buffer = cacheGet(key);
if (!buffer) {
buffer = await fetchArchiveBuffer(pkgkey);
buffer = await fetchArchiveBuffer(pkgName, pkgVersion);
cacheSet(key, buffer);
}
@ -148,8 +149,8 @@ async function getOrFetchArchiveBuffer(pkgkey: string): Promise<Buffer> {
}
}
async function fetchArchiveBuffer(key: string): Promise<Buffer> {
const { download: archivePath } = await fetchInfo(key);
async function fetchArchiveBuffer(pkgName: string, pkgVersion: string): Promise<Buffer> {
const { download: archivePath } = await fetchInfo(pkgName, pkgVersion);
const registryUrl = appContextService.getConfig()?.epm.registryUrl;
return getResponseStream(`${registryUrl}${archivePath}`).then(streamToBuffer);
}

View file

@ -120,7 +120,8 @@ async function addPackageToConfig(
) {
const packageInfo = await getPackageInfo({
savedObjectsClient: soClient,
pkgkey: `${packageToInstall.name}-${packageToInstall.version}`,
pkgName: packageToInstall.name,
pkgVersion: packageToInstall.version,
});
await datasourceService.create(
soClient,