github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Fleet] Support Input Packages (#140035)

Browse source 
* seperate types for integration and input packages

* Simplify input only package types

* add util for checking policy template type

* fix type errors now there are 2 kinds of policy template

* Package policies being generate for input packages

* fix types

* support input templates

* neaten for PR

* more PR tidy

* dont show input disbale switch for input only policies

* always read input only pkg info from the archive

* fix limited package check

* use negative check

* add unit tests

* fix isIntegrationPolicyTemplate

* generate index permissions

* do not use template for input

* fix types

* fix tests

* dont shwo description for input packages

* flip flag

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Mark Hopkin 2022-09-15 15:18:19 +01:00 • committed by GitHub
parent 3ee9383978
commit 47fe9e7a55
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 576 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

6
x-pack/plugins/apm/server/routes/fleet/get_latest_apm_package.ts
View file

@ -23,6 +23,10 @@ export async function getLatestApmPackage({
const registryPackage = await packageClient.getRegistryPackage(name, version);
const { title, policy_templates: policyTemplates } =
registryPackage.packageInfo;
const policyTemplateInputVars = policyTemplates?.[0].inputs?.[0].vars ?? [];
const firstTemplate = policyTemplates?.[0];
const policyTemplateInputVars =
firstTemplate && 'inputs' in firstTemplate
? firstTemplate.inputs?.[0].vars || []
: [];
return { package: { name, version, title }, policyTemplateInputVars };
}

6
x-pack/plugins/fleet/common/services/index.ts
View file

@ -19,6 +19,12 @@ export { isValidNamespace } from './is_valid_namespace';
export { isDiffPathProtocol } from './is_diff_path_protocol';
export { LicenseService } from './license';
export { isAgentUpgradeable } from './is_agent_upgradeable';
export {
isInputOnlyPolicyTemplate,
isIntegrationPolicyTemplate,
getNormalizedInputs,
getNormalizedDataStreams,
} from './policy_template';
export { doesPackageHaveIntegrations } from './packages_with_integrations';
export type {
PackagePolicyValidationResults,

12
x-pack/plugins/fleet/common/services/package_to_package_policy.ts
View file

@ -18,6 +18,11 @@ import type {
} from '../types';
import { doesPackageHaveIntegrations } from '.';
import {
getNormalizedDataStreams,
getNormalizedInputs,
isIntegrationPolicyTemplate,
} from './policy_template';
type PackagePolicyStream = RegistryStream & { release?: 'beta' | 'experimental' | 'ga' } & {
data_stream: { type: string; dataset: string };
@ -29,7 +34,7 @@ export const getStreamsForInputType = (
dataStreamPaths: string[] = []
): PackagePolicyStream[] => {
const streams: PackagePolicyStream[] = [];
const dataStreams = packageInfo.data_streams || [];
const dataStreams = getNormalizedDataStreams(packageInfo);
const dataStreamsToSearch = dataStreamPaths.length
? dataStreams.filter((dataStream) => dataStreamPaths.includes(dataStream.path))
: dataStreams;
@ -81,11 +86,12 @@ export const packageToPackagePolicyInputs = (
} = {};
packageInfo.policy_templates?.forEach((packagePolicyTemplate) => {
packagePolicyTemplate.inputs?.forEach((packageInput) => {
const normalizedInputs = getNormalizedInputs(packagePolicyTemplate);
normalizedInputs?.forEach((packageInput) => {
const inputKey = `${packagePolicyTemplate.name}-${packageInput.type}`;
const input = {
...packageInput,
...(packagePolicyTemplate.data_streams
...(isIntegrationPolicyTemplate(packagePolicyTemplate) && packagePolicyTemplate.data_streams
? { data_streams: packagePolicyTemplate.data_streams }
: {}),
policy_template: packagePolicyTemplate.name,

278
x-pack/plugins/fleet/common/services/policy_template.test.ts Normal file
View file

@ -0,0 +1,278 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
RegistryPolicyInputOnlyTemplate,
RegistryPolicyIntegrationTemplate,
PackageInfo,
RegistryVarType,
} from '../types';
import {
isInputOnlyPolicyTemplate,
isIntegrationPolicyTemplate,
getNormalizedInputs,
getNormalizedDataStreams,
} from './policy_template';
describe('isInputOnlyPolicyTemplate', () => {
it('should return true input only policy template', () => {
const inputOnlyPolicyTemplate: RegistryPolicyInputOnlyTemplate = {
input: 'string',
type: 'foo',
name: 'bar',
template_path: 'some/path.hbl',
title: 'hello',
description: 'desc',
};
expect(isInputOnlyPolicyTemplate(inputOnlyPolicyTemplate)).toEqual(true);
});
it('should return false for empty integration policy template', () => {
const emptyIntegrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(isInputOnlyPolicyTemplate(emptyIntegrationTemplate)).toEqual(false);
});
it('should return false for integration policy template with inputs', () => {
const integrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [
{
type: 'foo',
title: 'myFoo',
description: 'myFoo',
vars: [],
},
],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(isInputOnlyPolicyTemplate(integrationTemplate)).toEqual(false);
});
});
describe('isIntegrationPolicyTemplate', () => {
it('should return true input only policy template', () => {
const inputOnlyPolicyTemplate: RegistryPolicyInputOnlyTemplate = {
input: 'string',
type: 'foo',
name: 'bar',
template_path: 'some/path.hbl',
title: 'hello',
description: 'desc',
};
expect(isIntegrationPolicyTemplate(inputOnlyPolicyTemplate)).toEqual(false);
});
it('should return false for empty integration policy template', () => {
const emptyIntegrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(isIntegrationPolicyTemplate(emptyIntegrationTemplate)).toEqual(true);
});
it('should return false for integration policy template with inputs', () => {
const integrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [
{
type: 'foo',
title: 'myFoo',
description: 'myFoo',
vars: [],
},
],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(isIntegrationPolicyTemplate(integrationTemplate)).toEqual(true);
});
});
describe('getNormalizedInputs', () => {
it('should return empty array if template has no inputs', () => {
const emptyIntegrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(getNormalizedInputs(emptyIntegrationTemplate)).toEqual([]);
});
it('should return inputs if there are any', () => {
const emptyIntegrationTemplate: RegistryPolicyIntegrationTemplate = {
inputs: [
{
type: 'foo',
title: 'myFoo',
description: 'myFoo',
vars: [],
},
],
name: 'bar',
title: 'hello',
description: 'desc',
};
expect(getNormalizedInputs(emptyIntegrationTemplate)).toEqual([
{
type: 'foo',
title: 'myFoo',
description: 'myFoo',
vars: [],
},
]);
});
it('should return array with one input for input only', () => {
const inputOnlyTemplate: RegistryPolicyInputOnlyTemplate = {
input: 'string',
type: 'foo',
name: 'bar',
template_path: 'some/path.hbl',
title: 'myFoo',
description: 'myFoo',
};
expect(getNormalizedInputs(inputOnlyTemplate)).toEqual([
{
type: 'string',
title: 'myFoo',
description: 'myFoo',
},
]);
});
});
describe('getNormalizedDataStreams', () => {
const integrationPkg: PackageInfo = {
name: 'nginx',
title: 'Nginx',
version: '1.3.0',
release: 'ga',
description: 'Collect logs and metrics from Nginx HTTP servers with Elastic Agent.',
format_version: '',
owner: { github: '' },
assets: {} as any,
policy_templates: [],
data_streams: [
{
type: 'logs',
dataset: 'nginx.access',
title: 'Nginx access logs',
release: 'experimental',
ingest_pipeline: 'default',
streams: [
{
input: 'logfile',
vars: [
{
name: 'paths',
type: 'text',
title: 'Paths',
multi: true,
required: true,
show_user: true,
default: ['/var/log/nginx/access.log*'],
},
],
template_path: 'stream.yml.hbs',
title: 'Nginx access logs',
description: 'Collect Nginx access logs',
enabled: true,
},
],
package: 'nginx',
path: 'access',
},
],
latestVersion: '1.3.0',
keepPoliciesUpToDate: false,
status: 'not_installed',
};
it('should return data_streams for integration package', () => {
expect(getNormalizedDataStreams(integrationPkg)).toEqual(integrationPkg.data_streams);
});
it('should return data_streams for integration package with type specified', () => {
expect(getNormalizedDataStreams({ ...integrationPkg, type: 'integration' })).toEqual(
integrationPkg.data_streams
);
});
it('should return data_streams for empty integration package', () => {
expect(getNormalizedDataStreams({ ...integrationPkg, data_streams: [] })).toEqual([]);
});
it('should build data streams for input only package', () => {
expect(
getNormalizedDataStreams({
...integrationPkg,
type: 'input',
policy_templates: [
{
input: 'string',
type: 'foo',
name: 'bar',
template_path: 'some/path.hbl',
title: 'myFoo',
description: 'myFoo',
vars: [],
},
],
})
).toEqual([
{
type: 'foo',
dataset: 'nginx.foo',
title: expect.any(String),
release: 'ga',
package: 'nginx',
path: 'nginx',
streams: [
{
input: 'string',
vars: expect.any(Array),
template_path: 'some/path.hbl',
title: 'myFoo',
description: 'myFoo',
enabled: true,
},
],
},
]);
});
it('should not add dataset if already present', () => {
const datasetVar = {
name: 'data_stream.dataset',
type: 'text' as RegistryVarType,
title: 'local dataset',
description: 'some desc',
multi: false,
required: true,
show_user: true,
};
const result = getNormalizedDataStreams({
...integrationPkg,
type: 'input',
policy_templates: [
{
input: 'string',
type: 'foo',
name: 'bar',
template_path: 'some/path.hbl',
title: 'myFoo',
description: 'myFoo',
vars: [datasetVar],
},
],
});
expect(result).toHaveLength(1);
expect(result[0].streams).toHaveLength(1);
expect(result?.[0].streams?.[0]?.vars).toEqual([datasetVar]);
});
});

109
x-pack/plugins/fleet/common/services/policy_template.ts Normal file
View file

@ -0,0 +1,109 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type {
RegistryPolicyTemplate,
RegistryPolicyInputOnlyTemplate,
RegistryPolicyIntegrationTemplate,
RegistryInput,
PackageInfo,
RegistryVarsEntry,
RegistryDataStream,
} from '../types';
const DATA_STREAM_DATASET_VAR: RegistryVarsEntry = {
name: 'data_stream.dataset',
type: 'text',
title: 'Dataset name',
description:
"Set the name for your dataset. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).\n",
multi: false,
required: true,
show_user: true,
};
export function isInputOnlyPolicyTemplate(
policyTemplate: RegistryPolicyTemplate
): policyTemplate is RegistryPolicyInputOnlyTemplate {
return 'input' in policyTemplate;
}
export function isIntegrationPolicyTemplate(
policyTemplate: RegistryPolicyTemplate
): policyTemplate is RegistryPolicyIntegrationTemplate {
return !isInputOnlyPolicyTemplate(policyTemplate);
}
export const getNormalizedInputs = (policyTemplate: RegistryPolicyTemplate): RegistryInput[] => {
if (isIntegrationPolicyTemplate(policyTemplate)) {
return policyTemplate.inputs || [];
}
const input: RegistryInput = {
type: policyTemplate.input,
title: policyTemplate.title,
description: policyTemplate.description,
};
return [input];
};
export const getNormalizedDataStreams = (packageInfo: PackageInfo): RegistryDataStream[] => {
if (packageInfo.type !== 'input') {
return packageInfo.data_streams || [];
}
const policyTemplates = packageInfo.policy_templates as RegistryPolicyInputOnlyTemplate[];
if (!policyTemplates || policyTemplates.length === 0) {
return [];
}
return policyTemplates.map((policyTemplate) => {
const dataStream: RegistryDataStream = {
type: policyTemplate.type,
dataset: createDefaultDatasetName(packageInfo, policyTemplate),
title: policyTemplate.title + ' Dataset',
release: packageInfo.release || 'ga',
package: packageInfo.name,
path: packageInfo.name,
streams: [
{
input: policyTemplate.input,
vars: addDatasetVarIfNotPresent(policyTemplate.vars),
template_path: policyTemplate.template_path,
title: policyTemplate.title,
description: policyTemplate.title,
enabled: true,
},
],
};
return dataStream;
});
};
// Input only packages must provide a dataset name in order to differentiate their data streams
// here we add the dataset var if it is not defined in the package already.
const addDatasetVarIfNotPresent = (vars?: RegistryVarsEntry[]): RegistryVarsEntry[] => {
const newVars = vars ?? [];
const isDatasetAlreadyAdded = newVars.find(
(varEntry) => varEntry.name === DATA_STREAM_DATASET_VAR.name
);
if (isDatasetAlreadyAdded) {
return newVars;
} else {
return [...newVars, DATA_STREAM_DATASET_VAR];
}
};
const createDefaultDatasetName = (
packageInfo: PackageInfo,
policyTemplate: RegistryPolicyInputOnlyTemplate
): string => packageInfo.name + '.' + policyTemplate.type;

33
x-pack/plugins/fleet/common/services/validate_package_policy.ts
View file

@ -19,7 +19,13 @@ import type {
RegistryVarsEntry,
} from '../types';
import { isValidNamespace, doesPackageHaveIntegrations } from '.';
import {
isValidNamespace,
doesPackageHaveIntegrations,
isInputOnlyPolicyTemplate,
getNormalizedInputs,
getNormalizedDataStreams,
} from '.';
type Errors = string[] | null;
@ -90,7 +96,9 @@ export const validatePackagePolicy = (
!packageInfo.policy_templates ||
packageInfo.policy_templates.length === 0 ||
!packageInfo.policy_templates.find(
(policyTemplate) => policyTemplate.inputs && policyTemplate.inputs.length > 0
(policyTemplate) =>
isInputOnlyPolicyTemplate(policyTemplate) ||
(policyTemplate.inputs && policyTemplate.inputs.length > 0)
)
) {
validationResults.inputs = null;
@ -101,7 +109,8 @@ export const validatePackagePolicy = (
const inputVarDefsByPolicyTemplateAndType = packageInfo.policy_templates.reduce<
Record<string, Record<string, RegistryVarsEntry>>
>((varDefs, policyTemplate) => {
(policyTemplate.inputs || []).forEach((input) => {
const inputs = getNormalizedInputs(policyTemplate);
inputs.forEach((input) => {
const varDefKey = hasIntegrations ? `${policyTemplate.name}-${input.type}` : input.type;
if ((input.vars || []).length) {
@ -111,14 +120,16 @@ export const validatePackagePolicy = (
return varDefs;
}, {});
const streamsByDatasetAndInput = (packageInfo.data_streams || []).reduce<
Record<string, RegistryStream>
>((streams, dataStream) => {
dataStream.streams?.forEach((stream) => {
streams[`${dataStream.dataset}-${stream.input}`] = stream;
});
return streams;
}, {});
const dataStreams = getNormalizedDataStreams(packageInfo);
const streamsByDatasetAndInput = dataStreams.reduce<Record<string, RegistryStream>>(
(streams, dataStream) => {
dataStream.streams?.forEach((stream) => {
streams[`${dataStream.dataset}-${stream.input}`] = stream;
});
return streams;
},
{}
);
const streamVarDefsByDatasetAndInput = Object.entries(streamsByDatasetAndInput).reduce<
Record<string, Record<string, RegistryVarsEntry>>
>((varDefs, [path, stream]) => {

32
x-pack/plugins/fleet/common/types/models/epm.ts
View file

@ -157,31 +157,47 @@ export interface RegistryImage extends PackageSpecIcon {
}
export enum RegistryPolicyTemplateKeys {
name = 'name',
title = 'title',
description = 'description',
icons = 'icons',
screenshots = 'screenshots',
categories = 'categories',
data_streams = 'data_streams',
inputs = 'inputs',
readme = 'readme',
multiple = 'multiple',
type = 'type',
vars = 'vars',
input = 'input',
template_path = 'template_path',
name = 'name',
title = 'title',
description = 'description',
icons = 'icons',
screenshots = 'screenshots',
}
export interface RegistryPolicyTemplate {
interface BaseTemplate {
[RegistryPolicyTemplateKeys.name]: string;
[RegistryPolicyTemplateKeys.title]: string;
[RegistryPolicyTemplateKeys.description]: string;
[RegistryPolicyTemplateKeys.icons]?: RegistryImage[];
[RegistryPolicyTemplateKeys.screenshots]?: RegistryImage[];
[RegistryPolicyTemplateKeys.multiple]?: boolean;
}
export interface RegistryPolicyIntegrationTemplate extends BaseTemplate {
[RegistryPolicyTemplateKeys.categories]?: Array<PackageSpecCategory | undefined>;
[RegistryPolicyTemplateKeys.data_streams]?: string[];
[RegistryPolicyTemplateKeys.inputs]?: RegistryInput[];
[RegistryPolicyTemplateKeys.readme]?: string;
[RegistryPolicyTemplateKeys.multiple]?: boolean;
}
export interface RegistryPolicyInputOnlyTemplate extends BaseTemplate {
[RegistryPolicyTemplateKeys.type]: string;
[RegistryPolicyTemplateKeys.input]: string;
[RegistryPolicyTemplateKeys.template_path]: string;
[RegistryPolicyTemplateKeys.vars]?: RegistryVarsEntry[];
}
export type RegistryPolicyTemplate =
| RegistryPolicyIntegrationTemplate
| RegistryPolicyInputOnlyTemplate;
export enum RegistryInputKeys {
type = 'type',
title = 'title',

4
x-pack/plugins/fleet/common/types/models/package_spec.ts
View file

@ -15,7 +15,7 @@ export interface PackageSpecManifest {
description: string;
version: string;
license?: 'basic';
type?: 'integration';
type?: 'integration' | 'input';
release?: 'experimental' | 'beta' | 'ga';
categories?: Array<PackageSpecCategory | undefined>;
conditions?: PackageSpecConditions;
@ -26,6 +26,8 @@ export interface PackageSpecManifest {
owner: { github: string };
}
export type PackageSpecPackageType = 'integration' | 'input';
export type PackageSpecCategory =
| 'aws'
| 'azure'

30
x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx
View file

@ -129,26 +129,28 @@ export const PackagePolicyInputStreamConfig: React.FunctionComponent<{
alignItems="flexStart"
justifyContent="spaceBetween"
>
<EuiFlexItem grow={false}>
<EuiSwitch
label={packageInputStream.title}
disabled={packagePolicyInputStream.keep_enabled}
checked={packagePolicyInputStream.enabled}
onChange={(e) => {
const enabled = e.target.checked;
updatePackagePolicyInputStream({
enabled,
});
}}
/>
</EuiFlexItem>
{packageInfo.type !== 'input' && (
<EuiFlexItem grow={false}>
<EuiSwitch
label={packageInputStream.title}
disabled={packagePolicyInputStream.keep_enabled}
checked={packagePolicyInputStream.enabled}
onChange={(e) => {
const enabled = e.target.checked;
updatePackagePolicyInputStream({
enabled,
});
}}
/>
</EuiFlexItem>
)}
{packagePolicyInputStream.release && packagePolicyInputStream.release !== 'ga' ? (
<EuiFlexItem grow={false}>
<InlineReleaseBadge release={packagePolicyInputStream.release} />
</EuiFlexItem>
) : null}
</EuiFlexGroup>
{packageInputStream.description ? (
{packageInfo.type !== 'input' && packageInputStream.description ? (
<Fragment>
<EuiSpacer size="s" />
<EuiText size="s" color="subdued">

12
x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_configure_package.tsx
View file

@ -15,6 +15,11 @@ import {
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import {
getNormalizedInputs,
isIntegrationPolicyTemplate,
} from '../../../../../../../../common/services';
import type { PackageInfo, NewPackagePolicy, NewPackagePolicyInput } from '../../../../../types';
import { Loading } from '../../../../../components';
import { getStreamsForInputType, doesPackageHaveIntegrations } from '../../../../../services';
@ -57,7 +62,8 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{
{!noTopRule && <EuiHorizontalRule margin="m" />}
<EuiFlexGroup direction="column" gutterSize="none">
{packagePolicyTemplates.map((policyTemplate) => {
return (policyTemplate.inputs || []).map((packageInput) => {
const inputs = getNormalizedInputs(policyTemplate);
return inputs.map((packageInput) => {
const packagePolicyInput = packagePolicy.inputs.find(
(input) =>
input.type === packageInput.type &&
@ -66,7 +72,9 @@ export const StepConfigurePackagePolicy: React.FunctionComponent<{
const packageInputStreams = getStreamsForInputType(
packageInput.type,
packageInfo,
hasIntegrations ? policyTemplate.data_streams : []
hasIntegrations && isIntegrationPolicyTemplate(policyTemplate)
? policyTemplate.data_streams
: []
);
return packagePolicyInput ? (
<EuiFlexItem key={packageInput.type}>

2
x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/hooks/use_get_agent_policy_or_default.tsx
View file

@ -30,7 +30,7 @@ export const DEFAULT_AGENT_POLICY: NewAgentPolicy = Object.freeze({
defaultMessage: 'My first agent policy',
}),
namespace: 'default',
monitoring_enabled: ['logs', 'metrics'],
monitoring_enabled: ['logs', 'metrics'] as NewAgentPolicy['monitoring_enabled'],
});
const sendGetAgentPolicy = async (agentPolicyId: string) => {

10
x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/overview.tsx
View file

@ -10,6 +10,8 @@ import { EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiLink } from '@elas
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { isIntegrationPolicyTemplate } from '../../../../../../../../common/services';
import { useFleetStatus, useStartServices } from '../../../../../../../hooks';
import { isPackageUnverified } from '../../../../../../../services';
import type { PackageInfo, RegistryPolicyTemplate } from '../../../../../types';
@ -78,7 +80,13 @@ export const OverviewPage: React.FC<Props> = memo(({ packageInfo, integrationInf
{isUnverified && <UnverifiedCallout />}
{packageInfo.readme ? (
<Readme
readmePath={integrationInfo?.readme || packageInfo.readme}
readmePath={
integrationInfo &&
isIntegrationPolicyTemplate(integrationInfo) &&
integrationInfo?.readme
? integrationInfo?.readme
: packageInfo.readme
}
packageName={packageInfo.name}
version={packageInfo.version}
/>

22
x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx
View file

@ -26,6 +26,11 @@ import { TrackApplicationView } from '@kbn/usage-collection-plugin/public';
import type { CustomIntegration } from '@kbn/custom-integrations-plugin/common';
import {
isInputOnlyPolicyTemplate,
isIntegrationPolicyTemplate,
} from '../../../../../../../common/services';
import { useStartServices } from '../../../../hooks';
import { pagePathGetters } from '../../../../constants';
@ -132,8 +137,12 @@ function getAllCategoriesFromIntegrations(pkg: PackageListItem) {
return pkg.categories;
}
const allCategories = pkg.policy_templates?.reduce((accumulator, integration) => {
return [...accumulator, ...(integration.categories || [])];
const allCategories = pkg.policy_templates?.reduce((accumulator, policyTemplate) => {
if (isInputOnlyPolicyTemplate(policyTemplate)) {
// input only policy templates do not have categories
return accumulator;
}
return [...accumulator, ...(policyTemplate.categories || [])];
}, pkg.categories || []);
return _.uniq(allCategories);
@ -160,8 +169,13 @@ const packageListToIntegrationsList = (packages: PackageList): PackageList => {
...acc,
topPackage,
...(doesPackageHaveIntegrations(pkg)
? policyTemplates.map((integration) => {
const { name, title, description, icons, categories = [] } = integration;
? policyTemplates.map((policyTemplate) => {
const { name, title, description, icons } = policyTemplate;
const categories =
isIntegrationPolicyTemplate(policyTemplate) && policyTemplate.categories
? policyTemplate.categories
: [];
const allCategories = [...topCategories, ...categories];
return {
...restOfPackage,

1
x-pack/plugins/fleet/public/services/package_verification.test.ts
View file

@ -53,6 +53,7 @@ const createPackage = ({
describe('isPackageUnverified', () => {
describe('When experimental feature is disabled', () => {
beforeEach(() => {
// @ts-ignore don't want to define all experimental features here
mockGet.mockReturnValue({
packageVerification: false,
} as ReturnType<typeof ExperimentalFeaturesService['get']>);

13
x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { getNormalizedDataStreams } from '../../../common/services';
import type {
FullAgentPolicyOutputPermissions,
PackageInfo,
@ -40,7 +42,8 @@ export async function storedPackagePoliciesToAgentPermissions(
const pkg = packageInfoCache.get(pkgToPkgKey(packagePolicy.package))!;
if (!pkg.data_streams || pkg.data_streams.length === 0) {
const dataStreams = getNormalizedDataStreams(pkg);
if (!dataStreams || dataStreams.length === 0) {
return [packagePolicy.name, undefined];
}
@ -51,21 +54,21 @@ export async function storedPackagePoliciesToAgentPermissions(
// - Endpoint doesn't store the `data_stream` metadata in
// `packagePolicy.inputs`, so we will use _all_ data_streams from the
// package.
dataStreamsForPermissions = pkg.data_streams;
dataStreamsForPermissions = dataStreams;
break;
case 'apm':
// - APM doesn't store the `data_stream` metadata in
// `packagePolicy.inputs`, so we will use _all_ data_streams from
// the package.
dataStreamsForPermissions = pkg.data_streams;
dataStreamsForPermissions = dataStreams;
break;
case 'osquery_manager':
// - Osquery manager doesn't store the `data_stream` metadata in
// `packagePolicy.inputs`, so we will use _all_ data_streams from
// the package.
dataStreamsForPermissions = pkg.data_streams;
dataStreamsForPermissions = dataStreams;
break;
default:
@ -73,7 +76,7 @@ export async function storedPackagePoliciesToAgentPermissions(
// `packagePolicy.inputs[].streams[].data_stream`
// - The rest of the metadata needs to be fetched from the
// `data_stream` object in the package. The link is
// `packagePolicy.inputs[].type == pkg.data_streams.streams[].input`
// `packagePolicy.inputs[].type == dataStreams.streams[].input`
// - Some packages (custom logs) have a compiled dataset, stored in
// `input.streams.compiled_stream.data_stream.dataset`
dataStreamsForPermissions = packagePolicy.inputs

4
x-pack/plugins/fleet/server/services/epm/archive/parse.ts
View file

@ -439,6 +439,7 @@ export function parseAndVerifyPolicyTemplates(
title: policyTemplateTitle,
description,
inputs,
input,
multiple,
...restOfProps
} = policyTemplate;
@ -469,8 +470,9 @@ export function parseAndVerifyPolicyTemplates(
name,
title: policyTemplateTitle,
description,
inputs: parsedInputs,
multiple: parsedMultiple,
// template can only have one of input or inputs
...(!input ? { inputs: parsedInputs } : { input }),
} as RegistryPolicyTemplate
)
);

2
x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
View file

@ -366,7 +366,7 @@ export function prepareTemplate({
pkg,
dataStream,
}: {
pkg: Pick<PackageInfo, 'name' | 'version'>;
pkg: Pick<PackageInfo, 'name' | 'version' | 'type'>;
dataStream: RegistryDataStream;
}): { componentTemplates: TemplateMap; indexTemplate: IndexTemplateEntry } {
const { name: packageName, version: packageVersion } = pkg;

2
x-pack/plugins/fleet/server/services/epm/fields/field.ts
View file

@ -278,7 +278,7 @@ const isFields = (path: string) => {
*/
export const loadFieldsFromYaml = (
pkg: Pick<PackageInfo, 'version' | 'name'>,
pkg: Pick<PackageInfo, 'version' | 'name' | 'type'>,
datasetName?: string
): Field[] => {
// Fetch all field definition files

11
x-pack/plugins/fleet/server/services/epm/packages/assets.ts
View file

@ -17,7 +17,7 @@ import type { ArchiveEntry } from '../archive';
// and different package and version structure
export function getAssets(
packageInfo: Pick<PackageInfo, 'version' | 'name'>,
packageInfo: Pick<PackageInfo, 'version' | 'name' | 'type'>,
filter = (path: string): boolean => true,
datasetName?: string
): string[] {
@ -35,7 +35,10 @@ export function getAssets(
// if dataset, filter for them
if (datasetName) {
const comparePath = `${packageInfo.name}-${packageInfo.version}/data_stream/${datasetName}/`;
const comparePath =
packageInfo?.type === 'input'
? `${packageInfo.name}-${packageInfo.version}/agent/input/`
: `${packageInfo.name}-${packageInfo.version}/data_stream/${datasetName}/`;
if (!path.includes(comparePath)) {
continue;
}
@ -49,10 +52,8 @@ export function getAssets(
return assets;
}
// ASK: Does getAssetsData need an installSource now?
// if so, should it be an Installation vs InstallablePackage or add another argument?
export function getAssetsData(
packageInfo: Pick<PackageInfo, 'version' | 'name'>,
packageInfo: Pick<PackageInfo, 'version' | 'name' | 'type'>,
filter = (path: string): boolean => true,
datasetName?: string
): ArchiveEntry[] {

16
x-pack/plugins/fleet/server/services/epm/packages/get.ts
View file

@ -158,7 +158,9 @@ export async function getPackageInfo({
? await Registry.fetchInfo(pkgName, resolvedPkgVersion).catch(() => undefined)
: undefined;
if (packageInfo) {
// We need to get input only packages from source to get all fields
// see https://github.com/elastic/package-registry/issues/864
if (packageInfo && packageInfo.type !== 'input') {
// Fix the paths
paths =
packageInfo.assets?.map((path) =>
@ -170,6 +172,7 @@ export async function getPackageInfo({
pkgVersion: resolvedPkgVersion,
savedObjectsClient,
installedPkg: savedObject?.attributes,
getPkgInfoFromArchive: packageInfo?.type === 'input',
}));
}
@ -238,9 +241,16 @@ export async function getPackageFromSource(options: {
pkgVersion: string;
installedPkg?: Installation;
savedObjectsClient: SavedObjectsClientContract;
getPkgInfoFromArchive?: boolean;
}): Promise<PackageResponse> {
const logger = appContextService.getLogger();
const { pkgName, pkgVersion, installedPkg, savedObjectsClient } = options;
const {
pkgName,
pkgVersion,
installedPkg,
savedObjectsClient,
getPkgInfoFromArchive = true,
} = options;
let res: GetPackageResponse;
// If the package is installed
@ -283,7 +293,7 @@ export async function getPackageFromSource(options: {
}
} else {
// else package is not installed or installed and missing from cache and storage and installed from registry
res = await Registry.getRegistryPackage(pkgName, pkgVersion);
res = await Registry.getRegistryPackage(pkgName, pkgVersion, { getPkgInfoFromArchive });
logger.debug(`retrieved uninstalled package ${pkgName}-${pkgVersion} from registry`);
}
if (!res) {

22
x-pack/plugins/fleet/server/services/epm/registry/index.ts
View file

@ -33,6 +33,7 @@ import {
getVerificationResult,
getPackageInfo,
setPackageInfo,
generatePackageInfoFromArchiveBuffer,
} from '../archive';
import { streamToBuffer, streamToString } from '../streams';
import { appContextService } from '../..';
@ -220,12 +221,13 @@ export async function fetchCategories(
return fetchUrl(url.toString()).then(JSON.parse);
}
export async function getInfo(name: string, version: string) {
export async function getInfo(name: string, version: string, options: { cache?: boolean } = {}) {
const cache = options.cache ?? true;
return withPackageSpan('Fetch package info', async () => {
let packageInfo = getPackageInfo({ name, version });
if (!packageInfo) {
packageInfo = await fetchInfo(name, version);
setPackageInfo({ name, version, packageInfo });
if (cache) setPackageInfo({ name, version, packageInfo });
}
return packageInfo as RegistryPackage;
});
@ -234,7 +236,7 @@ export async function getInfo(name: string, version: string) {
export async function getRegistryPackage(
name: string,
version: string,
options?: { ignoreUnverified?: boolean }
options?: { ignoreUnverified?: boolean; getPkgInfoFromArchive?: boolean }
): Promise<{
paths: string[];
packageInfo: RegistryPackage;
@ -268,6 +270,14 @@ export async function getRegistryPackage(
contentType: ensureContentType(archivePath),
})
);
const cachedInfo = getPackageInfo({ name, version });
if (options?.getPkgInfoFromArchive && !cachedInfo) {
const { packageInfo } = await generatePackageInfoFromArchiveBuffer(
archiveBuffer,
ensureContentType(archivePath)
);
setPackageInfo({ packageInfo, name, version });
}
}
const packageInfo = await getInfo(name, version);
@ -298,7 +308,7 @@ export async function fetchArchiveBuffer({
verificationResult?: PackageVerificationResult;
}> {
const logger = appContextService.getLogger();
const { download: archivePath } = await getInfo(pkgName, pkgVersion);
const { download: archivePath } = await getInfo(pkgName, pkgVersion, { cache: false });
const archiveUrl = `${getRegistryUrl()}${archivePath}`;
const archiveBuffer = await getResponseStream(archiveUrl).then(streamToBuffer);
if (shouldVerify) {
@ -326,7 +336,9 @@ export async function getPackageArchiveSignatureOrUndefined({
pkgVersion: string;
logger: Logger;
}): Promise<string | undefined> {
const { signature_path: signaturePath } = await getInfo(pkgName, pkgVersion);
const { signature_path: signaturePath } = await getInfo(pkgName, pkgVersion, {
cache: false,
});
if (!signaturePath) {
logger.debug(

24
x-pack/plugins/fleet/server/services/package_policy.ts
View file

@ -33,6 +33,9 @@ import {
doesAgentPolicyAlreadyIncludePackage,
validatePackagePolicy,
validationHasErrors,
isInputOnlyPolicyTemplate,
getNormalizedDataStreams,
getNormalizedInputs,
} from '../../common/services';
import {
SO_SEARCH_LIMIT,
@ -1268,16 +1271,20 @@ async function _compilePackagePolicyInput(
)
: pkgInfo.policy_templates?.[0];
if (!input.enabled || !packagePolicyTemplate || !packagePolicyTemplate.inputs?.length) {
if (!input.enabled || !packagePolicyTemplate) {
return undefined;
}
const packageInputs = getNormalizedInputs(packagePolicyTemplate);
if (!packageInputs.length) {
return undefined;
}
const packageInputs = packagePolicyTemplate.inputs;
const packageInput = packageInputs.find((pkgInput) => pkgInput.type === input.type);
if (!packageInput) {
throw new Error(`Input template not found, unable to find input type ${input.type}`);
}
if (!packageInput.template_path) {
return undefined;
}
@ -1357,7 +1364,7 @@ async function _compilePackageStream(
return { ...stream, compiled_stream: undefined };
}
const packageDataStreams = pkgInfo.data_streams;
const packageDataStreams = getNormalizedDataStreams(pkgInfo);
if (!packageDataStreams) {
throw new Error('Stream template not found, no data streams');
}
@ -1529,10 +1536,11 @@ export function updatePackageInputs(
}
// Ignore any inputs removed from this policy template in the new package version
const policyTemplateStillIncludesInput =
policyTemplate.inputs?.some(
(policyTemplateInput) => policyTemplateInput.type === input.type
) ?? false;
const policyTemplateStillIncludesInput = isInputOnlyPolicyTemplate(policyTemplate)
? policyTemplate.type === input.type
: policyTemplate.inputs?.some(
(policyTemplateInput) => policyTemplateInput.type === input.type
) ?? false;
return policyTemplateStillIncludesInput;
}),