[8.16] [Automatic Import] Remove pipeline tests from the package (#211223) (#214622)

# Backport This will backport the following commits from `main` to `8.16`: - [[Automatic Import] Remove pipeline tests from the package (#211223)](https://github.com/elastic/kibana/pull/211223)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) \n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sqren/backport)\n\n\n\nCo-authored-by: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>"}},{"branch":"8.18","label":"v8.18.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/211398","number":211398,"state":"MERGED","mergeCommit":{"sha":"158cfac027e31dd6ca46340a775c019d2228d0ce","message":"[8.18] [Automatic Import] Remove pipeline tests from the package (#211223) (#211398)\n\n# Backport\n\nThis will backport the following commits from `main` to `8.18`:\n- [[Automatic Import] Remove pipeline tests from the package\n(#211223)](https://github.com/elastic/kibana/pull/211223)\n\n\n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sorenlouv/backport)\n\n"}},{"branch":"main","label":"v9.1.0","labelRegex":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/211223","number":211223,"mergeCommit":{"message":"[Automatic Import] Remove pipeline tests from the package (#211223)\n\n## Summary\r\n\r\nThis PR removes unused pipeline tests from the packaging of integration.\r\n\r\nThe pipeline tests are not run today when the integration is built.\r\nHence removing them for now.","sha":"54b6e65a2047cdcc727032e1734170e7c8307aff"}},{"branch":"8.x","label":"v8.19.0","labelRegex":"^v8.19.0$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/211393","number":211393,"state":"MERGED","mergeCommit":{"sha":"7f23710f4afb022b15aa1e5a1f25fcdeef2f1b9c","message":"[8.x] [Automatic Import] Remove pipeline tests from the package (#211223) (#211393)\n\n# Backport\n\nThis will backport the following commits from `main` to `8.x`:\n- [[Automatic Import] Remove pipeline tests from the package\n(#211223)](https://github.com/elastic/kibana/pull/211223)\n\n\n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sorenlouv/backport)\n\n"}},{"branch":"8.17","label":"v8.17.3","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/211405","number":211405,"state":"MERGED","mergeCommit":{"sha":"6b2ed55f13b5e6a4a6feed75906716653d2aa6b2","message":"[8.17] [Automatic Import] Remove pipeline tests from the package (#211223) (#211405)\n\n# Backport\n\nThis will backport the following commits from `main` to `8.17`:\n- [[Automatic Import] Remove pipeline tests from the package\n(#211223)](https://github.com/elastic/kibana/pull/211223)\n\n\n\n### Questions ?\nPlease refer to the [Backport tool\ndocumentation](https://github.com/sorenlouv/backport)\n\n"}}]}] BACKPORT--> --------- Co-authored-by: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Co-authored-by: Kylie Meli <kylie.geller@elastic.co>
2025-04-24 01:38:56 -04:00 · 2025-03-14 21:33:07 +01:00 · 2025-03-14 21:33:07 +01:00 · 4772a30390
commit 4772a30390
parent f219410251
20 changed files with 103 additions and 190 deletions
--- a/x-pack/plugins/integration_assistant/common/constants.ts
+++ b/x-pack/plugins/integration_assistant/common/constants.ts
@ -36,6 +36,7 @@ export enum GenerationErrorCode {
  UNSUPPORTED_LOG_SAMPLES_FORMAT = 'unsupported-log-samples-format',
  UNPARSEABLE_CSV_DATA = 'unparseable-csv-data',
  CEF_ERROR = 'cef-not-supported',
+  BUILD_INTEGRATION_ERROR = 'build-integration-error',
 }

 // Size limits
--- a/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
+++ b/x-pack/plugins/integration_assistant/public/components/create_integration/create_integration_assistant/steps/data_stream_step/translations.ts
@ -231,6 +231,12 @@ export const GENERATION_ERROR_TRANSLATION: Record<
    defaultMessage:
      'CEF format detected. Please decode the CEF logs into JSON format using filebeat decode_cef processor.',
  }),
+  [GenerationErrorCode.BUILD_INTEGRATION_ERROR]: i18n.translate(
+    'xpack.integrationAssistant.errors.buildIntegrationError',
+    {
+      defaultMessage: 'An error occurred while building the integration package. Please try again.',
+    }
+  ),
  [GenerationErrorCode.UNPARSEABLE_CSV_DATA]: (attributes) => {
    if (
      attributes.underlyingMessages !== undefined &&
--- a/x-pack/plugins/integration_assistant/server/graphs/ecs/graph.test.ts
+++ b/x-pack/plugins/integration_assistant/server/graphs/ecs/graph.test.ts
@ -85,7 +85,7 @@ describe('EcsGraph', () => {
        throw Error(`getEcsGraph threw an error: ${error}`);
      }

-      expect(response.results).toStrictEqual(ecsMappingExpectedResults);
+      expect(response.results).toEqual(ecsMappingExpectedResults);

      // Check if the functions were called
      expect(handleEcsMapping).toHaveBeenCalled();
--- a/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts
+++ b/x-pack/plugins/integration_assistant/server/integration_builder/build_integration.ts
@ -19,6 +19,7 @@ import { createFieldMapping } from './fields';
 import { createPipeline } from './pipeline';
 import { createReadme } from './readme_files';
 import { Field, flattenObjectsList } from '../util/samples';
+import { BuildIntegrationError } from '../lib/errors/build_integration_error';

 const initialVersion = '1.0.0';

@ -36,45 +37,56 @@ export async function buildPackage(integration: Integration): Promise<Buffer> {
  configureNunjucks();

  if (!isValidName(integration.name)) {
-    throw new Error(
+    throw new BuildIntegrationError(
      `Invalid integration name: ${integration.name}, Should only contain letters, numbers and underscores`
    );
  }
-
  const workingDir = joinPath(getDataPath(), `integration-assistant-${generateUniqueId()}`);
-  const packageDirectoryName = `${integration.name}-${initialVersion}`;
-  const packageDir = createDirectories(workingDir, integration, packageDirectoryName);

-  const dataStreamsDir = joinPath(packageDir, 'data_stream');
-  const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
-    const dataStreamName = dataStream.name;
-    if (!isValidName(dataStreamName)) {
-      throw new Error(
-        `Invalid datastream name: ${dataStreamName}, Should only contain letters, numbers and underscores`
+  try {
+    const packageDirectoryName = `${integration.name}-${initialVersion}`;
+    const packageDir = createDirectories(workingDir, integration, packageDirectoryName);
+
+    const dataStreamsDir = joinPath(packageDir, 'data_stream');
+    const fieldsPerDatastream = integration.dataStreams.map((dataStream) => {
+      const dataStreamName = dataStream.name;
+      if (!isValidName(dataStreamName)) {
+        throw new Error(
+          `Invalid datastream name: ${dataStreamName}, Should only contain letters, numbers and underscores`
+        );
+      }
+      const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);
+
+      const dataStreamFields = createDataStream(
+        integration.name,
+        specificDataStreamDir,
+        dataStream
+      );
+      createAgentInput(specificDataStreamDir, dataStream.inputTypes);
+      createPipeline(specificDataStreamDir, dataStream.pipeline);
+      const fields = createFieldMapping(
+        integration.name,
+        dataStreamName,
+        specificDataStreamDir,
+        dataStream.docs
      );
-    }
-    const specificDataStreamDir = joinPath(dataStreamsDir, dataStreamName);

-    const dataStreamFields = createDataStream(integration.name, specificDataStreamDir, dataStream);
-    createAgentInput(specificDataStreamDir, dataStream.inputTypes);
-    createPipeline(specificDataStreamDir, dataStream.pipeline);
-    const fields = createFieldMapping(
-      integration.name,
-      dataStreamName,
-      specificDataStreamDir,
-      dataStream.docs
-    );
+      return {
+        datastream: dataStreamName,
+        fields: mergeAndSortFields(fields, dataStreamFields),
+      };
+    });

-    return {
-      datastream: dataStreamName,
-      fields: mergeAndSortFields(fields, dataStreamFields),
-    };
-  });
+    createReadme(packageDir, integration.name, fieldsPerDatastream);
+    const zipBuffer = await createZipArchive(integration, workingDir, packageDirectoryName);

-  createReadme(packageDir, integration.name, fieldsPerDatastream);
-  const zipBuffer = await createZipArchive(integration, workingDir, packageDirectoryName);
-  removeDirSync(workingDir);
-  return zipBuffer;
+    removeDirSync(workingDir);
+    return zipBuffer;
+  } catch (error) {
+    throw new BuildIntegrationError('Building the Integration failed');
+  } finally {
+    removeDirSync(workingDir);
+  }
 }
 export function isValidName(input: string): boolean {
  return input.length > 0 && NAME_REGEX_PATTERN.test(input);
--- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts
+++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.test.ts
@ -85,17 +85,6 @@ describe('createDataStream', () => {
    // dataStream files
    expect(copySync).toHaveBeenCalledWith(expect.any(String), `${dataStreamPath}/fields`);

-    // test files
-    expect(ensureDirSync).toHaveBeenCalledWith(`${dataStreamPath}/_dev/test/pipeline`);
-    expect(copySync).toHaveBeenCalledWith(
-      expect.any(String),
-      `${dataStreamPath}/_dev/test/pipeline/test-common-config.yml`
-    );
-    expect(createSync).toHaveBeenCalledWith(
-      `${dataStreamPath}/_dev/test/pipeline/test-${packageName}-datastream-1.log`,
-      samples
-    );
-
    // // Manifest files
    expect(createSync).toHaveBeenCalledWith(`${dataStreamPath}/manifest.yml`, undefined);
    expect(render).toHaveBeenCalledWith(`filestream_manifest.yml.njk`, expect.anything());
--- a/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts
+++ b/x-pack/plugins/integration_assistant/server/integration_builder/data_stream.ts
@ -27,8 +27,6 @@ export function createDataStream(

  ensureDirSync(specificDataStreamDir);
  const fields = createDataStreamFolders(specificDataStreamDir, pipelineDir);
-  createPipelineTests(specificDataStreamDir, dataStream.rawSamples, packageName, dataStreamName);
-
  const dataStreams: string[] = [];
  for (const inputType of dataStream.inputTypes) {
    let mappedValues = {
@ -103,27 +101,3 @@ function loadFieldsFromFiles(sourcePath: string, files: string[]): Field[] {
    return safeLoad(content) as Field[];
  });
 }
-
-function createPipelineTests(
-  specificDataStreamDir: string,
-  rawSamples: string[],
-  packageName: string,
-  dataStreamName: string
-): void {
-  const pipelineTestTemplatesDir = joinPath(__dirname, '../templates/pipeline_tests');
-  const pipelineTestsDir = joinPath(specificDataStreamDir, '_dev/test/pipeline');
-  ensureDirSync(pipelineTestsDir);
-  const items = listDirSync(pipelineTestTemplatesDir);
-  for (const item of items) {
-    const s = joinPath(pipelineTestTemplatesDir, item);
-    const d = joinPath(pipelineTestsDir, item.replaceAll('_', '-'));
-    copySync(s, d);
-  }
-  const formattedPackageName = packageName.replace(/_/g, '-');
-  const formattedDataStreamName = dataStreamName.replace(/_/g, '-');
-  const testFileName = joinPath(
-    pipelineTestsDir,
-    `test-${formattedPackageName}-${formattedDataStreamName}.log`
-  );
-  createSync(testFileName, rawSamples.join('\n'));
-}
--- a/x-pack/plugins/integration_assistant/server/lib/errors/build_integration_error.ts
+++ b/x-pack/plugins/integration_assistant/server/lib/errors/build_integration_error.ts
@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { KibanaResponseFactory } from '@kbn/core/server';
+import { ErrorThatHandlesItsOwnResponse } from './types';
+import { GenerationErrorCode } from '../../../common/constants';
+
+export class BuildIntegrationError extends Error implements ErrorThatHandlesItsOwnResponse {
+  private readonly errorCode: GenerationErrorCode = GenerationErrorCode.BUILD_INTEGRATION_ERROR;
+
+  public sendResponse(res: KibanaResponseFactory) {
+    return res.badRequest({
+      body: { message: this.message, attributes: { errorCode: this.errorCode } },
+    });
+  }
+}
--- a/x-pack/plugins/integration_assistant/server/templates/pipeline_tests/test_common_config.yml
+++ b/x-pack/plugins/integration_assistant/server/templates/pipeline_tests/test_common_config.yml
@ -1,3 +0,0 @@
-fields:
-  tags:
-    - preserve_original_event
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/docker_compose.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/docker_compose.yml.njk
@ -1,3 +0,0 @@
-version: "{{ docker_compose_version }}"
-services: {% for service in services %}
-  {{ service }}{% endfor %}
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/service_filestream.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/service_filestream.njk
@ -1,6 +0,0 @@
-{{package_name}}-{{data_stream_name}}-filestream:
-    image: alpine
-    volumes:
-      - ./sample_logs:/sample_logs:ro
-      - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/service_gcs.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/service_gcs.njk
@ -1,7 +0,0 @@
-{{package_name}}-{{data_stream_name}}-gcs:
-    image: fsouza/fake-gcs-server:latest
-    command: -host=0.0.0.0 -public-host=elastic-package-service_{{package_name}}-{{data_stream_name}}-gcs_1 -port=4443 -scheme=http
-    volumes:
-      - ./sample_logs:/data
-    ports:
-      - 4443/tcp
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/service_logfile.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/service_logfile.njk
@ -1,6 +0,0 @@
-{{package_name}}-{{data_stream_name}}-logfile:
-    image: alpine
-    volumes:
-      - ./sample_logs:/sample_logs:ro
-      - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/service_tcp.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/service_tcp.njk
@ -1,6 +0,0 @@
-{{package_name}}-{{data_stream_name}}-tcp:
-    image: docker.elastic.co/observability/stream:{{stream_version}}
-    volumes:
-      - ./sample_logs:/sample_logs:ro
-    entrypoint: /bin/bash
-    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9025 -p=tcp /sample_logs/{{package_name}}.log"
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/service_udp.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/service_udp.njk
@ -1,6 +0,0 @@
-{{package_name}}-{{data_stream_name}}-udp:
-    image: docker.elastic.co/observability/stream:{{stream_version}}
-    volumes:
-      - ./sample_logs:/sample_logs:ro
-    entrypoint: /bin/bash
-    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:9025 -p=udp /sample_logs/{{package_name}}.log"
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/test_filestream_config.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/test_filestream_config.yml.njk
@ -1,13 +0,0 @@
-service: {{package_name}}-{{data_stream_name}}-filestream
-input: filestream
-data_stream:
-  vars:
-    preserve_original_event: true
-    paths:
-      - '{% raw %}{{SERVICE_LOGS_DIR}}{% endraw %}/test-{{package_name}}-{{data_stream_name}}.log'
-numeric_keyword_fields:
-  - log.file.device_id
-  - log.file.inode
-  - log.file.idxhi
-  - log.file.idxlo
-  - log.file.vol
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/test_gcs_config.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/test_gcs_config.yml.njk
@ -1,10 +0,0 @@
-service: {{package_name}}-{{data_stream_name}}-gcs
-input: gcs
-data_stream:
-  vars:
-    project_id: testproject
-    alternative_host: "http://{% raw %}{{Hostname}}:{{Port}}{% endraw %}"
-    buckets: |
-      - name: testbucket
-        poll: true
-        poll_interval: 15s
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/test_logfile_config.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/test_logfile_config.yml.njk
@ -1,13 +0,0 @@
-service: {{package_name}}-{{data_stream_name}}-logfile
-input: logfile
-data_stream:
-  vars:
-    preserve_original_event: true
-    paths:
-      - '{% raw %}{{SERVICE_LOGS_DIR}}{% endraw %}/{{package_name}}-{{data_stream_name}}.log'
-numeric_keyword_fields:
-  - log.file.device_id
-  - log.file.inode
-  - log.file.idxhi
-  - log.file.idxlo
-  - log.file.vol
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/test_tcp_config.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/test_tcp_config.yml.njk
@ -1,7 +0,0 @@
-service: {{package_name}}-{{data_stream_name}}-tcp
-input: tcp
-data_stream:
-  vars:
-    preserve_original_event: true
-    listen_address: 0.0.0.0
-    listen_port: 9025
--- a/x-pack/plugins/integration_assistant/server/templates/system_tests/test_udp_config.yml.njk
+++ b/x-pack/plugins/integration_assistant/server/templates/system_tests/test_udp_config.yml.njk
@ -1,7 +0,0 @@
-service: {{package_name}}-{{data_stream_name}}-udp
-input: udp
-data_stream:
-  vars:
-    preserve_original_event: true
-    listen_address: 0.0.0.0
-    listen_port: 9025
--- a/x-pack/plugins/integration_assistant/server/util/samples.ts
+++ b/x-pack/plugins/integration_assistant/server/util/samples.ts
@ -153,20 +153,46 @@ export function generateFields(mergedDocs: string): string {
  return yaml.safeDump(fieldsStructure, { sortKeys: false });
 }

+export function isObject(value: any): boolean {
+  return typeof value === 'object' && value !== null;
+}
+
+export function isEmptyValue(value: unknown): boolean {
+  if (value == null) return true;
+  if (isObject(value)) {
+    if (Array.isArray(value)) return value.length === 0;
+    return value && Object.keys(value).length === 0;
+  }
+  return false;
+}
+
+export function isUnsafeProperty(key: string, obj: Record<string, any>): boolean {
+  return (
+    key === '__proto__' || key === 'constructor' || key === 'prototype' || !Object.hasOwn(obj, key)
+  );
+}
+
 export function merge(
  target: Record<string, any>,
  source: Record<string, any>
 ): Record<string, unknown> {
-  const filteredTarget = filterOwnProperties(target);
+  const filteredTarget = Object.create(null);
+
+  for (const [key, targetValue] of Object.entries(target)) {
+    if (!isUnsafeProperty(key, target)) {
+      filteredTarget[key] = targetValue;
+    }
+  }
+
  for (const [key, sourceValue] of Object.entries(source)) {
-    if (!isBuiltInProperties(key, source)) {
+    if (!isUnsafeProperty(key, source)) {
      const targetValue = filteredTarget[key];
+
      if (Array.isArray(sourceValue)) {
-        // Directly assign arrays
-        filteredTarget[key] = sourceValue;
-      } else if (isObject(sourceValue) && !Array.isArray(targetValue)) {
+        filteredTarget[key] = [...sourceValue];
+      } else if (isObject(sourceValue) && !Array.isArray(sourceValue)) {
        if (!isObject(targetValue) || isEmptyValue(targetValue)) {
-          filteredTarget[key] = merge({}, sourceValue);
+          filteredTarget[key] = merge(Object.create(null), sourceValue);
        } else {
          filteredTarget[key] = merge(targetValue, sourceValue);
        }
@ -178,38 +204,10 @@ export function merge(
      }
    }
  }
+
  return filteredTarget;
 }

-function isEmptyValue(value: unknown): boolean {
-  if (value == null) return true;
-  if (isObject(value)) {
-    if (Array.isArray(value)) return value.length === 0;
-    return value && Object.keys(value).length === 0;
-  }
-  return false;
-}
-
-function isObject(value: any): boolean {
-  return typeof value === 'object' && value !== null;
-}
-
-function isBuiltInProperties(key: string, obj: Record<string, any>): boolean {
-  return key === 'constructor' || !Object.prototype.hasOwnProperty.call(obj, key);
-}
-
-function filterOwnProperties(obj: Record<string, any>): Record<string, any> {
-  const ownProps: Record<string, any> = {};
-
-  for (const key of Object.getOwnPropertyNames(obj)) {
-    if (!isBuiltInProperties(key, obj)) {
-      ownProps[key] = (obj as any)[key];
-    }
-  }
-
-  return ownProps;
-}
-
 export function mergeSamples(objects: any[]): string {
  let result: Record<string, unknown> = {};