[8.12] [Fleet] Update Fleet's custom ingest pipeline names to avoid collisions + add descriptions to each pipeline (#175448) (#175547)

# Backport

This will backport the following commits from `main` to `8.12`:
- [[Fleet] Update Fleet's custom ingest pipeline names to avoid
collisions + add descriptions to each pipeline
(#175448)](https://github.com/elastic/kibana/pull/175448)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kyle
Pollich","email":"kyle.pollich@elastic.co"},"sourceCommit":{"committedDate":"2024-01-25T14:07:43Z","message":"[Fleet]
Update Fleet's custom ingest pipeline names to avoid collisions + add
descriptions to each pipeline (#175448)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/175254\r\nRef
https://github.com/elastic/kibana/issues/168019\r\nRef
https://github.com/elastic/kibana/pull/170270\r\n\r\nIn 8.12.0, Fleet
unintentionally shipped a breaking change
in\r\nhttps://github.com/elastic/kibana/pull/170270 for APM users who
make use\r\nof a custom `traces-apm` data stream. If a user had
previously defined\r\nthis ingest pipeline to customize documents
ingested for the\r\n`traces-apm` data stream
(defined\r\n[here](9a36183f0b/packages/apm/data_stream/traces/manifest.yml (L2-L3)),\r\nthen
they would unexpectedly see that pipeline called when documents\r\nwere
ingested to the `traces-apm.rum` and `traces-apm.sampled`\r\ndatastreams
as well.\r\n\r\nThis PR addresses this collision by adding a `.package`
suffix to the\r\n\"package level\" ingest pipeline introduced in
8.12.0.\r\n\r\nSo, in 8.12.0 a processor would be defined as such on
the\r\n`traces-apm.rum` or `traces-apm.sampled` ingest
pipeline\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm@custom\",\r\n \"ignore_missing_pipeline\": true,\r\n
}\r\n},\r\n```\r\n\r\nThis PR replaces the pipeline with one that looks
as follows:\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm.package@custom\",\r\n \"ignore_missing_pipeline\":
true,\r\n \"description\": \"[Fleet] Pipeline for all data streams of
type `traces` defined by the `apm` integration\"\r\n
}\r\n},\r\n```\r\n\r\n**To be clear: this is a breaking change if you
have defined the\r\n`traces-apm@custom` integration on 8.12. In 8.12.1,
it will no longer be\r\ncalled for documents ingested to the
`traces-apm`, `traces-apm.rum`, or\r\n`traces-apm.sampled` data streams.
You will need to rename your pipeline\r\nto `traces-apm.package@custom`
to preserve this behavior.**\r\n\r\nThis change also applies to
`logs-elastic_agent.*` ingest pipelines.
See\r\n[this\r\ncomment](https://github.com/elastic/kibana/issues/175254#issuecomment-1906202137)\r\nfor
more information.\r\n\r\nThere is still technically room for a
collision, though it's unlikely,\r\nif the data stream name is
`package`. This will be handled by a package\r\nspec validation proposed
in\r\nhttps://github.com/elastic/package-spec/issues/699.\r\n\r\n---------\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9fe5a66faf4e06fc444c6078edafc29e91126f8d","branchLabelMapping":{"^v8.13.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:breaking","Team:Fleet","backport:prev-minor","v8.12.1","v8.13.0"],"title":"[Fleet]
Update Fleet's custom ingest pipeline names to avoid collisions + add
descriptions to each
pipeline","number":175448,"url":"https://github.com/elastic/kibana/pull/175448","mergeCommit":{"message":"[Fleet]
Update Fleet's custom ingest pipeline names to avoid collisions + add
descriptions to each pipeline (#175448)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/175254\r\nRef
https://github.com/elastic/kibana/issues/168019\r\nRef
https://github.com/elastic/kibana/pull/170270\r\n\r\nIn 8.12.0, Fleet
unintentionally shipped a breaking change
in\r\nhttps://github.com/elastic/kibana/pull/170270 for APM users who
make use\r\nof a custom `traces-apm` data stream. If a user had
previously defined\r\nthis ingest pipeline to customize documents
ingested for the\r\n`traces-apm` data stream
(defined\r\n[here](9a36183f0b/packages/apm/data_stream/traces/manifest.yml (L2-L3)),\r\nthen
they would unexpectedly see that pipeline called when documents\r\nwere
ingested to the `traces-apm.rum` and `traces-apm.sampled`\r\ndatastreams
as well.\r\n\r\nThis PR addresses this collision by adding a `.package`
suffix to the\r\n\"package level\" ingest pipeline introduced in
8.12.0.\r\n\r\nSo, in 8.12.0 a processor would be defined as such on
the\r\n`traces-apm.rum` or `traces-apm.sampled` ingest
pipeline\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm@custom\",\r\n \"ignore_missing_pipeline\": true,\r\n
}\r\n},\r\n```\r\n\r\nThis PR replaces the pipeline with one that looks
as follows:\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm.package@custom\",\r\n \"ignore_missing_pipeline\":
true,\r\n \"description\": \"[Fleet] Pipeline for all data streams of
type `traces` defined by the `apm` integration\"\r\n
}\r\n},\r\n```\r\n\r\n**To be clear: this is a breaking change if you
have defined the\r\n`traces-apm@custom` integration on 8.12. In 8.12.1,
it will no longer be\r\ncalled for documents ingested to the
`traces-apm`, `traces-apm.rum`, or\r\n`traces-apm.sampled` data streams.
You will need to rename your pipeline\r\nto `traces-apm.package@custom`
to preserve this behavior.**\r\n\r\nThis change also applies to
`logs-elastic_agent.*` ingest pipelines.
See\r\n[this\r\ncomment](https://github.com/elastic/kibana/issues/175254#issuecomment-1906202137)\r\nfor
more information.\r\n\r\nThere is still technically room for a
collision, though it's unlikely,\r\nif the data stream name is
`package`. This will be handled by a package\r\nspec validation proposed
in\r\nhttps://github.com/elastic/package-spec/issues/699.\r\n\r\n---------\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9fe5a66faf4e06fc444c6078edafc29e91126f8d"}},"sourceBranch":"main","suggestedTargetBranches":["8.12"],"targetPullRequestStates":[{"branch":"8.12","label":"v8.12.1","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.13.0","branchLabelMappingKey":"^v8.13.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/175448","number":175448,"mergeCommit":{"message":"[Fleet]
Update Fleet's custom ingest pipeline names to avoid collisions + add
descriptions to each pipeline (#175448)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/kibana/issues/175254\r\nRef
https://github.com/elastic/kibana/issues/168019\r\nRef
https://github.com/elastic/kibana/pull/170270\r\n\r\nIn 8.12.0, Fleet
unintentionally shipped a breaking change
in\r\nhttps://github.com/elastic/kibana/pull/170270 for APM users who
make use\r\nof a custom `traces-apm` data stream. If a user had
previously defined\r\nthis ingest pipeline to customize documents
ingested for the\r\n`traces-apm` data stream
(defined\r\n[here](9a36183f0b/packages/apm/data_stream/traces/manifest.yml (L2-L3)),\r\nthen
they would unexpectedly see that pipeline called when documents\r\nwere
ingested to the `traces-apm.rum` and `traces-apm.sampled`\r\ndatastreams
as well.\r\n\r\nThis PR addresses this collision by adding a `.package`
suffix to the\r\n\"package level\" ingest pipeline introduced in
8.12.0.\r\n\r\nSo, in 8.12.0 a processor would be defined as such on
the\r\n`traces-apm.rum` or `traces-apm.sampled` ingest
pipeline\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm@custom\",\r\n \"ignore_missing_pipeline\": true,\r\n
}\r\n},\r\n```\r\n\r\nThis PR replaces the pipeline with one that looks
as follows:\r\n\r\n```\r\n{\r\n \"pipeline\": {\r\n \"name\":
\"traces-apm.package@custom\",\r\n \"ignore_missing_pipeline\":
true,\r\n \"description\": \"[Fleet] Pipeline for all data streams of
type `traces` defined by the `apm` integration\"\r\n
}\r\n},\r\n```\r\n\r\n**To be clear: this is a breaking change if you
have defined the\r\n`traces-apm@custom` integration on 8.12. In 8.12.1,
it will no longer be\r\ncalled for documents ingested to the
`traces-apm`, `traces-apm.rum`, or\r\n`traces-apm.sampled` data streams.
You will need to rename your pipeline\r\nto `traces-apm.package@custom`
to preserve this behavior.**\r\n\r\nThis change also applies to
`logs-elastic_agent.*` ingest pipelines.
See\r\n[this\r\ncomment](https://github.com/elastic/kibana/issues/175254#issuecomment-1906202137)\r\nfor
more information.\r\n\r\nThere is still technically room for a
collision, though it's unlikely,\r\nif the data stream name is
`package`. This will be handled by a package\r\nspec validation proposed
in\r\nhttps://github.com/elastic/package-spec/issues/699.\r\n\r\n---------\r\n\r\nCo-authored-by:
Kibana Machine
<42973632+kibanamachine@users.noreply.github.com>","sha":"9fe5a66faf4e06fc444c6078edafc29e91126f8d"}}]}]
BACKPORT-->

Co-authored-by: Kyle Pollich <kyle.pollich@elastic.co>

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.test.ts

@@ -188,7 +188,7 @@ processors:
     });
 
     expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot(
-      `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true}}]}"`
+      `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test\` dataset\\"}}]}"`
     );
   });
 
@@ -231,12 +231,15 @@ processors:
           - pipeline:
               name: global@custom
               ignore_missing_pipeline: true
+              description: '[Fleet] Global pipeline for all data streams'
           - pipeline:
               name: logs@custom
               ignore_missing_pipeline: true
+              description: '[Fleet] Pipeline for all data streams of type \`logs\`'
           - pipeline:
               name: logs-test.access@custom
               ignore_missing_pipeline: true
+              description: '[Fleet] Pipeline for the \`test.access\` dataset'
           - reroute:
               tag: test.access
               dataset: test.reroute
@@ -280,7 +283,7 @@ processors:
       });
 
       expect(pipelineInstall.contentForInstallation).toMatchInlineSnapshot(
-        `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"`
+        `"{\\"processors\\":[{\\"set\\":{\\"field\\":\\"test\\",\\"value\\":\\"toto\\"}},{\\"pipeline\\":{\\"name\\":\\"global@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Global pipeline for all data streams\\"}},{\\"pipeline\\":{\\"name\\":\\"logs@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for all data streams of type \`logs\`\\"}},{\\"pipeline\\":{\\"name\\":\\"logs-test.access@custom\\",\\"ignore_missing_pipeline\\":true,\\"description\\":\\"[Fleet] Pipeline for the \`test.access\` dataset\\"}},{\\"reroute\\":{\\"tag\\":\\"test.access\\",\\"dataset\\":\\"test.reroute\\",\\"namespace\\":\\"default\\",\\"if\\":\\"true == true\\"}}]}"`
       );
     });
   });

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/helpers.ts

@@ -80,25 +80,30 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor(
     pipeline.dataStream?.routing_rules?.find(
       (rule) => rule.source_dataset === pipeline.dataStream?.dataset
     )?.rules ?? [];
+
   const customPipelineProcessors = [
     {
       pipeline: {
         name: 'global@custom',
         ignore_missing_pipeline: true,
+        description: '[Fleet] Global pipeline for all data streams',
       },
     },
     {
       pipeline: {
         name: `${pipeline.dataStream.type}@custom`,
         ignore_missing_pipeline: true,
+        description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\``,
       },
     },
     ...(pipeline.dataStream.package
       ? [
           {
             pipeline: {
-              name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}@custom`,
+              // This pipeline name gets the `.integration` suffix to avoid conflicts with the pipeline name for the dataset below
+              name: `${pipeline.dataStream.type}-${pipeline.dataStream.package}.integration@custom`,
               ignore_missing_pipeline: true,
+              description: `[Fleet] Pipeline for all data streams of type \`${pipeline.dataStream.type}\` defined by the \`${pipeline.dataStream.package}\` integration`,
             },
           },
         ]
@@ -107,6 +112,7 @@ export function addCustomPipelineAndLocalRoutingRulesProcessor(
       pipeline: {
         name: `${pipeline.dataStream.type}-${pipeline.dataStream.dataset}@custom`,
         ignore_missing_pipeline: true,
+        description: `[Fleet] Pipeline for the \`${pipeline.dataStream.dataset}\` dataset`,
       },
     },
   ];

x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts

@@ -105,12 +105,12 @@ export default function (providerContext: FtrProviderContext) {
         });
 
         await es.ingest.putPipeline({
-          id: `logs-log@custom`,
+          id: `logs-log.integration@custom`,
           processors: [
             {
               append: {
                 field: 'test',
-                value: ['logs-log'],
+                value: ['logs-log.integration'],
               },
             },
           ],
@@ -138,7 +138,7 @@ export default function (providerContext: FtrProviderContext) {
             id: 'logs@custom',
           }),
           es.ingest.deletePipeline({
-            id: 'logs-log@custom',
+            id: 'logs-log.integration@custom',
           }),
           es.ingest.deletePipeline({
             id: CUSTOM_PIPELINE,
@@ -158,7 +158,12 @@ export default function (providerContext: FtrProviderContext) {
           id: res._id,
           index: res._index,
         });
-        expect(doc._source?.test).be.eql(['global', 'logs', 'logs-log', 'logs-log.log']);
+        expect(doc._source?.test).be.eql([
+          'global',
+          'logs',
+          'logs-log.integration',
+          'logs-log.log',
+        ]);
       });
     });
   });