[Security solution] [Endpoint] Fix bad artifact migration (#111294)

* Working test that validate migrated artifact has same properties as SO artifact
* Checks if artifact is compressed and uncompress it if necessary before creating the new one from fleet

Co-authored-by: Paul Tavares <paul.tavares@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/server/services/artifacts/mocks.ts

@@ -12,7 +12,12 @@ import { ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks';
 import type { SearchHit, ESSearchResponse } from '../../../../../../src/core/types/elasticsearch';
 
-import type { Artifact, ArtifactElasticsearchProperties, ArtifactsClientInterface } from './types';
+import type {
+  Artifact,
+  ArtifactElasticsearchProperties,
+  ArtifactsClientInterface,
+  NewArtifact,
+} from './types';
 import { newArtifactToElasticsearchProperties } from './mappings';
 
 export const createArtifactsClientMock = (): jest.Mocked<ArtifactsClientInterface> => {
@@ -77,10 +82,12 @@ export const generateEsRequestErrorApiResponseMock = (
   );
 };
 
-export const generateArtifactEsGetSingleHitMock = (): SearchHit<ArtifactElasticsearchProperties> => {
+export const generateArtifactEsGetSingleHitMock = (
+  artifact?: NewArtifact
+): SearchHit<ArtifactElasticsearchProperties> => {
   const { id, created, ...newArtifact } = generateArtifactMock();
   const _source = {
-    ...newArtifactToElasticsearchProperties(newArtifact),
+    ...newArtifactToElasticsearchProperties(artifact ?? newArtifact),
     created,
   };
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts

@@ -15,14 +15,20 @@ import {
   SavedObjectsFindResponse,
   SavedObjectsFindResult,
 } from 'kibana/server';
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
 import { migrateArtifactsToFleet } from './migrate_artifacts_to_fleet';
 import { createEndpointArtifactClientMock } from '../../services/artifacts/mocks';
-import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock';
+import { InternalArtifactCompleteSchema } from '../../schemas';
+import { generateArtifactEsGetSingleHitMock } from '../../../../../fleet/server/services/artifacts/mocks';
+import { NewArtifact } from '../../../../../fleet/server/services';
+import { CreateRequest } from '@elastic/elasticsearch/api/types';
 
 describe('When migrating artifacts to fleet', () => {
   let soClient: jest.Mocked<SavedObjectsClient>;
   let logger: jest.Mocked<Logger>;
   let artifactClient: ReturnType<typeof createEndpointArtifactClientMock>;
+  /** An artifact that was created prior to 7.14 */
+  let soArtifactEntry: InternalArtifactCompleteSchema;
 
   const createSoFindResult = (
     soHits: SavedObjectsFindResult[] = [],
@@ -41,6 +47,41 @@ describe('When migrating artifacts to fleet', () => {
     soClient = savedObjectsClientMock.create() as jest.Mocked<SavedObjectsClient>;
     logger = loggingSystemMock.create().get() as jest.Mocked<Logger>;
     artifactClient = createEndpointArtifactClientMock();
+    // pre-v7.14 artifact, which is compressed
+    soArtifactEntry = {
+      identifier: 'endpoint-exceptionlist-macos-v1',
+      compressionAlgorithm: 'zlib',
+      encryptionAlgorithm: 'none',
+      decodedSha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+      encodedSha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+      decodedSize: 14,
+      encodedSize: 22,
+      body: 'eJyrVkrNKynKTC1WsoqOrQUAJxkFKQ==',
+    };
+
+    // Mock the esClient create response to include the artifact properties that were provide
+    // to it by fleet artifact client
+    artifactClient._esClient.create.mockImplementation(<T>(props: CreateRequest<T>) => {
+      return elasticsearchServiceMock.createSuccessTransportRequestPromise({
+        ...generateArtifactEsGetSingleHitMock({
+          ...((props?.body ?? {}) as NewArtifact),
+        }),
+        _index: '.fleet-artifacts-7',
+        _id: `endpoint:endpoint-exceptionlist-macos-v1-${
+          // @ts-ignore
+          props?.body?.decodedSha256 ?? 'UNKNOWN?'
+        }`,
+        _version: 1,
+        result: 'created',
+        _shards: {
+          total: 1,
+          successful: 1,
+          failed: 0,
+        },
+        _seq_no: 0,
+        _primary_term: 1,
+      });
+    });
 
     soClient.find.mockResolvedValue(createSoFindResult([], 0)).mockResolvedValueOnce(
       createSoFindResult([
@@ -49,7 +90,7 @@ describe('When migrating artifacts to fleet', () => {
           type: '',
           id: 'abc123',
           references: [],
-          attributes: await getInternalArtifactMock('windows', 'v1'),
+          attributes: soArtifactEntry,
         },
       ])
     );
@@ -70,6 +111,17 @@ describe('When migrating artifacts to fleet', () => {
     expect(soClient.delete).toHaveBeenCalled();
   });
 
+  it('should create artifact in fleet with attributes that match the SO version', async () => {
+    await migrateArtifactsToFleet(soClient, artifactClient, logger);
+
+    await expect(artifactClient.createArtifact.mock.results[0].value).resolves.toEqual(
+      expect.objectContaining({
+        ...soArtifactEntry,
+        compressionAlgorithm: 'zlib',
+      })
+    );
+  });
+
   it('should ignore 404 responses for SO delete (multi-node kibana setup)', async () => {
     const notFoundError: Error & { output?: { statusCode: number } } = new Error('not found');
     notFoundError.output = { statusCode: 404 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts

@@ -5,9 +5,11 @@
  * 2.0.
  */
 
+import { inflate as _inflate } from 'zlib';
+import { promisify } from 'util';
 import { SavedObjectsClient, Logger } from 'kibana/server';
 import { EndpointArtifactClientInterface } from '../../services';
-import { InternalArtifactCompleteSchema } from '../../schemas';
+import { InternalArtifactCompleteSchema, InternalArtifactSchema } from '../../schemas';
 import { ArtifactConstants } from './common';
 
 class ArtifactMigrationError extends Error {
@@ -16,6 +18,12 @@ class ArtifactMigrationError extends Error {
   }
 }
 
+const inflateAsync = promisify(_inflate);
+
+function isCompressed(artifact: InternalArtifactSchema) {
+  return artifact.compressionAlgorithm === 'zlib';
+}
+
 /**
  * With v7.13, artifact storage was moved from a security_solution saved object to a fleet index
  * in order to support Fleet Server.
@@ -57,6 +65,15 @@ export const migrateArtifactsToFleet = async (
       }
 
       for (const artifact of artifactList) {
+        if (isCompressed(artifact.attributes)) {
+          artifact.attributes = {
+            ...artifact.attributes,
+            body: (await inflateAsync(Buffer.from(artifact.attributes.body, 'base64'))).toString(
+              'base64'
+            ),
+          };
+        }
+
         // Create new artifact in fleet index
         await endpointArtifactClient.createArtifact(artifact.attributes);
         // Delete old artifact from SO and if there are errors here, then ignore 404's

x-pack/plugins/security_solution/server/endpoint/services/artifacts/mocks.ts

@@ -8,7 +8,14 @@
 import { SavedObjectsClientContract } from 'src/core/server';
 import { savedObjectsClientMock } from 'src/core/server/mocks';
 import { ManifestClient } from './manifest_client';
-import { EndpointArtifactClientInterface } from './artifact_client';
+import { EndpointArtifactClient, EndpointArtifactClientInterface } from './artifact_client';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { ElasticsearchClientMock } from '../../../../../../../src/core/server/elasticsearch/client/mocks';
+import { elasticsearchServiceMock } from '../../../../../../../src/core/server/mocks';
+// Because mocks are for testing only, should be ok to import the FleetArtifactsClient directly
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { FleetArtifactsClient } from '../../../../../fleet/server/services';
+import { createArtifactsClientMock } from '../../../../../fleet/server/mocks';
 
 export const getManifestClientMock = (
   savedObjectsClient?: SavedObjectsClientContract
@@ -19,10 +26,29 @@ export const getManifestClientMock = (
   return new ManifestClient(savedObjectsClientMock.create(), 'v1');
 };
 
-export const createEndpointArtifactClientMock = (): jest.Mocked<EndpointArtifactClientInterface> => {
+/**
+ * Returns back a mocked EndpointArtifactClient along with the internal FleetArtifactsClient and the Es Clients that are being used
+ * @param esClient
+ */
+export const createEndpointArtifactClientMock = (
+  esClient: ElasticsearchClientMock = elasticsearchServiceMock.createScopedClusterClient()
+    .asInternalUser
+): jest.Mocked<EndpointArtifactClientInterface> & {
+  _esClient: ElasticsearchClientMock;
+} => {
+  const fleetArtifactClientMocked = createArtifactsClientMock();
+  const endpointArtifactClientMocked = new EndpointArtifactClient(fleetArtifactClientMocked);
+
+  // Return the interface mocked with jest.fn() that fowards calls to the real instance
   return {
-    createArtifact: jest.fn(),
-    getArtifact: jest.fn(),
-    deleteArtifact: jest.fn(),
+    createArtifact: jest.fn(async (...args) => {
+      const fleetArtifactClient = new FleetArtifactsClient(esClient, 'endpoint');
+      const endpointArtifactClient = new EndpointArtifactClient(fleetArtifactClient);
+      const response = await endpointArtifactClient.createArtifact(...args);
+      return response;
+    }),
+    getArtifact: jest.fn((...args) => endpointArtifactClientMocked.getArtifact(...args)),
+    deleteArtifact: jest.fn((...args) => endpointArtifactClientMocked.deleteArtifact(...args)),
+    _esClient: esClient,
   };
 };