Fixed usage data collection for import and copy (#106305)

src/core/server/core_usage_data/core_usage_data_service.test.ts

@@ -122,6 +122,7 @@ describe('CoreUsageDataService', () => {
           hidden: true,
           namespaceType: 'agnostic',
           mappings: expect.anything(),
+          migrations: expect.anything(),
         });
       });
     });

src/core/server/core_usage_data/core_usage_stats.ts

@@ -8,6 +8,7 @@
 
 import { SavedObjectsType } from '../saved_objects';
 import { CORE_USAGE_STATS_TYPE } from './constants';
+import { migrateTo7141 } from './migrations';
 
 /** @internal */
 export const coreUsageStatsType: SavedObjectsType = {
@@ -18,4 +19,7 @@ export const coreUsageStatsType: SavedObjectsType = {
     dynamic: false, // we aren't querying or aggregating over this data, so we don't need to specify any fields
     properties: {},
   },
+  migrations: {
+    '7.14.1': migrateTo7141,
+  },
 };

src/core/server/core_usage_data/core_usage_stats_client.test.ts

@@ -790,8 +790,14 @@ describe('CoreUsageStatsClient', () => {
         createNewCopies: true,
         overwrite: true,
       } as IncrementSavedObjectsImportOptions);
-      expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1);
+      await usageStatsClient.incrementSavedObjectsImport({
-      expect(repositoryMock.incrementCounter).toHaveBeenCalledWith(
+        request,
+        createNewCopies: false,
+        overwrite: true,
+      } as IncrementSavedObjectsImportOptions);
+      expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(2);
+      expect(repositoryMock.incrementCounter).toHaveBeenNthCalledWith(
+        1,
         CORE_USAGE_STATS_TYPE,
         CORE_USAGE_STATS_ID,
         [
@@ -799,6 +805,19 @@ describe('CoreUsageStatsClient', () => {
           `${IMPORT_STATS_PREFIX}.namespace.default.total`,
           `${IMPORT_STATS_PREFIX}.namespace.default.kibanaRequest.yes`,
           `${IMPORT_STATS_PREFIX}.createNewCopiesEnabled.yes`,
+          // excludes 'overwriteEnabled.yes' and 'overwriteEnabled.no' when createNewCopies is true
+        ],
+        incrementOptions
+      );
+      expect(repositoryMock.incrementCounter).toHaveBeenNthCalledWith(
+        2,
+        CORE_USAGE_STATS_TYPE,
+        CORE_USAGE_STATS_ID,
+        [
+          `${IMPORT_STATS_PREFIX}.total`,
+          `${IMPORT_STATS_PREFIX}.namespace.default.total`,
+          `${IMPORT_STATS_PREFIX}.namespace.default.kibanaRequest.yes`,
+          `${IMPORT_STATS_PREFIX}.createNewCopiesEnabled.no`,
           `${IMPORT_STATS_PREFIX}.overwriteEnabled.yes`,
         ],
         incrementOptions

src/core/server/core_usage_data/core_usage_stats_client.ts

@@ -150,7 +150,7 @@ export class CoreUsageStatsClient {
     const { createNewCopies, overwrite } = options;
     const counterFieldNames = [
       `createNewCopiesEnabled.${createNewCopies ? 'yes' : 'no'}`,
-      `overwriteEnabled.${overwrite ? 'yes' : 'no'}`,
+      ...(!createNewCopies ? [`overwriteEnabled.${overwrite ? 'yes' : 'no'}`] : []), // the overwrite option is ignored when createNewCopies is true
     ];
     await this.updateUsageStats(counterFieldNames, IMPORT_STATS_PREFIX, options);
   }

src/core/server/core_usage_data/migrations.test.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { SavedObjectUnsanitizedDoc } from '../saved_objects';
+import { migrateTo7141 } from './migrations';
+import type { CoreUsageStats } from './types';
+
+const type = 'obj-type';
+const id = 'obj-id';
+
+describe('#migrateTo7141', () => {
+  it('Resets targeted counter fields and leaves others unchanged', () => {
+    const doc = {
+      type,
+      id,
+      attributes: {
+        foo: 'bar',
+        'apiCalls.savedObjectsImport.total': 10,
+      },
+    } as SavedObjectUnsanitizedDoc<CoreUsageStats>;
+
+    expect(migrateTo7141(doc)).toEqual({
+      type,
+      id,
+      attributes: {
+        foo: 'bar',
+        'apiCalls.savedObjectsImport.total': 0,
+        'apiCalls.savedObjectsImport.namespace.default.total': 0,
+        'apiCalls.savedObjectsImport.namespace.default.kibanaRequest.yes': 0,
+        'apiCalls.savedObjectsImport.namespace.default.kibanaRequest.no': 0,
+        'apiCalls.savedObjectsImport.namespace.custom.total': 0,
+        'apiCalls.savedObjectsImport.namespace.custom.kibanaRequest.yes': 0,
+        'apiCalls.savedObjectsImport.namespace.custom.kibanaRequest.no': 0,
+        'apiCalls.savedObjectsImport.createNewCopiesEnabled.yes': 0,
+        'apiCalls.savedObjectsImport.createNewCopiesEnabled.no': 0,
+        'apiCalls.savedObjectsImport.overwriteEnabled.yes': 0,
+        'apiCalls.savedObjectsImport.overwriteEnabled.no': 0,
+      },
+    });
+  });
+});

src/core/server/core_usage_data/migrations.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { cloneDeep } from 'lodash';
+import type { SavedObjectUnsanitizedDoc } from '../saved_objects';
+import type { CoreUsageStats } from './types';
+
+export const migrateTo7141 = (doc: SavedObjectUnsanitizedDoc<CoreUsageStats>) => {
+  try {
+    return resetFields(doc, [
+      // Prior to this, we were counting the `overwrite` option incorrectly; reset all import API counter fields so we get clean data
+      'apiCalls.savedObjectsImport.total',
+      'apiCalls.savedObjectsImport.namespace.default.total',
+      'apiCalls.savedObjectsImport.namespace.default.kibanaRequest.yes',
+      'apiCalls.savedObjectsImport.namespace.default.kibanaRequest.no',
+      'apiCalls.savedObjectsImport.namespace.custom.total',
+      'apiCalls.savedObjectsImport.namespace.custom.kibanaRequest.yes',
+      'apiCalls.savedObjectsImport.namespace.custom.kibanaRequest.no',
+      'apiCalls.savedObjectsImport.createNewCopiesEnabled.yes',
+      'apiCalls.savedObjectsImport.createNewCopiesEnabled.no',
+      'apiCalls.savedObjectsImport.overwriteEnabled.yes',
+      'apiCalls.savedObjectsImport.overwriteEnabled.no',
+    ]);
+  } catch (err) {
+    // fail-safe
+  }
+  return doc;
+};
+
+function resetFields(
+  doc: SavedObjectUnsanitizedDoc<CoreUsageStats>,
+  fieldsToReset: Array<keyof CoreUsageStats>
+) {
+  const newDoc = cloneDeep(doc);
+  const { attributes = {} } = newDoc;
+  for (const field of fieldsToReset) {
+    attributes[field] = 0;
+  }
+  return { ...newDoc, attributes };
+}

x-pack/plugins/spaces/server/saved_objects/migrations/index.ts

@@ -5,4 +5,7 @@
  * 2.0.
  */
 
-export { migrateToKibana660 } from './migrate_6x';
+import * as spaceMigrations from './space_migrations';
+import * as usageStatsMigrations from './usage_stats_migrations';
+
+export { spaceMigrations, usageStatsMigrations };

x-pack/plugins/spaces/server/saved_objects/migrations/space_migrations.test.ts

@@ -5,23 +5,18 @@
  * 2.0.
  */
 
-import type { SavedObjectMigrationContext } from 'src/core/server';
+import type { Space } from 'src/plugins/spaces_oss/common';
 
-import { migrateToKibana660 } from './migrate_6x';
+import { migrateTo660 } from './space_migrations';
-
-const mockContext = {} as SavedObjectMigrationContext;
 
 describe('migrateTo660', () => {
   it('adds a "disabledFeatures" attribute initialized as an empty array', () => {
     expect(
-      migrateToKibana660(
+      migrateTo660({
-        {
+        id: 'space:foo',
-          id: 'space:foo',
+        type: 'space',
-          type: 'space',
+        attributes: {} as Space,
-          attributes: {},
+      })
-        },
-        mockContext
-      )
     ).toEqual({
       id: 'space:foo',
       type: 'space',
@@ -34,16 +29,13 @@ describe('migrateTo660', () => {
   it('does not initialize "disabledFeatures" if the property already exists', () => {
     // This scenario shouldn't happen organically. Protecting against defects in the migration.
     expect(
-      migrateToKibana660(
+      migrateTo660({
-        {
+        id: 'space:foo',
-          id: 'space:foo',
+        type: 'space',
-          type: 'space',
+        attributes: {
-          attributes: {
+          disabledFeatures: ['foo', 'bar', 'baz'],
-            disabledFeatures: ['foo', 'bar', 'baz'],
+        } as Space,
-          },
+      })
-        },
-        mockContext
-      )
     ).toEqual({
       id: 'space:foo',
       type: 'space',

x-pack/plugins/spaces/server/saved_objects/migrations/space_migrations.ts

@@ -5,9 +5,10 @@
  * 2.0.
  */
 
-import type { SavedObjectMigrationFn } from 'src/core/server';
+import type { SavedObjectUnsanitizedDoc } from 'src/core/server';
+import type { Space } from 'src/plugins/spaces_oss/common';
 
-export const migrateToKibana660: SavedObjectMigrationFn<any, any> = (doc) => {
+export const migrateTo660 = (doc: SavedObjectUnsanitizedDoc<Space>) => {
   if (!doc.attributes.hasOwnProperty('disabledFeatures')) {
     doc.attributes.disabledFeatures = [];
   }

x-pack/plugins/spaces/server/saved_objects/migrations/usage_stats_migrations.test.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectUnsanitizedDoc } from 'src/core/server';
+
+import type { UsageStats } from '../../usage_stats';
+import { migrateTo7141 } from './usage_stats_migrations';
+
+const type = 'obj-type';
+const id = 'obj-id';
+
+describe('#migrateTo7141', () => {
+  it('Resets targeted counter fields and leaves others unchanged', () => {
+    const doc = {
+      type,
+      id,
+      attributes: {
+        foo: 'bar',
+        'apiCalls.copySavedObjects.total': 10,
+      },
+    } as SavedObjectUnsanitizedDoc<UsageStats>;
+
+    expect(migrateTo7141(doc)).toEqual({
+      type,
+      id,
+      attributes: {
+        foo: 'bar',
+        'apiCalls.copySavedObjects.total': 0,
+        'apiCalls.copySavedObjects.kibanaRequest.yes': 0,
+        'apiCalls.copySavedObjects.kibanaRequest.no': 0,
+        'apiCalls.copySavedObjects.createNewCopiesEnabled.yes': 0,
+        'apiCalls.copySavedObjects.createNewCopiesEnabled.no': 0,
+        'apiCalls.copySavedObjects.overwriteEnabled.yes': 0,
+        'apiCalls.copySavedObjects.overwriteEnabled.no': 0,
+      },
+    });
+  });
+});

x-pack/plugins/spaces/server/saved_objects/migrations/usage_stats_migrations.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+
+import type { SavedObjectUnsanitizedDoc } from 'src/core/server';
+
+import type { UsageStats } from '../../usage_stats';
+
+export const migrateTo7141 = (doc: SavedObjectUnsanitizedDoc<UsageStats>) => {
+  try {
+    return resetFields(doc, [
+      // Prior to this, we were counting the `overwrite` option incorrectly; reset all copy API counter fields so we get clean data
+      'apiCalls.copySavedObjects.total',
+      'apiCalls.copySavedObjects.kibanaRequest.yes',
+      'apiCalls.copySavedObjects.kibanaRequest.no',
+      'apiCalls.copySavedObjects.createNewCopiesEnabled.yes',
+      'apiCalls.copySavedObjects.createNewCopiesEnabled.no',
+      'apiCalls.copySavedObjects.overwriteEnabled.yes',
+      'apiCalls.copySavedObjects.overwriteEnabled.no',
+    ]);
+  } catch (err) {
+    // fail-safe
+  }
+  return doc;
+};
+
+function resetFields(
+  doc: SavedObjectUnsanitizedDoc<UsageStats>,
+  fieldsToReset: Array<keyof UsageStats>
+) {
+  const newDoc = cloneDeep(doc);
+  const { attributes = {} } = newDoc;
+  for (const field of fieldsToReset) {
+    attributes[field] = 0;
+  }
+  return { ...newDoc, attributes };
+}

x-pack/plugins/spaces/server/saved_objects/saved_objects_service.ts

@@ -10,7 +10,7 @@ import type { CoreSetup } from 'src/core/server';
 import type { SpacesServiceStart } from '../spaces_service';
 import { SPACES_USAGE_STATS_TYPE } from '../usage_stats';
 import { SpacesSavedObjectMappings, UsageStatsMappings } from './mappings';
-import { migrateToKibana660 } from './migrations';
+import { spaceMigrations, usageStatsMigrations } from './migrations';
 import { spacesSavedObjectsClientWrapperFactory } from './saved_objects_client_wrapper_factory';
 
 interface SetupDeps {
@@ -26,7 +26,7 @@ export class SpacesSavedObjectsService {
       namespaceType: 'agnostic',
       mappings: SpacesSavedObjectMappings,
       migrations: {
-        '6.6.0': migrateToKibana660,
+        '6.6.0': spaceMigrations.migrateTo660,
       },
     });
 
@@ -35,6 +35,9 @@ export class SpacesSavedObjectsService {
       hidden: true,
       namespaceType: 'agnostic',
       mappings: UsageStatsMappings,
+      migrations: {
+        '7.14.1': usageStatsMigrations.migrateTo7141,
+      },
     });
 
     core.savedObjects.addClientWrapper(

x-pack/plugins/spaces/server/usage_stats/usage_stats_client.test.ts

@@ -117,14 +117,32 @@ describe('UsageStatsClient', () => {
         createNewCopies: true,
         overwrite: true,
       } as IncrementCopySavedObjectsOptions);
-      expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1);
+      await usageStatsClient.incrementCopySavedObjects({
-      expect(repositoryMock.incrementCounter).toHaveBeenCalledWith(
+        headers: firstPartyRequestHeaders,
+        createNewCopies: false,
+        overwrite: true,
+      } as IncrementCopySavedObjectsOptions);
+      expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(2);
+      expect(repositoryMock.incrementCounter).toHaveBeenNthCalledWith(
+        1,
         SPACES_USAGE_STATS_TYPE,
         SPACES_USAGE_STATS_ID,
         [
           `${COPY_STATS_PREFIX}.total`,
           `${COPY_STATS_PREFIX}.kibanaRequest.yes`,
           `${COPY_STATS_PREFIX}.createNewCopiesEnabled.yes`,
+          // excludes 'overwriteEnabled.yes' and 'overwriteEnabled.no' when createNewCopies is true
+        ],
+        incrementOptions
+      );
+      expect(repositoryMock.incrementCounter).toHaveBeenNthCalledWith(
+        2,
+        SPACES_USAGE_STATS_TYPE,
+        SPACES_USAGE_STATS_ID,
+        [
+          `${COPY_STATS_PREFIX}.total`,
+          `${COPY_STATS_PREFIX}.kibanaRequest.yes`,
+          `${COPY_STATS_PREFIX}.createNewCopiesEnabled.no`,
           `${COPY_STATS_PREFIX}.overwriteEnabled.yes`,
         ],
         incrementOptions

x-pack/plugins/spaces/server/usage_stats/usage_stats_client.ts

@@ -71,7 +71,7 @@ export class UsageStatsClient {
       'total',
       `kibanaRequest.${isKibanaRequest ? 'yes' : 'no'}`,
       `createNewCopiesEnabled.${createNewCopies ? 'yes' : 'no'}`,
-      `overwriteEnabled.${overwrite ? 'yes' : 'no'}`,
+      ...(!createNewCopies ? [`overwriteEnabled.${overwrite ? 'yes' : 'no'}`] : []), // the overwrite option is ignored when createNewCopies is true
     ];
     await this.updateUsageStats(counterFieldNames, COPY_STATS_PREFIX);
   }