Make sure shard size stays stable for low number of sizes in TSVB, Lens and Agg based (#139791)

* make sure shard size stays stable for low number of sizes

* add terms test

* adjust test

* fix tests

* add explanatory comment

src/plugins/data/common/search/aggs/buckets/terms.test.ts

@@ -364,6 +364,32 @@ describe('Terms Agg', () => {
       expect(params.order).toEqual({ 'test-orderAgg.50': 'desc' });
     });
 
+    // 25 is the default shard size set for size:10 by Elasticsearch.
+    // Setting it to 25 for every size below 10 makes sure the shard size doesn't change for sizes 1-10, keeping the top terms stable.
+    test('set shard_size to 25 if size is smaller or equal than 10', () => {
+      const aggConfigs = getAggConfigs({
+        field: 'string_field',
+        orderAgg: {
+          type: 'count',
+        },
+        size: 5,
+      });
+      const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+      expect(params.shard_size).toEqual(25);
+    });
+
+    test('do not set shard_size if size is bigger than 10', () => {
+      const aggConfigs = getAggConfigs({
+        field: 'string_field',
+        orderAgg: {
+          type: 'count',
+        },
+        size: 15,
+      });
+      const { [BUCKET_TYPES.TERMS]: params } = aggConfigs.aggs[0].toDsl();
+      expect(params.shard_size).toBeUndefined();
+    });
+
     test('optionally supports shard_size', () => {
       const aggConfigs = getAggConfigs({
         field: 'string_field',

src/plugins/data/common/search/aggs/buckets/terms.ts

@@ -121,7 +121,13 @@ export const getTermsBucketAgg = () =>
       {
         name: 'shardSize',
         write: (aggConfig, output) => {
-          output.params.shard_size = aggConfig.params.shardSize;
+          if (aggConfig.params.shardSize) {
+            output.params.shard_size = aggConfig.params.shardSize;
+          } else if (aggConfig.params.size <= 10) {
+            // 25 is the default shard size set for size:10 by Elasticsearch.
+            // Setting it to 25 for every size below 10 makes sure the shard size doesn't change for sizes 1-10, keeping the top terms stable.
+            output.params.shard_size = 25;
+          }
         },
       },
       {

src/plugins/unified_field_list/common/utils/field_stats_utils.ts

@@ -234,6 +234,9 @@ export async function getStringSamples(
           terms: {
             ...fieldRef,
             size,
+            // 25 is the default shard size set for size:10 by Elasticsearch.
+            // Setting it to 25 for every size below 10 makes sure the shard size doesn't change for sizes 1-10, keeping the top terms stable.
+            shard_size: size <= 10 ? 25 : undefined,
           },
         },
       },

src/plugins/vis_types/timeseries/server/lib/vis_data/request_processors/series/split_by_terms.js

@@ -46,6 +46,11 @@ export function splitByTerms(req, panel, series, esQueryConfig, seriesIndex) {
       }
 
       overwrite(doc, `aggs.${series.id}.${termsType}.size`, series.terms_size);
+      if (series.terms_size <= 10) {
+        // 25 is the default shard size set for size:10 by Elasticsearch.
+        // Setting it to 25 for every size below 10 makes sure the shard size doesn't change for sizes 1-10, keeping the top terms stable.
+        overwrite(doc, `aggs.${series.id}.${termsType}.shard_size`, 25);
+      }
 
       if (metric && metric.type !== 'count' && ~basicAggs.indexOf(metric.type)) {
         const sortAggKey = `${orderByTerms}-SORT`;

src/plugins/vis_types/timeseries/server/lib/vis_data/request_processors/series/split_by_terms.test.js

@@ -59,6 +59,7 @@ describe('splitByTerms', () => {
               _count: 'desc',
             },
             size: 10,
+            shard_size: 25,
           },
         },
       },
@@ -79,6 +80,7 @@ describe('splitByTerms', () => {
               _key: 'asc',
             },
             size: 10,
+            shard_size: 25,
           },
         },
       },
@@ -95,6 +97,7 @@ describe('splitByTerms', () => {
           terms: {
             field: 'host',
             size: 10,
+            shard_size: 25,
             order: {
               'avgmetric-SORT': 'desc',
             },
@@ -126,6 +129,7 @@ describe('splitByTerms', () => {
               "order": Object {
                 "_count": "desc",
               },
+              "shard_size": 25,
               "size": 10,
               "terms": Array [
                 Object {

x-pack/test/functional/apps/lens/group3/time_shift.ts

@@ -60,8 +60,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       expect(await PageObjects.lens.hasFixAction()).to.be(true);
       await PageObjects.lens.useFixAction();
 
-      expect(await PageObjects.lens.getDatatableCellText(1, 2)).to.eql('5,541.5');
-      expect(await PageObjects.lens.getDatatableCellText(1, 3)).to.eql('3,628');
+      expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('6,976');
+      expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('4,182.5');
 
       expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Filters of ip');
     });