[Monitoring] Ignore Duplicate Shards (#21057)

[Monitoring] Ignore Duplicate Shards

This eliminates duplicate shards from the shard table by actively ignoring
them from the response.

x-pack/plugins/monitoring/public/directives/elasticsearch/shard_allocation/index.js

@@ -6,8 +6,8 @@
 
 import { uiModules } from 'ui/modules';
 import { labels } from './lib/labels';
-import { indicesByNodes } from './transformers/indicesByNodes';
-import { nodesByIndices } from './transformers/nodesByIndices';
+import { indicesByNodes } from './transformers/indices_by_nodes';
+import { nodesByIndices } from './transformers/nodes_by_indices';
 import template from './index.html';
 
 const uiModule = uiModules.get('monitoring/directives', []);

x-pack/plugins/monitoring/public/directives/elasticsearch/shard_allocation/lib/decorate_shards.js

@@ -25,7 +25,7 @@ export function decorateShards(shards, nodes) {
   }
 
   return shards.map((shard) => {
-    const node = nodes[shard.resolver];
+    const node = nodes[shard.node];
     shard.nodeName = (node && node.name) || null;
     shard.type = 'shard';
     shard.tooltip_message = getTooltipMessage(shard);

x-pack/plugins/monitoring/public/directives/elasticsearch/shard_allocation/lib/decorate_shards.test.js

@@ -4,26 +4,23 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import expect from 'expect.js';
-import { decorateShards } from '../decorateShards';
+import { decorateShards } from './decorate_shards';
 
 const nodes = {
-  '127.0.0.1:9300': {
+  '8WuXSoE6Q_-etoIhx0R3ag': {
     attributes: {},
     indexCount: 8,
     name: 'node01',
     node_ids: [ '8WuXSoE6Q_-etoIhx0R3ag' ],
-    resolver: '127.0.0.1:9300',
     shardCount: 10,
     transport_address: '127.0.0.1:9300',
     type: 'master'
   },
-  '127.0.0.1:9301': {
+  'ZRnQRUBBQHugqD-rqicFJw': {
     attributes: {},
     indexCount: 7,
     name: 'node02',
     node_ids: [ 'ZRnQRUBBQHugqD-rqicFJw' ],
-    resolver: '127.0.0.1:9301',
     shardCount: 8,
     transport_address: '127.0.0.1:9301',
     type: 'node'
@@ -37,18 +34,16 @@ describe('decorateShards', () => {
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       primary: true,
       relocating_node: 'ZRnQRUBBQHugqD-rqicFJw',
-      resolver: '127.0.0.1:9300',
       shard: 0,
       state: 'RELOCATING'
     };
     const result = decorateShards([ shard ], nodes);
-    expect(result[0]).to.be.eql({
+    expect(result[0]).toMatchObject({
       index: 'test',
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       nodeName: 'node01',
       primary: true,
       relocating_node: 'ZRnQRUBBQHugqD-rqicFJw',
-      resolver: '127.0.0.1:9300',
       shard: 0,
       state: 'RELOCATING',
       tooltip_message: 'Relocating to node02',
@@ -62,19 +57,17 @@ describe('decorateShards', () => {
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       primary: true,
       relocating_node: 'ZRnQRUBBQHugqD-rqicFJw',
-      resolver: '127.0.0.1:9300',
       shard: 0,
       state: 'RELOCATING'
     };
     // pass nodes object with only node01 value
-    const result = decorateShards([ shard ], { '127.0.0.1:9300': nodes['127.0.0.1:9300'] });
-    expect(result[0]).to.be.eql({
+    const result = decorateShards([ shard ], { '8WuXSoE6Q_-etoIhx0R3ag': nodes['8WuXSoE6Q_-etoIhx0R3ag'] });
+    expect(result[0]).toMatchObject({
       index: 'test',
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       nodeName: 'node01',
       primary: true,
       relocating_node: 'ZRnQRUBBQHugqD-rqicFJw',
-      resolver: '127.0.0.1:9300',
       shard: 0,
       state: 'RELOCATING',
       tooltip_message: 'Relocating',
@@ -88,18 +81,16 @@ describe('decorateShards', () => {
       node: 'ZRnQRUBBQHugqD-rqicFJw',
       primary: true,
       relocating_node: null,
-      resolver: '127.0.0.1:9301',
       shard: 3,
       state: 'STARTED'
     };
     const result = decorateShards([ shard ], nodes);
-    expect(result[0]).to.be.eql({
+    expect(result[0]).toMatchObject({
       index: 'test2',
       node: 'ZRnQRUBBQHugqD-rqicFJw',
       nodeName: 'node02',
       primary: true,
       relocating_node: null,
-      resolver: '127.0.0.1:9301',
       shard: 3,
       state: 'STARTED',
       tooltip_message: 'Started',
@@ -113,18 +104,16 @@ describe('decorateShards', () => {
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       primary: false,
       relocating_node: null,
-      resolver: '127.0.0.1:9300',
       shard: 3,
       state: 'INITIALIZING'
     };
     const result = decorateShards([ shard ], nodes);
-    expect(result[0]).to.be.eql({
+    expect(result[0]).toMatchObject({
       index: 'test2',
       node: '8WuXSoE6Q_-etoIhx0R3ag',
       nodeName: 'node01',
       primary: false,
       relocating_node: null,
-      resolver: '127.0.0.1:9300',
       shard: 3,
       state: 'INITIALIZING',
       tooltip_message: 'Initializing',
@@ -142,7 +131,7 @@ describe('decorateShards', () => {
       state: 'UNASSIGNED'
     };
     const result = decorateShards([ shard ], nodes);
-    expect(result[0]).to.be.eql({
+    expect(result[0]).toMatchObject({
       index: 'test2',
       node: null,
       nodeName: null,

x-pack/plugins/monitoring/public/directives/elasticsearch/shard_allocation/transformers/indices_by_nodes.js

@@ -7,7 +7,7 @@
 
 
 import _ from 'lodash';
-import { decorateShards } from '../lib/decorateShards';
+import { decorateShards } from '../lib/decorate_shards';
 
 export function indicesByNodes() {
   return function indicesByNodesFn(shards, nodes) {
@@ -29,8 +29,7 @@ export function indicesByNodes() {
     }
 
     function createNodeAddShard(obj, shard) {
-      const node = shard.resolver;
-      const index = shard.index;
+      const { node, index } = shard;
 
       // If the node is null then it's an unassigned shard and we need to
       // add it to the unassigned array.

x-pack/plugins/monitoring/public/directives/elasticsearch/shard_allocation/transformers/nodes_by_indices.js

@@ -8,7 +8,7 @@
 
 import _ from 'lodash';
 import { hasPrimaryChildren } from '../lib/hasPrimaryChildren';
-import { decorateShards } from '../lib/decorateShards';
+import { decorateShards } from '../lib/decorate_shards';
 
 export function nodesByIndices() {
   return function nodesByIndicesFn(shards, nodes) {
@@ -30,7 +30,7 @@ export function nodesByIndices() {
     }
 
     function createIndexAddShard(obj, shard) {
-      const node = shard.resolver || 'unassigned';
+      const node = shard.node || 'unassigned';
       const index = shard.index;
       if (!obj[node]) {
         createNode(obj, nodes[node], node);

x-pack/plugins/monitoring/server/lib/elasticsearch/nodes/get_node_summary.js

@@ -12,16 +12,16 @@ import { getDefaultNodeFromId } from './get_default_node_from_id';
 import { calculateNodeType } from './calculate_node_type';
 import { getNodeTypeClassLabel } from './get_node_type_class_label';
 
-export function handleResponse(clusterState, shardStats, resolver) {
+export function handleResponse(clusterState, shardStats, nodeUuid) {
   return response => {
     let nodeSummary = {};
     const nodeStatsHits = get(response, 'hits.hits', []);
     const nodes = nodeStatsHits.map(hit => hit._source.source_node); // using [0] value because query results are sorted desc per timestamp
-    const node = nodes[0] || getDefaultNodeFromId(resolver);
+    const node = nodes[0] || getDefaultNodeFromId(nodeUuid);
     const sourceStats = get(response, 'hits.hits[0]._source.node_stats');
-    const clusterNode = get(clusterState, [ 'nodes', resolver ]);
+    const clusterNode = get(clusterState, [ 'nodes', nodeUuid ]);
     const stats = {
-      resolver,
+      resolver: nodeUuid,
       node_ids: nodes.map(node => node.uuid),
       attributes: node.attributes,
       transport_address: node.transport_address,
@@ -30,7 +30,7 @@ export function handleResponse(clusterState, shardStats, resolver) {
     };
 
     if (clusterNode) {
-      const _shardStats = get(shardStats, [ 'nodes', resolver ], {});
+      const _shardStats = get(shardStats, [ 'nodes', nodeUuid ], {});
       const calculatedNodeType = calculateNodeType(stats, get(clusterState, 'master_node')); // set type for labeling / iconography
       const { nodeType, nodeTypeLabel, nodeTypeClass } = getNodeTypeClassLabel(node, calculatedNodeType);
 
@@ -62,17 +62,13 @@ export function handleResponse(clusterState, shardStats, resolver) {
   };
 }
 
-export function getNodeSummary(req, esIndexPattern, clusterState, shardStats, { clusterUuid, resolver, start, end }) {
+export function getNodeSummary(req, esIndexPattern, clusterState, shardStats, { clusterUuid, nodeUuid, start, end }) {
   checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getNodeSummary');
 
-  // Get the params from the POST body for the request
-  const config = req.server.config();
-
   // Build up the Elasticsearch request
-  const resolverKey = config.get('xpack.monitoring.node_resolver');
   const metric = ElasticsearchMetric.getMetricFields();
   const filters = [{
-    term: { [`source_node.${resolverKey}`]: resolver }
+    term: { 'source_node.uuid': nodeUuid }
   }];
   const params = {
     index: esIndexPattern,
@@ -86,6 +82,6 @@ export function getNodeSummary(req, esIndexPattern, clusterState, shardStats, {
 
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
   return callWithRequest(req, 'search', params)
-    .then(handleResponse(clusterState, shardStats, resolver));
+    .then(handleResponse(clusterState, shardStats, nodeUuid));
 }
 

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.js

@@ -9,24 +9,34 @@ import { checkParam } from '../../error_missing_required';
 import { createQuery } from '../../create_query';
 import { ElasticsearchMetric } from '../../metrics';
 
-export function handleResponse(nodeResolver) {
-  return response => {
-    const hits = get(response, 'hits.hits');
-    if (!hits) {
-      return [];
+export function handleResponse(response) {
+  const hits = get(response, 'hits.hits');
+  if (!hits) {
+    return [];
+  }
+
+  // deduplicate any shards from earlier days with the same cluster state state_uuid
+  const uniqueShards = new Set();
+
+  // map into object with shard and source properties
+  return hits.reduce((shards, hit) => {
+    const shard = hit._source.shard;
+
+    if (shard) {
+      // note: if the request is for a node, then it's enough to deduplicate without primary, but for indices it displays both
+      const shardId = `${shard.index}-${shard.shard}-${shard.primary}-${shard.relocating_node}`;
+
+      if (!uniqueShards.has(shardId)) {
+        shards.push(shard);
+        uniqueShards.add(shardId);
+      }
     }
 
-    // map into object with shard and source properties
-    return hits.map(hit => {
-      return {
-        ...hit._source.shard,
-        resolver: get(hit, `_source.source_node[${nodeResolver}]`)
-      };
-    });
-  };
+    return shards;
+  }, []);
 }
 
-export function getShardAllocation(req, esIndexPattern, { nodeResolver, shardFilter, stateUuid, showSystemIndices = false }) {
+export function getShardAllocation(req, esIndexPattern, { shardFilter, stateUuid, showSystemIndices = false }) {
   checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardAllocation');
 
   const filters = [ { term: { state_uuid: stateUuid } }, shardFilter ];
@@ -52,5 +62,5 @@ export function getShardAllocation(req, esIndexPattern, { nodeResolver, shardFil
 
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
   return callWithRequest(req, 'search', params)
-    .then(handleResponse(nodeResolver));
+    .then(handleResponse);
 }

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.test.js

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { handleResponse } from './get_shard_allocation';
+
+describe('get_shard_allocation', () => {
+
+  const exampleShardSource = {
+    cluster_uuid: "Xb_iFZeMSDialSlUwnH53w",
+    state_uuid: "Xobm9shMQGa2p52j-Vh61A",
+    type: "shards",
+    timestamp: "2018-07-05T23:59:51.259Z",
+  };
+
+  const shards = [
+    {
+      node: "X7Cq5UJ9TrS6gWVLItV-0A",
+      index: "my-index-v1",
+      relocating_node: null,
+      state: "STARTED",
+      shard: 0,
+      primary: true,
+    },
+    {
+      node: "Y8Cq5UJ9TrS6gWVLItV-0A",
+      index: "my-index-v1",
+      relocating_node: null,
+      state: "STARTED",
+      shard: 0,
+      primary: false,
+    },
+  ];
+
+  describe('handleResponse', () => {
+
+    it('deduplicates shards', () => {
+      const nextTimestamp = "2018-07-06T00:00:01.259Z";
+      const hits = shards.map(shard => {
+        return {
+          _source: {
+            ...exampleShardSource,
+            shard
+          }
+        };
+      });
+
+      // duplicate all of them; this is how a response would really come back, with only the timestamp changed
+      hits.concat(hits.map(hit => {
+        return {
+          ...hit,
+          timestamp: nextTimestamp,
+        };
+      }));
+
+      const deduplicatedShards = handleResponse({ hits: { hits } });
+
+      expect(deduplicatedShards).toHaveLength(shards.length);
+      deduplicatedShards.forEach((shard, index) => {
+        expect(shard).toMatchObject(shards[index]);
+      });
+    });
+
+  });
+
+});

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stat_aggs.js

@@ -7,9 +7,8 @@
 /*
  * @param {Object} config - Kibana config service
  * @param {Boolean} includeNodes - whether to add the aggs for node shards
- * @param {String} nodeResolver
  */
-export function getShardAggs(config, includeNodes, nodeResolver) {
+export function getShardAggs(config, includeNodes) {
   const maxBucketSize = config.get('xpack.monitoring.max_bucket_size');
   const aggSize = 10;
   const indicesAgg = {
@@ -26,7 +25,7 @@ export function getShardAggs(config, includeNodes, nodeResolver) {
   };
   const nodesAgg = {
     terms: {
-      field: `source_node.${nodeResolver}`,
+      field: 'shard.node',
       size: maxBucketSize
     },
     aggs: {

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.js

@@ -38,7 +38,6 @@ export function getShardStats(req, esIndexPattern, cluster, { includeNodes = fal
   checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getShardStats');
 
   const config = req.server.config();
-  const nodeResolver = config.get('xpack.monitoring.node_resolver');
   const metric = ElasticsearchMetric.getMetricFields();
   const params = {
     index: esIndexPattern,
@@ -53,7 +52,7 @@ export function getShardStats(req, esIndexPattern, cluster, { includeNodes = fal
         filters: [ { term: { state_uuid: get(cluster, 'cluster_state.state_uuid') } } ]
       }),
       aggs: {
-        ...getShardAggs(config, includeNodes, nodeResolver)
+        ...getShardAggs(config, includeNodes)
       }
     }
   };

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index_detail.js

@@ -62,7 +62,6 @@ export function esIndexRoute(server) {
           const shardFilter = { term: { 'shard.index': indexUuid } };
           const stateUuid = get(cluster, 'cluster_state.state_uuid');
           const allocationOptions = {
-            nodeResolver: config.get('xpack.monitoring.node_resolver'),
             shardFilter,
             stateUuid,
             showSystemIndices,

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/node_detail.js

@@ -19,12 +19,12 @@ const { advanced: metricSetAdvanced, overview: metricSetOverview } = metricSets;
 export function esNodeRoute(server) {
   server.route({
     method: 'POST',
-    path: '/api/monitoring/v1/clusters/{clusterUuid}/elasticsearch/nodes/{resolver}',
+    path: '/api/monitoring/v1/clusters/{clusterUuid}/elasticsearch/nodes/{nodeUuid}',
     config: {
       validate: {
         params: Joi.object({
           clusterUuid: Joi.string().required(),
-          resolver: Joi.string().required()
+          nodeUuid: Joi.string().required()
         }),
         payload: Joi.object({
           ccs: Joi.string().optional(),
@@ -42,7 +42,7 @@ export function esNodeRoute(server) {
       const ccs = req.payload.ccs;
       const showSystemIndices = req.payload.showSystemIndices;
       const clusterUuid = req.params.clusterUuid;
-      const resolver = req.params.resolver;
+      const nodeUuid = req.params.nodeUuid;
       const start = req.payload.timeRange.min;
       const end = req.payload.timeRange.max;
       const esIndexPattern = prefixIndexPattern(config, 'xpack.monitoring.elasticsearch.index_pattern', ccs);
@@ -65,20 +65,18 @@ export function esNodeRoute(server) {
 
       try {
         const cluster = await getClusterStats(req, esIndexPattern, clusterUuid);
-        const nodeResolver = config.get('xpack.monitoring.node_resolver');
 
         const clusterState = get(cluster, 'cluster_state', { nodes: {} });
         const shardStats = await getShardStats(req, esIndexPattern, cluster, { includeIndices: true, includeNodes: true });
-        const nodeSummary = await getNodeSummary(req, esIndexPattern, clusterState, shardStats, { clusterUuid, resolver, start, end });
-        const metrics = await getMetrics(req, esIndexPattern, metricSet, [{ term: { [`source_node.${nodeResolver}`]: resolver } }]);
+        const nodeSummary = await getNodeSummary(req, esIndexPattern, clusterState, shardStats, { clusterUuid, nodeUuid, start, end });
+        const metrics = await getMetrics(req, esIndexPattern, metricSet, [{ term: { 'source_node.uuid': nodeUuid } }]);
 
         let shardAllocation;
         if (!isAdvanced) {
           // TODO: Why so many fields needed for a single component (shard legend)?
-          const shardFilter = { term: { [`source_node.${nodeResolver}`]: resolver } };
+          const shardFilter = { term: { 'shard.node': nodeUuid } };
           const stateUuid = get(cluster, 'cluster_state.state_uuid');
           const allocationOptions = {
-            nodeResolver,
             shardFilter,
             stateUuid,
             showSystemIndices,

x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/index_detail.json

@@ -1096,8 +1096,7 @@
       "node": "xcP6ue7eRCieNNitFTT0EA",
       "relocating_node": null,
       "shard": 4,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "xcP6ue7eRCieNNitFTT0EA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1105,8 +1104,7 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "relocating_node": null,
       "shard": 4,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1114,8 +1112,7 @@
       "node": "xcP6ue7eRCieNNitFTT0EA",
       "relocating_node": null,
       "shard": 1,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "xcP6ue7eRCieNNitFTT0EA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1123,8 +1120,7 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "relocating_node": null,
       "shard": 1,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1132,8 +1128,7 @@
       "node": "xcP6ue7eRCieNNitFTT0EA",
       "relocating_node": null,
       "shard": 2,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "xcP6ue7eRCieNNitFTT0EA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1141,8 +1136,7 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "relocating_node": null,
       "shard": 2,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1150,8 +1144,7 @@
       "node": "xcP6ue7eRCieNNitFTT0EA",
       "relocating_node": null,
       "shard": 3,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "xcP6ue7eRCieNNitFTT0EA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1159,8 +1152,7 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "relocating_node": null,
       "shard": 3,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1168,8 +1160,7 @@
       "node": "xcP6ue7eRCieNNitFTT0EA",
       "relocating_node": null,
       "shard": 0,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "xcP6ue7eRCieNNitFTT0EA"
+      "index": "avocado-tweets-2017.10.02"
     },
     {
       "state": "STARTED",
@@ -1177,8 +1168,7 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "relocating_node": null,
       "shard": 0,
-      "index": "avocado-tweets-2017.10.02",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA"
+      "index": "avocado-tweets-2017.10.02"
     }
   ],
   "shardStats": {

x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/node_detail.json

@@ -1223,7 +1223,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 4,
       "state": "STARTED"
     },
@@ -1232,7 +1231,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 1,
       "state": "STARTED"
     },
@@ -1241,7 +1239,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 2,
       "state": "STARTED"
     },
@@ -1250,7 +1247,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 3,
       "state": "STARTED"
     },
@@ -1259,7 +1255,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 4,
       "state": "STARTED"
     },
@@ -1268,7 +1263,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 1,
       "state": "STARTED"
     },
@@ -1277,7 +1271,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": false,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 2,
       "state": "STARTED"
     },
@@ -1286,7 +1279,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 3,
       "state": "STARTED"
     },
@@ -1295,7 +1287,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": false,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 0,
       "state": "STARTED"
     },
@@ -1304,7 +1295,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 4,
       "state": "STARTED"
     },
@@ -1313,7 +1303,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 1,
       "state": "STARTED"
     },
@@ -1322,7 +1311,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 2,
       "state": "STARTED"
     },
@@ -1331,7 +1319,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 3,
       "state": "STARTED"
     },
@@ -1340,7 +1327,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 0,
       "state": "STARTED"
     },
@@ -1349,7 +1335,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 4,
       "state": "STARTED"
     },
@@ -1358,7 +1343,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 2,
       "state": "STARTED"
     },
@@ -1367,7 +1351,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 3,
       "state": "STARTED"
     },
@@ -1376,7 +1359,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 0,
       "state": "STARTED"
     },
@@ -1385,7 +1367,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 4,
       "state": "STARTED"
     },
@@ -1394,7 +1375,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": false,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 1,
       "state": "STARTED"
     },
@@ -1403,7 +1383,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": false,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 2,
       "state": "STARTED"
     },
@@ -1412,7 +1391,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 3,
       "state": "STARTED"
     },
@@ -1421,7 +1399,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": null,
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 0,
       "state": "STARTED"
     },
@@ -1430,7 +1407,6 @@
       "node": "jUT5KdxfRbORSCWkb5zjmA",
       "primary": true,
       "relocating_node": "bwQWH-7IQY-mFPpfoaoFXQ",
-      "resolver": "jUT5KdxfRbORSCWkb5zjmA",
       "shard": 0,
       "state": "RELOCATING"
     }