Handle another node already having deleted the temporary index (#88332) (#88949)

* Handle another node already having deleted the temporary index

* Make run_multiple_kibana_nodes.sh script more generic

* Add note about dependency on jq

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/core/server/saved_objects/migrationsv2/model.test.ts

@@ -911,7 +911,7 @@ describe('migrations v2 model', () => {
         expect(newState.retryCount).toEqual(0);
         expect(newState.retryDelay).toEqual(0);
       });
-      test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if someone else removed the current alias from the source index', () => {
+      test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if another removed the current alias from the source index', () => {
         const res: ResponseType<'MARK_VERSION_INDEX_READY'> = Either.left({
           type: 'alias_not_found_exception',
         });
@@ -920,6 +920,16 @@ describe('migrations v2 model', () => {
         expect(newState.retryCount).toEqual(0);
         expect(newState.retryDelay).toEqual(0);
       });
+      test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if another node removed the temporary index', () => {
+        const res: ResponseType<'MARK_VERSION_INDEX_READY'> = Either.left({
+          type: 'index_not_found_exception',
+          index: '.kibana_7.11.0_reindex_temp',
+        });
+        const newState = model(markVersionIndexReadyState, res);
+        expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY_CONFLICT');
+        expect(newState.retryCount).toEqual(0);
+        expect(newState.retryDelay).toEqual(0);
+      });
     });
     describe('MARK_VERSION_INDEX_READY_CONFLICT', () => {
       const aliasActions = Option.some([Symbol('alias action')] as unknown) as Option.Some<

src/core/server/saved_objects/migrationsv2/model.ts

@@ -638,12 +638,20 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
         // alias_not_found_exception another instance has completed a
         // migration from the same source.
         return { ...stateP, controlState: 'MARK_VERSION_INDEX_READY_CONFLICT' };
-      } else if (
-        left.type === 'remove_index_not_a_concrete_index' ||
-        left.type === 'index_not_found_exception'
-      ) {
-        // We don't handle these errors as the migration algorithm will never
-        // cause them to occur (these are only relevant to the LEGACY_DELETE
+      } else if (left.type === 'index_not_found_exception') {
+        if (left.index === stateP.tempIndex) {
+          // another instance has already completed the migration and deleted
+          // the temporary index
+          return { ...stateP, controlState: 'MARK_VERSION_INDEX_READY_CONFLICT' };
+        } else {
+          // The migration algorithm will never cause a
+          // index_not_found_exception for an index other than the temporary
+          // index handled above.
+          throwBadResponse(stateP, left as never);
+        }
+      } else if (left.type === 'remove_index_not_a_concrete_index') {
+        // We don't handle this error as the migration algorithm will never
+        // cause it to occur (this error is only relevant to the LEGACY_DELETE
         // step).
         throwBadResponse(stateP, left as never);
       } else {

test/scripts/run_multiple_kibana_nodes.sh

@@ -0,0 +1,86 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/bash
+
+#
+# Script to run multiple kibana nodes in parallel on the same machine.
+# Make sure to run the script from kibana root directory. Some functions depend on the jq command-line utility
+# being installed.
+#
+# bash test/scripts/run_multiple_kibana_nodes.sh <function> [options]
+# functions:
+#   start [instances] [args] - start multiple kibanas (3 default)
+#   es [args] - run elasticsearch
+#   tail - show logs of all kibanas
+#   kill - kills all started kibana processes
+#   clean - clean up nohup files
+#   kibana_index - search .kibana index against es
+#
+
+FN="$1"
+
+if [ "${FN}" == "kill" ]; then
+  echo "killing main processes"
+  for pid in $(cat processes.out); do kill -9 $pid; done
+  echo "killing trailing processes"
+  for pid in $(pgrep -f scripts/kibana); do kill -9 $pid; done
+  exit 0;
+fi
+
+if [ "${FN}" == "tail" ]; then
+  tail -f nohup_*
+  exit 0;
+fi
+
+if [ "${FN}" == "clean" ]; then
+  rm -r nohup_*.out
+  rm processes.out
+  exit 0;
+fi
+
+if [ "${FN}" == "es" ]; then
+  ARGS="$2"
+  yarn es snapshot $ARGS
+  exit 0;
+fi
+
+if [ "${FN}" == "kibana_index" ]; then
+  # search the kibana index
+  curl -XPOST http://elastic:changeme@localhost:9200/.kibana/_search -u elastic:changeme -d '' | jq
+  exit 0;
+fi
+
+if [ "${FN}" == "start" ]; then
+  NUM="$2"
+  ARGS="$3"
+  if test ! "${NUM-}"; then
+    NUM=3
+  fi
+  node scripts/build_kibana_platform_plugins --no-examples
+  rm processes.out
+  for i in $(seq 0 $(expr $NUM - 1))
+  do
+    PORT="56${i}1"
+    PROXY="56${i}3"
+    echo "starting kibana on port $PORT"
+    nohup node scripts/kibana.js --dev.basePathProxyTarget=$PROXY --server.port=$PORT --dev --no-watch --no-optimizer --no-base-path $ARGS > nohup_$i.out &
+    PROCESS_ID=$!
+    echo "${PROCESS_ID}" >> processes.out
+  done
+  exit 0;
+fi