[esArchiver] combine elasticDump and ScenarioManager (#10359)

* As a part of bringing functional testing to plugins, esArchiver gives these plugins a way to capture and reload es indexes without needing to write a bunch of custom code. It works similarly to the elasticDump and ScenarioManager tools that it replaces.

Differences:
  - Streaming implementation allows for much larger archives
  - CLI for creating and using archives
  - Configurable archive location
  - Stores the data in gzipped files (better for source control, searching, large archives)
  - Automatically identifies and upgrades Kibana config documents

Methods:
  - `#load(name)`: import an archive
  - `#loadIfNeeded(name)`: import an archive, but skip the documents what belong to any existing index
  - `#unload(name)`: delete the indexes stored in an archive

CLI operations:
  - `./bin/es_archiver save <name> [index patterns...]`: save the mapping and documents in one or more indexes that match the wild-card patterns into an the `<name>` archive
  - `./bin/es_archiver load <name>`: load the mapping and documents from the `<name>` archive

* [functional_tests/common/nagivate] check for statusPage

* [es_archiver] move bins into new scripts dir

* [functional_tests/apps/context] use esArchiver

* [esArchiver] general improvements after showing to a few folks

 - remove auto-upgrading config doc logic (until we have better access to kibana version info)
 - export unload command
 - remove preemptive checks in favor of reacting to errors
 - use type "doc" vs "hit" for doc records (consistency)
 - wrote a bunch of pending tests to think though and plan

* [esArchiver] make log a stream that writes to itself

* [esArchiver] fill in stats and archive format tests

* [esArchiver] splitup action logic

* [esArchiver/cli] fix cli --help output and comment

* [esArchiver] remove type-based param coercion

* [esArchiver/log] use strings for log levels

* [esArchvier] remove unused var

* [esArchiver/indexDocRecordsStream] add tests

* [esArchive] fill in remaining tests

* [esArchiver] fix dem tests

* [eslint] remove unused vars

* [esArchiver/loadIfNeeded] fix call to load()

* [esArchiver] remove loadDumpData helpers

test/functional/apps/console/_console.js

@@ -18,7 +18,7 @@ GET _search
 bdd.describe('console app', function describeIndexTests() {
   bdd.before(function () {
     PageObjects.common.debug('navigateTo console');
-    return PageObjects.common.navigateToApp('console', false);
+    return PageObjects.common.navigateToApp('console');
   });
 
   bdd.it('should show the default request', function () {

test/functional/apps/console/index.js

@@ -1,5 +1,9 @@
 
-import { bdd, remote, defaultTimeout } from '../../../support';
+import {
+  bdd,
+  remote,
+  defaultTimeout,
+} from '../../../support';
 
 bdd.describe('console app', function () {
   this.timeout = defaultTimeout;

test/functional/apps/context/index.js

@@ -1,5 +1,9 @@
 
-import { bdd, defaultTimeout, elasticDump, scenarioManager } from '../../../support';
+import {
+  bdd,
+  defaultTimeout,
+  esArchiver,
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 
@@ -8,13 +12,13 @@ bdd.describe('context app', function () {
 
   bdd.before(async function () {
     await PageObjects.remote.setWindowSize(1200,800);
-    await scenarioManager.loadIfEmpty('logstashFunctional');
-    await elasticDump.elasticLoad('visualize','.kibana');
+    await esArchiver.loadIfNeeded('logstash_functional');
+    await esArchiver.load('visualize');
     await PageObjects.common.navigateToApp('discover');
   });
 
   bdd.after(function unloadMakelogs() {
-    return scenarioManager.unload('logstashFunctional');
+    return esArchiver.unload('logstash_functional');
   });
 
   require('./_discover_navigation');

test/functional/apps/dashboard/index.js

@@ -1,5 +1,9 @@
 
-import { bdd, remote, defaultTimeout } from '../../../support';
+import {
+  bdd,
+  remote,
+  defaultTimeout,
+} from '../../../support';
 
 bdd.describe('dashboard app', function () {
   this.timeout = defaultTimeout;

test/functional/apps/discover/_collapse_expand.js

@@ -3,9 +3,8 @@ import expect from 'expect.js';
 
 import {
   bdd,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
 } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -19,11 +18,11 @@ bdd.describe('discover tab', function describeIndexTests() {
     return esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' })
     .then(function loadkibanaIndexPattern() {
       PageObjects.common.debug('load kibana index with default index pattern');
-      return elasticDump.elasticLoad('discover','.kibana');
+      return esArchiver.load('discover');
     })
     // and load a set of makelogs data
     .then(function loadIfEmptyMakelogs() {
-      return scenarioManager.loadIfEmpty('logstashFunctional');
+      return esArchiver.loadIfNeeded('logstash_functional');
     })
     .then(function () {
       PageObjects.common.debug('discover');

test/functional/apps/discover/_discover.js

@@ -3,9 +3,8 @@ import expect from 'expect.js';
 
 import {
   bdd,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
 } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -20,10 +19,10 @@ bdd.describe('discover app', function describeIndexTests() {
     // delete .kibana index and update configDoc
     await esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' });
     PageObjects.common.debug('load kibana index with default index pattern');
-    await elasticDump.elasticLoad('discover','.kibana');
+    await esArchiver.load('discover');
 
     // and load a set of makelogs data
-    await scenarioManager.loadIfEmpty('logstashFunctional');
+    await esArchiver.loadIfNeeded('logstash_functional');
     PageObjects.common.debug('discover');
     await PageObjects.common.navigateToApp('discover');
     PageObjects.common.debug('setAbsoluteRange');

test/functional/apps/discover/_field_data.js

@@ -3,9 +3,8 @@ import expect from 'expect.js';
 
 import {
   bdd,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
 } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -19,11 +18,11 @@ bdd.describe('discover app', function describeIndexTests() {
     return esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' })
     .then(function loadkibanaIndexPattern() {
       PageObjects.common.debug('load kibana index with default index pattern');
-      return elasticDump.elasticLoad('discover','.kibana');
+      return esArchiver.load('discover');
     })
     // and load a set of makelogs data
     .then(function loadIfEmptyMakelogs() {
-      return scenarioManager.loadIfEmpty('logstashFunctional');
+      return esArchiver.loadIfNeeded('logstash_functional');
     })
     .then(function () {
       PageObjects.common.debug('discover');

test/functional/apps/discover/_shared_links.js

@@ -3,9 +3,8 @@ import expect from 'expect.js';
 
 import {
   bdd,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
 } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -33,11 +32,11 @@ bdd.describe('shared links', function describeIndexTests() {
     return esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' })
     .then(function loadkibanaIndexPattern() {
       PageObjects.common.debug('load kibana index with default index pattern');
-      return elasticDump.elasticLoad('discover','.kibana');
+      return esArchiver.load('discover');
     })
     // and load a set of makelogs data
     .then(function loadIfEmptyMakelogs() {
-      return scenarioManager.loadIfEmpty('logstashFunctional');
+      return esArchiver.loadIfNeeded('logstash_functional');
     })
     .then(function () {
       PageObjects.common.debug('discover');

test/functional/apps/discover/_source_filters.js

@@ -1,8 +1,7 @@
 import {
   bdd,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
 } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -19,11 +18,11 @@ bdd.describe('source filters', function describeIndexTests() {
     return esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' })
     .then(function loadkibanaIndexPattern() {
       PageObjects.common.debug('load kibana index with default index pattern');
-      return elasticDump.elasticLoad('visualize_source-filters','.kibana');
+      return esArchiver.load('visualize_source-filters');
     })
     // and load a set of makelogs data
     .then(function loadIfEmptyMakelogs() {
-      return scenarioManager.loadIfEmpty('logstashFunctional');
+      return esArchiver.loadIfNeeded('logstash_functional');
     })
     .then(function () {
       PageObjects.common.debug('discover');

test/functional/apps/discover/index.js

@@ -1,5 +1,9 @@
 
-import { bdd, scenarioManager, defaultTimeout } from '../../../support';
+import {
+  bdd,
+  esArchiver,
+  defaultTimeout,
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 
@@ -11,7 +15,7 @@ bdd.describe('discover app', function () {
   });
 
   bdd.after(function unloadMakelogs() {
-    return scenarioManager.unload('logstashFunctional');
+    return esArchiver.unload('logstash_functional');
   });
 
   require('./_discover');

test/functional/apps/management/_creation_form_changes.js

@@ -1,7 +1,10 @@
 
 import expect from 'expect.js';
 
-import { bdd, esClient } from '../../../support';
+import {
+  bdd,
+  esClient,
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/_index_pattern_create_delete.js

@@ -1,7 +1,11 @@
 
 import expect from 'expect.js';
 
-import { bdd, remote, esClient } from '../../../support';
+import {
+  bdd,
+  remote,
+  esClient
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/_index_pattern_popularity.js

@@ -1,7 +1,10 @@
 
 import expect from 'expect.js';
 
-import { bdd, esClient } from '../../../support';
+import {
+  bdd,
+  esClient
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/_index_pattern_results_sort.js

@@ -1,7 +1,10 @@
 
 import expect from 'expect.js';
 
-import { bdd, esClient } from '../../../support';
+import {
+  bdd,
+  esClient
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/_initial_state.js

@@ -1,7 +1,10 @@
 
 import expect from 'expect.js';
 
-import { bdd, esClient } from '../../../support';
+import {
+  bdd,
+  esClient
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/_kibana_settings.js

@@ -1,7 +1,10 @@
 
 import expect from 'expect.js';
 
-import { bdd, esClient } from '../../../support';
+import {
+  bdd,
+  esClient
+} from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
 

test/functional/apps/management/index.js

@@ -1,4 +1,8 @@
-import { bdd, defaultTimeout, scenarioManager, esClient } from '../../../support';
+import {
+  bdd,
+  defaultTimeout,
+  esArchiver,
+} from '../../../support';
 
 bdd.describe('settings app', function () {
   this.timeout = defaultTimeout;
@@ -6,13 +10,14 @@ bdd.describe('settings app', function () {
   // on setup, we create an settingsPage instance
   // that we will use for all the tests
   bdd.before(async function () {
-    await scenarioManager.unload('logstashFunctional');
-    await scenarioManager.loadIfEmpty('makelogs');
+    await esArchiver.unload('logstash_functional');
+    await esArchiver.load('empty_kibana');
+    await esArchiver.loadIfNeeded('makelogs');
   });
 
   bdd.after(async function () {
-    await scenarioManager.unload('makelogs');
-    await esClient.delete('.kibana');
+    await esArchiver.unload('makelogs');
+    await esArchiver.unload('empty_kibana');
   });
 
   require('./_initial_state');

test/functional/apps/visualize/index.js

@@ -3,9 +3,8 @@ import {
   bdd,
   remote,
   defaultTimeout,
-  scenarioManager,
+  esArchiver,
   esClient,
-  elasticDump
  } from '../../../support';
 
 import PageObjects from '../../../support/page_objects';
@@ -17,12 +16,12 @@ bdd.describe('visualize app', function () {
     remote.setWindowSize(1280,800);
 
     PageObjects.common.debug('Starting visualize before method');
-    const logstash = scenarioManager.loadIfEmpty('logstashFunctional');
+    const logstash = esArchiver.loadIfNeeded('logstash_functional');
     // delete .kibana index and update configDoc
     return esClient.deleteAndUpdateConfigDoc({ 'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*' })
     .then(function loadkibanaIndexPattern() {
       PageObjects.common.debug('load kibana index with default index pattern');
-      return elasticDump.elasticLoad('visualize','.kibana');
+      return esArchiver.load('visualize');
     })
     // wait for the logstash data load to finish if it hasn't already
     .then(function () {