[CI] Initial TeamCity implementation (#81043)

.ci/teamcity/bootstrap.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/util.sh"
+
+tc_start_block "Bootstrap"
+
+tc_start_block "yarn install and kbn bootstrap"
+verify_no_git_changes yarn kbn bootstrap --prefer-offline
+tc_end_block "yarn install and kbn bootstrap"
+
+tc_start_block "build kbn-pm"
+verify_no_git_changes yarn kbn run build -i @kbn/pm
+tc_end_block "build kbn-pm"
+
+tc_start_block "build plugin list docs"
+verify_no_git_changes node scripts/build_plugin_list_docs
+tc_end_block "build plugin list docs"
+
+tc_end_block "Bootstrap"

.ci/teamcity/checks/bundle_limits.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+node scripts/build_kibana_platform_plugins --validate-limits

.ci/teamcity/checks/doc_api_changes.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkDocApiChanges

.ci/teamcity/checks/file_casing.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkFileCasing

.ci/teamcity/checks/i18n.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:i18nCheck

.ci/teamcity/checks/licenses.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:licenses

.ci/teamcity/checks/telemetry.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:telemetryCheck

.ci/teamcity/checks/test_hardening.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:test_hardening

.ci/teamcity/checks/ts_projects.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:checkTsProjects

.ci/teamcity/checks/type_check.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:typeCheck

.ci/teamcity/checks/verify_dependency_versions.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyDependencyVersions

.ci/teamcity/checks/verify_notice.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:verifyNotice

.ci/teamcity/ci_stats.js

@@ -0,0 +1,59 @@
+const https = require('https');
+const token = process.env.CI_STATS_TOKEN;
+const host = process.env.CI_STATS_HOST;
+
+const request = (url, options, data = null) => {
+  const httpOptions = {
+    ...options,
+    headers: {
+      ...(options.headers || {}),
+      Authorization: `token ${token}`,
+    },
+  };
+
+  return new Promise((resolve, reject) => {
+    console.log(`Calling https://${host}${url}`);
+
+    const req = https.request(`https://${host}${url}`, httpOptions, (res) => {
+      if (res.statusCode < 200 || res.statusCode >= 300) {
+        return reject(new Error(`Status Code: ${res.statusCode}`));
+      }
+
+      const data = [];
+      res.on('data', (d) => {
+        data.push(d);
+      })
+
+      res.on('end', () => {
+        try {
+          let resp = Buffer.concat(data).toString();
+
+          try {
+            if (resp.trim()) {
+              resp = JSON.parse(resp);
+            }
+          } catch (ex) {
+            console.error(ex);
+          }
+
+          resolve(resp);
+        } catch (ex) {
+          reject(ex);
+        }
+      });
+    })
+
+    req.on('error', reject);
+
+    if (data) {
+      req.write(JSON.stringify(data));
+    }
+
+    req.end();
+  });
+}
+
+module.exports = {
+  get: (url) => request(url, { method: 'GET' }),
+  post: (url, data) => request(url, { method: 'POST' }, data),
+}

.ci/teamcity/ci_stats_complete.js

@@ -0,0 +1,18 @@
+const ciStats = require('./ci_stats');
+
+// This might be better as an API call in the future.
+// Instead, it relies on a separate step setting the BUILD_STATUS env var. BUILD_STATUS is not something provided by TeamCity.
+const BUILD_STATUS = process.env.BUILD_STATUS === 'SUCCESS' ? 'SUCCESS' : 'FAILURE';
+
+(async () => {
+  try {
+    if (process.env.CI_STATS_BUILD_ID) {
+      await ciStats.post(`/v1/build/_complete?id=${process.env.CI_STATS_BUILD_ID}`, {
+        result: BUILD_STATUS,
+      });
+    }
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/default/accessibility.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-accessibility
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/accessibility/config.ts

.ci/teamcity/default/build.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+export KBN_NP_PLUGINS_BUILT=true
+
+tc_start_block "Build Default Distribution"
+
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$KIBANA_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+tc_end_block "Build Default Distribution"

.ci/teamcity/default/build_plugins.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
+  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
+  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
+  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+tc_set_env KBN_NP_PLUGINS_BUILT true

.ci/teamcity/default/ci_group.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export CI_GROUP="$1"
+export JOB=kibana-default-ciGroup${CI_GROUP}
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Default Distro Chrome Functional tests / Group ${CI_GROUP}" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "ciGroup$CI_GROUP"

.ci/teamcity/default/firefox.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-firefoxSmoke
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack firefox smoke test" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "includeFirefox" \
+    --config test/functional/config.firefox.js \
+    --config test/functional_embedded/config.firefox.ts

.ci/teamcity/default/saved_object_field_metrics.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-savedObjectFieldMetrics
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Capture Kibana Saved Objects field count metrics" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/saved_objects_field_count/config.ts

.ci/teamcity/default/security_solution.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-default-securitySolution
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-default"
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Security Solution Cypress Tests" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+   --config test/security_solution_cypress/cli_config.ts

.ci/teamcity/es_snapshots/build.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+cd ..
+destination="$(pwd)/es-build"
+mkdir -p "$destination"
+
+cd elasticsearch
+
+# These turn off automation in the Elasticsearch repo
+export BUILD_NUMBER=""
+export JENKINS_URL=""
+export BUILD_URL=""
+export JOB_NAME=""
+export NODE_NAME=""
+
+# Reads the ES_BUILD_JAVA env var out of .ci/java-versions.properties and exports it
+export "$(grep '^ES_BUILD_JAVA' .ci/java-versions.properties | xargs)"
+
+export PATH="$HOME/.java/$ES_BUILD_JAVA/bin:$PATH"
+export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+
+tc_start_block "Build Elasticsearch"
+./gradlew -Dbuild.docker=true assemble --parallel
+tc_end_block "Build Elasticsearch"
+
+tc_start_block "Create distribution archives"
+find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \;
+tc_end_block "Create distribution archives"
+
+ls -alh "$destination"
+
+tc_start_block "Create docker image archives"
+docker images "docker.elastic.co/elasticsearch/elasticsearch"
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
+tc_end_block "Create docker image archives"
+
+cd "$destination"
+
+find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;
+ls -alh "$destination"

.ci/teamcity/es_snapshots/create_manifest.js

@@ -0,0 +1,82 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+(async () => {
+  const destination = process.argv[2] || __dirname + '/test';
+
+  let ES_BRANCH = process.env.ELASTICSEARCH_BRANCH;
+  let GIT_COMMIT = process.env.ELASTICSEARCH_GIT_COMMIT;
+  let GIT_COMMIT_SHORT = execSync(`git rev-parse --short '${GIT_COMMIT}'`).toString().trim();
+
+  let VERSION = '';
+  let SNAPSHOT_ID = '';
+  let DESTINATION = '';
+
+  const now = new Date()
+
+  // format: yyyyMMdd-HHmmss
+  const date = [
+    now.getFullYear(),
+    (now.getMonth()+1).toString().padStart(2, '0'),
+    now.getDate().toString().padStart(2, '0'),
+    '-',
+    now.getHours().toString().padStart(2, '0'),
+    now.getMinutes().toString().padStart(2, '0'),
+    now.getSeconds().toString().padStart(2, '0'),
+  ].join('')
+
+  try {
+    const files = fs.readdirSync(destination);
+    const manifestEntries = files
+      .filter(f => !f.match(/.sha512$/))
+      .filter(f => !f.match(/.json$/))
+      .map(filename => {
+        const parts = filename.replace("elasticsearch-oss", "oss").split("-")
+
+        VERSION = VERSION || parts[1];
+        SNAPSHOT_ID = SNAPSHOT_ID || `${date}_${GIT_COMMIT_SHORT}`;
+        DESTINATION = DESTINATION || `${VERSION}/archives/${SNAPSHOT_ID}`;
+
+        return {
+          filename: filename,
+          checksum: filename + '.sha512',
+          url: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily-teamcity/${DESTINATION}/${filename}`,
+          version: parts[1],
+          platform: parts[3],
+          architecture: parts[4].split('.')[0],
+          license: parts[0] == 'oss' ? 'oss' : 'default',
+        }
+      });
+
+    const manifest = {
+      id: SNAPSHOT_ID,
+      bucket: `kibana-ci-es-snapshots-daily-teamcity/${DESTINATION}`.toString(),
+      branch: ES_BRANCH,
+      sha: GIT_COMMIT,
+      sha_short: GIT_COMMIT_SHORT,
+      version: VERSION,
+      generated: now.toISOString(),
+      archives: manifestEntries,
+    };
+
+    const manifestJSON = JSON.stringify(manifest, null, 2);
+    fs.writeFileSync(`${destination}/manifest.json`, manifestJSON);
+
+    execSync(`
+      set -euo pipefail
+      cd "${destination}"
+      gsutil -m cp -r *.* gs://kibana-ci-es-snapshots-daily-teamcity/${DESTINATION}
+      cp manifest.json manifest-latest.json
+      gsutil cp manifest-latest.json gs://kibana-ci-es-snapshots-daily-teamcity/${VERSION}
+    `, { shell: '/bin/bash' });
+
+    console.log(`##teamcity[setParameter name='env.ES_SNAPSHOT_MANIFEST' value='https://storage.googleapis.com/kibana-ci-es-snapshots-daily-teamcity/${DESTINATION}/manifest.json']`);
+    console.log(`##teamcity[setParameter name='env.ES_SNAPSHOT_VERSION' value='${VERSION}']`);
+    console.log(`##teamcity[setParameter name='env.ES_SNAPSHOT_ID' value='${SNAPSHOT_ID}']`);
+
+    console.log(`##teamcity[buildNumber '{build.number}-${VERSION}-${SNAPSHOT_ID}']`);
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/es_snapshots/promote_manifest.js

@@ -0,0 +1,53 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+const BASE_BUCKET_DAILY = 'kibana-ci-es-snapshots-daily-teamcity';
+const BASE_BUCKET_PERMANENT = 'kibana-ci-es-snapshots-daily-teamcity/permanent';
+
+(async () => {
+  try {
+    const MANIFEST_URL = process.argv[2];
+
+    if (!MANIFEST_URL) {
+      throw Error('Manifest URL missing');
+    }
+
+    if (!fs.existsSync('snapshot-promotion')) {
+      fs.mkdirSync('snapshot-promotion');
+    }
+    process.chdir('snapshot-promotion');
+
+    execSync(`curl '${MANIFEST_URL}' > manifest.json`);
+
+    const manifest = JSON.parse(fs.readFileSync('manifest.json'));
+    const { id, bucket, version } = manifest;
+
+    console.log(`##teamcity[buildNumber '{build.number}-${version}-${id}']`);
+
+    const manifestPermanent = {
+      ...manifest,
+      bucket: bucket.replace(BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT),
+    };
+
+    fs.writeFileSync('manifest-permanent.json', JSON.stringify(manifestPermanent, null, 2));
+
+    execSync(
+      `
+      set -euo pipefail
+
+      cp manifest.json manifest-latest-verified.json
+      gsutil cp manifest-latest-verified.json gs://${BASE_BUCKET_DAILY}/${version}/
+
+      rm manifest.json
+      cp manifest-permanent.json manifest.json
+      gsutil -m cp -r gs://${bucket}/* gs://${BASE_BUCKET_PERMANENT}/${version}/
+      gsutil cp manifest.json gs://${BASE_BUCKET_PERMANENT}/${version}/
+
+    `,
+      { shell: '/bin/bash' }
+    );
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/oss/accessibility.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-oss-accessibility
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-oss"
+
+checks-reporter-with-killswitch "Kibana accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --config test/accessibility/config.ts

.ci/teamcity/oss/build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins"
+node scripts/build_kibana_platform_plugins \
+  --oss \
+  --filter '!alertingExample' \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins"
+
+export KBN_NP_PLUGINS_BUILT=true
+
+tc_start_block "Build OSS Distribution"
+node scripts/build --debug --oss
+
+# Renaming the build directory to a static one, so that we can put a static one in the TeamCity artifact rules
+mv build/oss/kibana-*-SNAPSHOT-linux-x86_64 build/oss/kibana-build-oss
+tc_end_block "Build OSS Distribution"

.ci/teamcity/oss/build_plugins.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+tc_start_block "Build Platform Plugins - OSS"
+
+node scripts/build_kibana_platform_plugins \
+  --oss \
+  --filter '!alertingExample' \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
+  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
+  --verbose
+tc_end_block "Build Platform Plugins - OSS"

.ci/teamcity/oss/ci_group.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export CI_GROUP="$1"
+export JOB="kibana-ciGroup$CI_GROUP"
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-oss"
+
+checks-reporter-with-killswitch "Functional tests / Group $CI_GROUP" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "ciGroup$CI_GROUP"

.ci/teamcity/oss/firefox.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-firefoxSmoke
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-oss"
+
+checks-reporter-with-killswitch "Firefox smoke test" \
+  node scripts/functional_tests \
+    --bail --debug \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+    --include-tag "includeFirefox" \
+    --config test/functional/config.firefox.js

.ci/teamcity/oss/plugin_functional.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+export JOB=kibana-oss-pluginFunctional
+export KIBANA_INSTALL_DIR="$PARENT_DIR/build/kibana-build-oss"
+
+cd test/plugin_functional/plugins/kbn_sample_panel_action
+if [[ ! -d "target" ]]; then
+  yarn build
+fi
+cd -
+
+yarn run grunt run:pluginFunctionalTestsRelease --from=source
+yarn run grunt run:exampleFunctionalTestsRelease --from=source
+yarn run grunt run:interpreterFunctionalTestsRelease

.ci/teamcity/setup_ci_stats.js

@@ -0,0 +1,33 @@
+const ciStats = require('./ci_stats');
+
+(async () => {
+  try {
+    const build = await ciStats.post('/v1/build', {
+      jenkinsJobName: process.env.TEAMCITY_BUILDCONF_NAME,
+      jenkinsJobId: process.env.TEAMCITY_BUILD_ID,
+      jenkinsUrl: process.env.TEAMCITY_BUILD_URL,
+      prId: process.env.GITHUB_PR_NUMBER || null,
+    });
+
+    const config = {
+      apiUrl: `https://${process.env.CI_STATS_HOST}`,
+      apiToken: process.env.CI_STATS_TOKEN,
+      buildId: build.id,
+    };
+
+    const configJson = JSON.stringify(config);
+    process.env.KIBANA_CI_STATS_CONFIG = configJson;
+    console.log(`\n##teamcity[setParameter name='env.KIBANA_CI_STATS_CONFIG' display='hidden' password='true' value='${configJson}']\n`);
+    console.log(`\n##teamcity[setParameter name='env.CI_STATS_BUILD_ID' value='${build.id}']\n`);
+
+    await ciStats.post(`/v1/git_info?buildId=${build.id}`, {
+      branch: process.env.GIT_BRANCH.replace(/^(refs\/heads\/|origin\/)/, ''),
+      commit: process.env.GIT_COMMIT,
+      targetBranch: process.env.GITHUB_PR_TARGET_BRANCH || null,
+      mergeBase: null, // TODO
+    });
+  } catch (ex) {
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.ci/teamcity/setup_env.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/util.sh"
+
+tc_set_env KIBANA_DIR "$(cd "$(dirname "$0")/../.." && pwd)"
+tc_set_env XPACK_DIR "$KIBANA_DIR/x-pack"
+
+tc_set_env CACHE_DIR "$HOME/.kibana"
+tc_set_env PARENT_DIR "$(cd "$KIBANA_DIR/.."; pwd)"
+tc_set_env WORKSPACE "${WORKSPACE:-$PARENT_DIR}"
+
+tc_set_env KIBANA_PKG_BRANCH "$(jq -r .branch "$KIBANA_DIR/package.json")"
+tc_set_env KIBANA_BASE_BRANCH "$KIBANA_PKG_BRANCH"
+
+tc_set_env GECKODRIVER_CDNURL "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+tc_set_env CHROMEDRIVER_CDNURL "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+tc_set_env RE2_DOWNLOAD_MIRROR "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
+tc_set_env CYPRESS_DOWNLOAD_MIRROR "https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
+
+tc_set_env NODE_OPTIONS "${NODE_OPTIONS:-} --max-old-space-size=4096"
+
+tc_set_env FORCE_COLOR 1
+tc_set_env TEST_BROWSER_HEADLESS 1
+
+tc_set_env ELASTIC_APM_ENVIRONMENT ci
+
+if [[ "${KIBANA_CI_REPORTER_KEY_BASE64-}" ]]; then
+  tc_set_env KIBANA_CI_REPORTER_KEY "$(echo "$KIBANA_CI_REPORTER_KEY_BASE64" | base64 -d)"
+fi
+
+if is_pr; then
+  tc_set_env CHECKS_REPORTER_ACTIVE true
+
+  # These can be removed once we're not supporting Jenkins and TeamCity at the same time
+  # These are primarily used by github checks reporter and can be configured via /github_checks_api.json
+  tc_set_env ghprbGhRepository "elastic/kibana" # TODO?
+  tc_set_env ghprbActualCommit "$GITHUB_PR_TRIGGERED_SHA"
+  tc_set_env BUILD_URL "$TEAMCITY_BUILD_URL"
+else
+  tc_set_env CHECKS_REPORTER_ACTIVE false
+fi
+
+tc_set_env FLEET_PACKAGE_REGISTRY_PORT 6104 # Any unused port is fine, used by ingest manager tests
+
+if [[ "$(which google-chrome-stable)" || "$(which google-chrome)" ]]; then
+  echo "Chrome detected, setting DETECT_CHROMEDRIVER_VERSION=true"
+  tc_set_env DETECT_CHROMEDRIVER_VERSION true
+  tc_set_env CHROMEDRIVER_FORCE_DOWNLOAD true
+else
+  echo "Chrome not detected, installing default chromedriver binary for the package version"
+fi

.ci/teamcity/setup_node.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/util.sh"
+
+tc_start_block "Setup Node"
+
+tc_set_env NODE_VERSION "$(cat "$KIBANA_DIR/.node-version")"
+tc_set_env NODE_DIR "$CACHE_DIR/node/$NODE_VERSION"
+tc_set_env NODE_BIN_DIR "$NODE_DIR/bin"
+tc_set_env YARN_OFFLINE_CACHE "$CACHE_DIR/yarn-offline-cache"
+
+if [[ ! -d "$NODE_DIR" ]]; then
+  nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz"
+
+  echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl"
+
+  mkdir -p "$NODE_DIR"
+  curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
+else
+  echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using"
+  ls -alh "$NODE_BIN_DIR"
+fi
+
+tc_set_env PATH "$NODE_BIN_DIR:$PATH"
+
+tc_end_block "Setup Node"
+tc_start_block "Setup Yarn"
+
+tc_set_env YARN_VERSION "$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")"
+
+if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
+  npm install -g "yarn@^${YARN_VERSION}"
+fi
+
+yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"
+
+tc_set_env YARN_GLOBAL_BIN "$(yarn global bin)"
+tc_set_env PATH "$PATH:$YARN_GLOBAL_BIN"
+
+tc_end_block "Setup Yarn"

.ci/teamcity/tests/mocha.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:mocha

.ci/teamcity/tests/test_hardening.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:test_hardening

.ci/teamcity/tests/test_projects.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+yarn run grunt run:test_projects

.ci/teamcity/tests/xpack_list_cyclic_dependency.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+cd x-pack
+checks-reporter-with-killswitch "X-Pack List cyclic dependency test" node plugins/lists/scripts/check_circular_deps

.ci/teamcity/tests/xpack_siem_cyclic_dependency.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/../util.sh"
+
+cd x-pack
+checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node plugins/security_solution/scripts/check_circular_deps

.ci/teamcity/util.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+tc_escape() {
+  escaped="$1"
+
+  # See https://www.jetbrains.com/help/teamcity/service-messages.html#Escaped+values
+
+  escaped="$(echo "$escaped" | sed -z 's/|/||/g')"
+  escaped="$(echo "$escaped" | sed -z "s/'/|'/g")"
+  escaped="$(echo "$escaped" | sed -z 's/\[/|\[/g')"
+  escaped="$(echo "$escaped" | sed -z 's/\]/|\]/g')"
+  escaped="$(echo "$escaped" | sed -z 's/\n/|n/g')"
+  escaped="$(echo "$escaped" | sed -z 's/\r/|r/g')"
+
+  echo "$escaped"
+}
+
+# Sets up an environment variable locally, and also makes it available for subsequent steps in the build
+# NOTE: env vars set up this way will be visible in the UI when logged in unless you set them up as blank password parameters ahead of time.
+tc_set_env() {
+  export "$1"="$2"
+  echo "##teamcity[setParameter name='env.$1' value='$(tc_escape "$2")']"
+}
+
+verify_no_git_changes() {
+  RED='\033[0;31m'
+  C_RESET='\033[0m' # Reset color
+
+  "$@"
+
+  GIT_CHANGES="$(git ls-files --modified)"
+  if [ "$GIT_CHANGES" ]; then
+    echo -e "\n${RED}ERROR: '$*' caused changes to the following files:${C_RESET}\n"
+    echo -e "$GIT_CHANGES\n"
+    exit 1
+  fi
+}
+
+tc_start_block() {
+  echo "##teamcity[blockOpened name='$1']"
+}
+
+tc_end_block() {
+  echo "##teamcity[blockClosed name='$1']"
+}
+
+checks-reporter-with-killswitch() {
+  if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
+    yarn run github-checks-reporter "$@"
+  else
+    arguments=("$@");
+    "${arguments[@]:1}";
+  fi
+}
+
+is_pr() {
+  [[ "${GITHUB_PR_NUMBER-}" ]] && return
+  false
+}
+
+# This function is specifcally for retrying test runner steps one time
+# A different solution should be used for retrying general steps (e.g. bootstrap)
+tc_retry() {
+  tc_start_block "Retryable Step - Attempt #1"
+  "$@" || {
+    tc_end_block "Retryable Step - Attempt #1"
+    tc_start_block "Retryable Step - Attempt #2"
+    >&2 echo "First attempt failed. Retrying $*"
+    if "$@"; then
+      echo 'Second attempt successful'
+      echo "##teamcity[buildStatus status='SUCCESS' text='{build.status.text} with a flaky failure']"
+      echo "##teamcity[setParameter name='elastic.build.flaky' value='true']"
+      tc_end_block "Retryable Step - Attempt #2"
+    else
+      status="$?"
+      tc_end_block "Retryable Step - Attempt #2"
+      return "$status"
+    fi
+  }
+  tc_end_block "Retryable Step - Attempt #1"
+}

.github/CODEOWNERS

@@ -162,6 +162,8 @@
 /src/cli/keystore/ @elastic/kibana-operations
 /src/legacy/server/warnings/ @elastic/kibana-operations
 /.ci/es-snapshots/ @elastic/kibana-operations
+/.ci/teamcity/ @elastic/kibana-operations
+/.teamcity/ @elastic/kibana-operations
 /vars/ @elastic/kibana-operations
 #CC# /packages/kbn-expect/ @elastic/kibana-operations
 

.teamcity/.editorconfig

@@ -0,0 +1,4 @@
+[*.{kt,kts}]
+disabled_rules=no-wildcard-imports
+indent_size=2
+kotlin_imports_layout=idea

.teamcity/README.md

@@ -0,0 +1,156 @@
+# Kibana TeamCity
+
+## Implemented so far
+
+- Project configuration with ability to provide configuration values that are unique per TeamCity instance (e.g. dev vs prod)
+- Read-only configuration (no editing through the UI)
+- Secrets stored in TeamCity outside of source control
+- Setting secret environment variables (they get filtered from console if output on accident)
+- GCP agent configurations
+  - One-time use agents
+  - Multiple agents configured, of different sizes (cpu, memory)
+  - Require specific agents per build configuration
+- Unit testable DSL code
+- Build artifact generation and consumption
+- DSL Extensions of various kinds to easily share common configuration between build configurations in the same repo
+- Barebones Slack notifications via plugin
+- Dynamically creating environment variables / secrets at runtime for subsequent steps
+- "Baseline CI" job that runs a subset of CI for every commit
+- "Hourly CI" job that runs full CI hourly, if changes are detected. Re-uses builds that ran during "Baseline CI" for same commit
+- Performance monitoring enabled for all jobs
+- Jobs with multiple VCS roots (Kibana + Elasticsearch)
+- GCS uploading using service account key file and gsutil
+- Job that has a version string as an "output", rather than an artifact/file, with consumption in a different job
+- Clone a list of jobs and modify dependencies/configuration for a second pipeline
+- Promote/deploy a built artifact through the UI by selecting previously built artifact (or automatically build a new one and deploy if successful)
+- Custom Build IDs using service messages
+
+## Pull Requests
+
+The `Pull Request` feature in TeamCity:
+
+- Automatically discovers pull request branches in GitHub
+  - Option to filter by contributor type (members of same org, org+external contributor, everyone)
+  - Option to filter by target branch (e.g. only discover Pull Requests targeting master)
+  - Works by essentially modifying the VCS root branch spec (so you should NOT add anything related to PRs to branch spec if you are using this)
+  - Draft PRs do get discovered
+- Adds some Pull Request information to build overview pages
+- Adds a few parameters available to build configurations:
+  - teamcity.pullRequest.number
+  - teamcity.pullRequest.title
+  - teamcity.pullRequest.source.branch
+  - teamcity.pullRequest.target.branch
+  - (Notice that source owner is not available - there's no information for forks)
+- Requires a token for API interaction
+
+That's it. There's no interaction with labels/comments/etc. Triggering is handled via the standard triggering options.
+
+So, if you only want to:
+
+- Build on new commit (e.g. not via comment) or via the TeamCity UI
+- Start builds for users not covered by the filter options using the TeamCity UI
+
+The Pull Request feature may be enough to cover your needs. Otherwise, you'll need something additional (an external bot, or a new teamcity plugin, etc).
+
+### Other PR notes
+
+- TeamCity doesn't have the ability to cancel currently-running builds when a new commit is pushed
+- TeamCity does not add fork information (e.g. the owner) to build configuration parameters
+- Builds CAN be triggered for branches not yet discovered
+  - You can turn off discovery altogether, and a branch will still be build-able. When triggered externally, it will show up in the UI and build.
+
+How to [trigger a build via API](https://www.jetbrains.com/help/teamcity/rest-api-reference.html#Triggering+a+Build):
+
+```
+POST https://teamcity-server/app/rest/buildQueue
+
+<build branchName="pull/78662">
+  <buildType id="Your_Build_Configuration_ID" />
+</build>
+```
+
+and with additional properties:
+
+```
+<build branchName="pull/78662">
+  <buildType id="Your_Build_Configuration_ID" />
+  <properties>
+    <property name="env.GITHUB_PR_OWNER" value="cool-username"/>
+    <property name="env.GITHUB_PR_REPO" value="kibana"/>
+  </properties>
+</build>
+```
+
+## Kibana Builds
+
+### Baseline CI
+
+- Generates baseline metrics needed for PR comparisons
+- Only runs OSS and default builds, and generates default saved object field metrics
+- Runs for each commit (each build should build a single commit)
+
+### Full CI
+
+- Runs everything in CI - all tests and builds
+- Re-uses builds from Baseline CI if they are finished or in-progress
+- Not generally triggered directly, is triggered by other jobs
+
+### Hourly CI
+
+- Triggers every hour and groups up all changes since the last run
+- Runs whatever is in `Full CI`
+
+### Pull Request CI
+
+- Kibana TeamCity PR bot triggers this build for PRs (new commits, trigger comments)
+- Sets many PR related parameters/env vars, then runs `Full CI`
+
+![Diagram](Kibana.png)
+
+### ES Snapshot Verification
+
+Build Configurations:
+
+- Build Snapshot
+- Test Builds (e.g. OSS CI Group 1, Default CI Group 3, etc)
+- Verify Snapshot
+- Promote Snapshot
+- Immediately Promote Snapshot
+
+Desires:
+
+- Build ES snapshot on a daily basis, run E2E tests against it, promote when successful
+- Ability to easily promote old builds that have been verified
+- Ability to run verification without promoting it
+
+#### Build Snapshot
+
+- checks out both Kibana and ES codebases
+- builds ES artifacts
+- uses scripts from Kibana repo to create JSON manifest and assemble snapshot files
+- uploads artifacts to GCS
+- sets parameters via service message that contains the snapshot URL, ID, version so they can be consumed by downstream jobs
+- triggers on timer, once per day
+
+#### Test Builds
+
+- builds are clones of all "essential ci" functional and integration tests with irrelevant features disabled
+  - they are clones because runs of this build and runs of the essential ci versions for the same commit hash mean different things
+- snapshot dependency on `Build Elasticsearch Snapshot` is added to clones
+- set `env.ES_SNAPSHOT_MANIFEST` = `dep.<BUILD_ES_BUILD_ID>.ES_SNAPSHOT_MANIFEST` to "consume" the built artifact
+
+#### Verify Snapshot
+
+- composite build that contains all of the cloned test builds
+
+#### Promote Snapshot
+
+- snapshot dependency on `Build Snapshot` and `Verify Snapshot`
+- uses scripts from Kibana repo to promote elasticsearch snapshot from `Build Snapshot` by updating manifest files in GCS
+- triggers whenever a build of `Verify Snapshot` completes successfully
+
+#### Immediately Promote Snapshot
+
+- snapshot dependency only on `Build Snapshot`
+- same as `Promote Snapshot` but skips testing
+- can only be triggered manually

.teamcity/pom.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to Elasticsearch under one or more contributor
+  ~ license agreements. See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright
+  ~ ownership. Elasticsearch licenses this file to you under
+  ~ the Apache License, Version 2.0 (the "License"); you may
+  ~ not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <name>Kibana Teamcity Config DSL Script</name>
+  <groupId>org.elastic.kibana</groupId>
+  <artifactId>kibana-teamcity-dsl</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <parent>
+    <groupId>org.jetbrains.teamcity</groupId>
+    <artifactId>configs-dsl-kotlin-parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+  </parent>
+
+  <repositories>
+    <repository>
+      <id>jetbrains-all</id>
+      <url>https://download.jetbrains.com/teamcity-repository</url>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+    <repository>
+      <id>teamcity-server</id>
+      <url>https://ci.elastic.dev/app/dsl-plugins-repository</url>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
+  <pluginRepositories>
+    <pluginRepository>
+      <id>JetBrains</id>
+      <url>https://download.jetbrains.com/teamcity-repository</url>
+    </pluginRepository>
+  </pluginRepositories>
+
+  <build>
+    <testSourceDirectory>tests</testSourceDirectory>
+    <sourceDirectory>src</sourceDirectory>
+    <plugins>
+      <plugin>
+        <artifactId>kotlin-maven-plugin</artifactId>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <version>${kotlin.version}</version>
+        <configuration />
+        <executions>
+          <execution>
+            <id>compile</id>
+            <phase>process-sources</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>test-compile</id>
+            <phase>process-test-sources</phase>
+            <goals>
+              <goal>test-compile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.jetbrains.teamcity</groupId>
+        <artifactId>teamcity-configs-maven-plugin</artifactId>
+        <version>${teamcity.dsl.version}</version>
+        <configuration>
+          <format>kotlin</format>
+          <dstDir>target/generated-configs</dstDir>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.jetbrains.teamcity</groupId>
+      <artifactId>configs-dsl-kotlin</artifactId>
+      <version>${teamcity.dsl.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.teamcity</groupId>
+      <artifactId>configs-dsl-kotlin-plugins</artifactId>
+      <version>1.0-SNAPSHOT</version>
+      <type>pom</type>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
+      <version>${kotlin.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-script-runtime</artifactId>
+      <version>${kotlin.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13</version>
+    </dependency>
+  </dependencies>
+</project>

.teamcity/settings.kts

@@ -0,0 +1,12 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import projects.Kibana
+import projects.KibanaConfiguration
+
+version = "2020.1"
+
+val config = KibanaConfiguration {
+  agentNetwork = DslContext.getParameter("agentNetwork", "teamcity")
+  agentSubnet = DslContext.getParameter("agentSubnet", "teamcity")
+}
+
+project(Kibana(config))

.teamcity/src/Extensions.kt

@@ -0,0 +1,169 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.notifications
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.v2019_2.ui.insert
+import projects.kibanaConfiguration
+
+fun BuildFeatures.junit(dirs: String = "target/**/TEST-*.xml") {
+  feature {
+    type = "xml-report-plugin"
+    param("xmlReportParsing.reportType", "junit")
+    param("xmlReportParsing.reportDirs", dirs)
+  }
+}
+
+fun ProjectFeatures.kibanaAgent(init: ProjectFeature.() -> Unit) {
+  feature {
+    type = "CloudImage"
+    param("network", kibanaConfiguration.agentNetwork)
+    param("subnet", kibanaConfiguration.agentSubnet)
+    param("growingId", "true")
+    param("agent_pool_id", "-2")
+    param("preemptible", "false")
+    param("sourceProject", "elastic-images-prod")
+    param("sourceImageFamily", "elastic-kibana-ci-ubuntu-1804-lts")
+    param("zone", "us-central1-a")
+    param("profileId", "kibana")
+    param("diskType", "pd-ssd")
+    param("machineCustom", "false")
+    param("maxInstances", "200")
+    param("imageType", "ImageFamily")
+    param("diskSizeGb", "75") // TODO
+    init()
+  }
+}
+
+fun ProjectFeatures.kibanaAgent(size: String, init: ProjectFeature.() -> Unit = {}) {
+  kibanaAgent {
+    id = "KIBANA_STANDARD_$size"
+    param("source-id", "kibana-standard-$size-")
+    param("machineType", "n2-standard-$size")
+    init()
+  }
+}
+
+fun BuildType.kibanaAgent(size: String) {
+  requirements {
+    startsWith("teamcity.agent.name", "kibana-standard-$size-", "RQ_AGENT_NAME")
+  }
+}
+
+fun BuildType.kibanaAgent(size: Int) {
+  kibanaAgent(size.toString())
+}
+
+val testArtifactRules = """
+    target/kibana-*
+    target/test-metrics/*
+    target/kibana-security-solution/**/*.png
+    target/junit/**/*
+    target/test-suites-ci-plan.json
+    test/**/screenshots/session/*.png
+    test/**/screenshots/failure/*.png
+    test/**/screenshots/diff/*.png
+    test/functional/failure_debug/html/*.html
+    x-pack/test/**/screenshots/session/*.png
+    x-pack/test/**/screenshots/failure/*.png
+    x-pack/test/**/screenshots/diff/*.png
+    x-pack/test/functional/failure_debug/html/*.html
+    x-pack/test/functional/apps/reporting/reports/session/*.pdf
+  """.trimIndent()
+
+fun BuildType.addTestSettings() {
+  artifactRules += "\n" + testArtifactRules
+  steps {
+    failedTestReporter()
+  }
+  features {
+    junit()
+  }
+}
+
+fun BuildType.addSlackNotifications(to: String = "#kibana-teamcity-testing") {
+  params {
+    param("elastic.slack.enabled", "true")
+    param("elastic.slack.channels", to)
+  }
+}
+
+fun BuildType.dependsOn(buildType: BuildType, init: SnapshotDependency.() -> Unit = {}) {
+  dependencies {
+    snapshot(buildType) {
+      reuseBuilds = ReuseBuilds.SUCCESSFUL
+      onDependencyCancel = FailureAction.ADD_PROBLEM
+      onDependencyFailure = FailureAction.ADD_PROBLEM
+      synchronizeRevisions = true
+      init()
+    }
+  }
+}
+
+fun BuildType.dependsOn(vararg buildTypes: BuildType, init: SnapshotDependency.() -> Unit = {}) {
+  buildTypes.forEach { dependsOn(it, init) }
+}
+
+fun BuildSteps.failedTestReporter(init: ScriptBuildStep.() -> Unit = {}) {
+  script {
+    name = "Failed Test Reporter"
+    scriptContent =
+      """
+        #!/bin/bash
+        node scripts/report_failed_tests
+      """.trimIndent()
+    executionMode = BuildStep.ExecutionMode.RUN_ON_FAILURE
+    init()
+  }
+}
+
+// Note: This is currently only used for tests and has a retry in it for flaky tests.
+// The retry should be refactored if runbld is ever needed for other tasks.
+fun BuildSteps.runbld(stepName: String, script: String) {
+  script {
+    name = stepName
+
+    // The indentation for this string is like this to ensure 100% that the RUNBLD-SCRIPT heredoc termination will not have spaces at the beginning
+    scriptContent =
+"""#!/bin/bash
+
+set -euo pipefail
+
+source .ci/teamcity/util.sh
+
+branchName="${'$'}GIT_BRANCH"
+branchName="${'$'}{branchName#refs\/heads\/}"
+
+if [[ "${'$'}{GITHUB_PR_NUMBER-}" ]]; then
+  branchName=pull-request
+fi
+
+project=kibana
+if [[ "${'$'}{ES_SNAPSHOT_MANIFEST-}" ]]; then
+  project=kibana-es-snapshot-verify
+fi
+
+# These parameters are only for runbld reporting
+export JENKINS_HOME="${'$'}HOME"
+export BUILD_URL="%teamcity.serverUrl%/build/%teamcity.build.id%"
+export branch_specifier=${'$'}branchName
+export NODE_LABELS='teamcity'
+export BUILD_NUMBER="%build.number%"
+export EXECUTOR_NUMBER=''
+export NODE_NAME=''
+
+export OLD_PATH="${'$'}PATH"
+
+file=${'$'}(mktemp)
+
+(
+cat <<RUNBLD-SCRIPT
+#!/bin/bash
+export PATH="${'$'}OLD_PATH"
+$script
+RUNBLD-SCRIPT
+) > ${'$'}file
+
+tc_retry /usr/local/bin/runbld -d "${'$'}(pwd)" --job-name="elastic+${'$'}project+${'$'}branchName" ${'$'}file
+"""
+  }
+}

.teamcity/src/builds/BaselineCi.kt

@@ -0,0 +1,38 @@
+package builds
+
+import addSlackNotifications
+import builds.default.DefaultBuild
+import builds.default.DefaultSavedObjectFieldMetrics
+import builds.oss.OssBuild
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
+import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.vcs
+import templates.KibanaTemplate
+
+object BaselineCi : BuildType({
+  id("Baseline_CI")
+  name = "Baseline CI"
+  description = "Runs builds, saved object field metrics for every commit"
+  type = Type.COMPOSITE
+  paused = true
+
+  templates(KibanaTemplate)
+
+  triggers {
+    vcs {
+      branchFilter = "refs/heads/master_teamcity"
+//      perCheckinTriggering = true // TODO re-enable this later, it wreaks havoc when I merge upstream
+    }
+  }
+
+  dependsOn(
+    OssBuild,
+    DefaultBuild,
+    DefaultSavedObjectFieldMetrics
+  ) {
+    onDependencyCancel = FailureAction.ADD_PROBLEM
+  }
+
+  addSlackNotifications()
+})

.teamcity/src/builds/Checks.kt

@@ -0,0 +1,37 @@
+package builds
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import kibanaAgent
+
+object Checks : BuildType({
+  name = "Checks"
+  description = "Executes Various Checks"
+
+  kibanaAgent(4)
+
+  val checkScripts = mapOf(
+    "Check Telemetry Schema" to ".ci/teamcity/checks/telemetry.sh",
+    "Check TypeScript Projects" to ".ci/teamcity/checks/ts_projects.sh",
+    "Check File Casing" to ".ci/teamcity/checks/file_casing.sh",
+    "Check Licenses" to ".ci/teamcity/checks/licenses.sh",
+    "Verify NOTICE" to ".ci/teamcity/checks/verify_notice.sh",
+    "Test Hardening" to ".ci/teamcity/checks/test_hardening.sh",
+    "Check Types" to ".ci/teamcity/checks/type_check.sh",
+    "Check Doc API Changes" to ".ci/teamcity/checks/doc_api_changes.sh",
+    "Check Bundle Limits" to ".ci/teamcity/checks/bundle_limits.sh",
+    "Check i18n" to ".ci/teamcity/checks/i18n.sh"
+  )
+
+  steps {
+    for (checkScript in checkScripts) {
+      script {
+        name = checkScript.key
+        scriptContent = """
+          #!/bin/bash
+          ${checkScript.value}
+        """.trimIndent()
+      }
+    }
+  }
+})

.teamcity/src/builds/FullCi.kt

@@ -0,0 +1,30 @@
+package builds
+
+import builds.default.*
+import builds.oss.*
+import builds.test.AllTests
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+
+object FullCi : BuildType({
+  id("Full_CI")
+  name = "Full CI"
+  description = "Runs everything in CI. For tracked branches and PRs."
+  type = Type.COMPOSITE
+
+  dependsOn(
+    Lint,
+    Checks,
+    AllTests,
+    OssBuild,
+    OssAccessibility,
+    OssPluginFunctional,
+    OssCiGroups,
+    OssFirefox,
+    DefaultBuild,
+    DefaultCiGroups,
+    DefaultFirefox,
+    DefaultAccessibility,
+    DefaultSecuritySolution
+  )
+})

.teamcity/src/builds/HourlyCi.kt

@@ -0,0 +1,34 @@
+package builds
+
+import addSlackNotifications
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
+import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.schedule
+
+object HourlyCi : BuildType({
+  id("Hourly_CI")
+  name = "Hourly CI"
+  description = "Runs everything in CI, hourly"
+  type = Type.COMPOSITE
+
+  triggers {
+    schedule {
+      schedulingPolicy = cron {
+        hours = "*"
+        minutes = "0"
+      }
+      branchFilter = "refs/heads/master_teamcity"
+      triggerBuild = always()
+      withPendingChangesOnly = true
+    }
+  }
+
+  dependsOn(
+    FullCi
+  ) {
+    onDependencyCancel = FailureAction.ADD_PROBLEM
+  }
+
+  addSlackNotifications()
+})

.teamcity/src/builds/Lint.kt

@@ -0,0 +1,33 @@
+package builds
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import kibanaAgent
+
+object Lint : BuildType({
+  name = "Lint"
+  description = "Executes Linting, such as eslint and sasslint"
+
+  kibanaAgent(2)
+
+  steps {
+    script {
+      name = "Sasslint"
+
+      scriptContent =
+        """
+          #!/bin/bash
+          yarn run grunt run:sasslint
+        """.trimIndent()
+    }
+
+    script {
+      name = "ESLint"
+      scriptContent =
+        """
+          #!/bin/bash
+          yarn run grunt run:eslint
+        """.trimIndent()
+    }
+  }
+})

.teamcity/src/builds/PullRequestCi.kt

@@ -0,0 +1,78 @@
+package builds
+
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.AbsoluteId
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.commitStatusPublisher
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests
+import vcs.Kibana
+
+object PullRequestCi : BuildType({
+  id = AbsoluteId("Kibana_PullRequest_CI")
+  name = "Pull Request CI"
+  type = Type.COMPOSITE
+
+   buildNumberPattern = "%build.counter%-%env.GITHUB_PR_OWNER%-%env.GITHUB_PR_BRANCH%"
+
+  vcs {
+    root(Kibana)
+    checkoutDir = "kibana"
+
+    branchFilter = "+:pull/*"
+    excludeDefaultBranchChanges = true
+  }
+
+  val prAllowedList = listOf(
+    "brianseeders",
+    "alexwizp",
+    "barlowm",
+    "DziyanaDzeraviankina",
+    "maryia-lapata",
+    "renovate[bot]",
+    "sulemanof",
+    "VladLasitsa"
+  )
+
+  params {
+    param("elastic.pull_request.enabled", "true")
+    param("elastic.pull_request.target_branch", "master_teamcity")
+    param("elastic.pull_request.allow_org_users", "true")
+    param("elastic.pull_request.allowed_repo_permissions", "admin,write")
+    param("elastic.pull_request.allowed_list", prAllowedList.joinToString(","))
+    param("elastic.pull_request.cancel_in_progress_builds_on_update", "true")
+
+    // These params should get filled in by the app that triggers builds
+    param("env.GITHUB_PR_TARGET_BRANCH", "")
+    param("env.GITHUB_PR_NUMBER", "")
+    param("env.GITHUB_PR_OWNER", "")
+    param("env.GITHUB_PR_REPO", "")
+    param("env.GITHUB_PR_BRANCH", "")
+    param("env.GITHUB_PR_TRIGGERED_SHA", "")
+    param("env.GITHUB_PR_LABELS", "")
+    param("env.GITHUB_PR_TRIGGER_COMMENT", "")
+
+    param("reverse.dep.*.env.GITHUB_PR_TARGET_BRANCH", "")
+    param("reverse.dep.*.env.GITHUB_PR_NUMBER", "")
+    param("reverse.dep.*.env.GITHUB_PR_OWNER", "")
+    param("reverse.dep.*.env.GITHUB_PR_REPO", "")
+    param("reverse.dep.*.env.GITHUB_PR_BRANCH", "")
+    param("reverse.dep.*.env.GITHUB_PR_TRIGGERED_SHA", "")
+    param("reverse.dep.*.env.GITHUB_PR_LABELS", "")
+    param("reverse.dep.*.env.GITHUB_PR_TRIGGER_COMMENT", "")
+  }
+
+  features {
+    commitStatusPublisher {
+      vcsRootExtId = "${Kibana.id}"
+      publisher = github {
+        githubUrl = "https://api.github.com"
+        authType = personalToken {
+          token = "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b"
+        }
+      }
+    }
+  }
+
+  dependsOn(FullCi)
+})

.teamcity/src/builds/default/DefaultAccessibility.kt

@@ -0,0 +1,12 @@
+package builds.default
+
+import runbld
+
+object DefaultAccessibility : DefaultFunctionalBase({
+  id("DefaultAccessibility")
+  name = "Accessibility"
+
+  steps {
+    runbld("Default Accessibility", "./.ci/teamcity/default/accessibility.sh")
+  }
+})

.teamcity/src/builds/default/DefaultBuild.kt

@@ -0,0 +1,56 @@
+package builds.default
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.Dependencies
+import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+
+object DefaultBuild : BuildType({
+  name = "Build Default"
+  description = "Generates Default Build Distribution artifact"
+
+  artifactRules = """
+    +:install/kibana/**/* => kibana-default.tar.gz
+    target/kibana-*
+    +:src/**/target/public/**/* => kibana-default-plugins.tar.gz!/src/
+    +:x-pack/plugins/**/target/public/**/* => kibana-default-plugins.tar.gz!/x-pack/plugins/
+    +:x-pack/test/**/target/public/**/* => kibana-default-plugins.tar.gz!/x-pack/test/
+    +:examples/**/target/public/**/* => kibana-default-plugins.tar.gz!/examples/
+    +:test/**/target/public/**/* => kibana-default-plugins.tar.gz!/test/
+  """.trimIndent()
+
+  requirements {
+    startsWith("teamcity.agent.name", "kibana-c2-16-", "RQ_AGENT_NAME")
+  }
+
+  steps {
+    script {
+      name = "Build Default Distribution"
+      scriptContent =
+        """
+          #!/bin/bash
+          ./.ci/teamcity/default/build.sh
+        """.trimIndent()
+    }
+  }
+})
+
+fun Dependencies.defaultBuild(rules: String = "+:kibana-default.tar.gz!** => ../build/kibana-build-default") {
+  dependency(DefaultBuild) {
+    snapshot {
+      onDependencyFailure = FailureAction.FAIL_TO_START
+      onDependencyCancel = FailureAction.FAIL_TO_START
+    }
+
+    artifacts {
+      artifactRules = rules
+    }
+  }
+}
+
+fun Dependencies.defaultBuildWithPlugins() {
+  defaultBuild("""
+    +:kibana-default.tar.gz!** => ../build/kibana-build-default
+    +:kibana-default-plugins.tar.gz!**
+  """.trimIndent())
+}

.teamcity/src/builds/default/DefaultCiGroup.kt

@@ -0,0 +1,15 @@
+package builds.default
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import runbld
+
+class DefaultCiGroup(val ciGroup: Int = 0, init: BuildType.() -> Unit = {}) : DefaultFunctionalBase({
+  id("DefaultCiGroup_$ciGroup")
+  name = "CI Group $ciGroup"
+
+  steps {
+    runbld("Default CI Group $ciGroup", "./.ci/teamcity/default/ci_group.sh $ciGroup")
+  }
+
+  init()
+})

.teamcity/src/builds/default/DefaultCiGroups.kt

@@ -0,0 +1,15 @@
+package builds.default
+
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+
+const val DEFAULT_CI_GROUP_COUNT = 10
+val defaultCiGroups = (1..DEFAULT_CI_GROUP_COUNT).map { DefaultCiGroup(it) }
+
+object DefaultCiGroups : BuildType({
+  id("Default_CIGroups_Composite")
+  name = "CI Groups"
+  type = Type.COMPOSITE
+
+  dependsOn(*defaultCiGroups.toTypedArray())
+})

.teamcity/src/builds/default/DefaultFirefox.kt

@@ -0,0 +1,12 @@
+package builds.default
+
+import runbld
+
+object DefaultFirefox : DefaultFunctionalBase({
+  id("DefaultFirefox")
+  name = "Firefox"
+
+  steps {
+    runbld("Default Firefox", "./.ci/teamcity/default/firefox.sh")
+  }
+})

.teamcity/src/builds/default/DefaultFunctionalBase.kt

@@ -0,0 +1,19 @@
+package builds.default
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+
+open class DefaultFunctionalBase(init: BuildType.() -> Unit = {}) : BuildType({
+  params {
+    param("env.KBN_NP_PLUGINS_BUILT", "true")
+  }
+
+  dependencies {
+    defaultBuildWithPlugins()
+  }
+
+  init()
+
+  addTestSettings()
+})
+

.teamcity/src/builds/default/DefaultSavedObjectFieldMetrics.kt

@@ -0,0 +1,28 @@
+package builds.default
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+
+object DefaultSavedObjectFieldMetrics : BuildType({
+  id("DefaultSavedObjectFieldMetrics")
+  name = "Default Saved Object Field Metrics"
+
+  params {
+    param("env.KBN_NP_PLUGINS_BUILT", "true")
+  }
+
+  steps {
+    script {
+      name = "Default Saved Object Field Metrics"
+      scriptContent =
+        """
+          #!/bin/bash
+          ./.ci/teamcity/default/saved_object_field_metrics.sh
+        """.trimIndent()
+    }
+  }
+
+  dependencies {
+    defaultBuild()
+  }
+})

.teamcity/src/builds/default/DefaultSecuritySolution.kt

@@ -0,0 +1,15 @@
+package builds.default
+
+import addTestSettings
+import runbld
+
+object DefaultSecuritySolution : DefaultFunctionalBase({
+  id("DefaultSecuritySolution")
+  name = "Security Solution"
+
+  steps {
+    runbld("Default Security Solution", "./.ci/teamcity/default/security_solution.sh")
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/es_snapshots/Build.kt

@@ -0,0 +1,84 @@
+package builds.es_snapshots
+
+import addSlackNotifications
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import vcs.Elasticsearch
+import vcs.Kibana
+
+object ESSnapshotBuild : BuildType({
+  name = "Build Snapshot"
+  paused = true
+
+  requirements {
+    startsWith("teamcity.agent.name", "kibana-c2-16-", "RQ_AGENT_NAME")
+  }
+
+  vcs {
+    root(Kibana, "+:. => kibana")
+    root(Elasticsearch, "+:. => elasticsearch")
+    checkoutDir = ""
+  }
+
+  params {
+    param("env.ELASTICSEARCH_BRANCH", "%vcsroot.${Elasticsearch.id.toString()}.branch%")
+    param("env.ELASTICSEARCH_GIT_COMMIT", "%build.vcs.number.${Elasticsearch.id.toString()}%")
+
+    param("env.GOOGLE_APPLICATION_CREDENTIALS", "%teamcity.build.workingDir%/gcp-credentials.json")
+    password("env.GOOGLE_APPLICATION_CREDENTIALS_JSON", "credentialsJSON:6e0acb7c-f89c-4225-84b8-4fc102f1a5ef", display = ParameterDisplay.HIDDEN)
+  }
+
+  steps {
+    script {
+      name = "Setup Environment"
+      scriptContent =
+        """
+          #!/bin/bash
+          cd kibana
+          ./.ci/teamcity/setup_env.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Node and Yarn"
+      scriptContent =
+        """
+          #!/bin/bash
+          cd kibana
+          ./.ci/teamcity/setup_node.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Build Elasticsearch Distribution"
+      scriptContent =
+        """
+          #!/bin/bash
+          cd kibana
+          ./.ci/teamcity/es_snapshots/build.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Google Cloud Credentials"
+      scriptContent =
+        """#!/bin/bash
+          echo "${"$"}GOOGLE_APPLICATION_CREDENTIALS_JSON" > "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+          gcloud auth activate-service-account --key-file "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+        """.trimIndent()
+    }
+
+    script {
+      name = "Create Snapshot Manifest"
+      scriptContent =
+        """#!/bin/bash
+          cd kibana
+          node ./.ci/teamcity/es_snapshots/create_manifest.js "$(cd ../es-build && pwd)"
+        """.trimIndent()
+    }
+  }
+
+  artifactRules = "+:es-build/**/*.json"
+
+  addSlackNotifications()
+})

.teamcity/src/builds/es_snapshots/Promote.kt

@@ -0,0 +1,87 @@
+package builds.es_snapshots
+
+import addSlackNotifications
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.finishBuildTrigger
+import vcs.Kibana
+
+object ESSnapshotPromote : BuildType({
+  name = "Promote Snapshot"
+  paused = true
+  type = Type.DEPLOYMENT
+
+  vcs {
+    root(Kibana, "+:. => kibana")
+    checkoutDir = ""
+  }
+
+  params {
+    param("env.ES_SNAPSHOT_MANIFEST", "${ESSnapshotBuild.depParamRefs["env.ES_SNAPSHOT_MANIFEST"]}")
+    param("env.GOOGLE_APPLICATION_CREDENTIALS", "%teamcity.build.workingDir%/gcp-credentials.json")
+    password("env.GOOGLE_APPLICATION_CREDENTIALS_JSON", "credentialsJSON:6e0acb7c-f89c-4225-84b8-4fc102f1a5ef", display = ParameterDisplay.HIDDEN)
+  }
+
+  triggers {
+    finishBuildTrigger {
+      buildType = Verify.id.toString()
+      successfulOnly = true
+    }
+  }
+
+  steps {
+    script {
+      name = "Setup Environment"
+      scriptContent =
+        """
+                #!/bin/bash
+                cd kibana
+                ./.ci/teamcity/setup_env.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Node and Yarn"
+      scriptContent =
+        """
+                #!/bin/bash
+                cd kibana
+                ./.ci/teamcity/setup_node.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Google Cloud Credentials"
+      scriptContent =
+        """#!/bin/bash
+          echo "${"$"}GOOGLE_APPLICATION_CREDENTIALS_JSON" > "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+          gcloud auth activate-service-account --key-file "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+        """.trimIndent()
+    }
+
+    script {
+      name = "Promote Snapshot Manifest"
+      scriptContent =
+        """#!/bin/bash
+          cd kibana
+          node ./.ci/teamcity/es_snapshots/promote_manifest.js "${"$"}ES_SNAPSHOT_MANIFEST"
+        """.trimIndent()
+    }
+  }
+
+  dependencies {
+    dependency(ESSnapshotBuild) {
+      snapshot {  }
+
+      // This is just here to allow build selection in the UI, the file isn't actually used
+      artifacts {
+        artifactRules = "manifest.json"
+      }
+    }
+    dependency(Verify) {
+      snapshot {  }
+    }
+  }
+
+  addSlackNotifications()
+})

.teamcity/src/builds/es_snapshots/PromoteImmediate.kt

@@ -0,0 +1,79 @@
+package builds.es_snapshots
+
+import addSlackNotifications
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.finishBuildTrigger
+import vcs.Elasticsearch
+import vcs.Kibana
+
+object ESSnapshotPromoteImmediate : BuildType({
+  name = "Immediately Promote Snapshot"
+  description = "Skip testing and immediately promote the selected snapshot"
+  paused = true
+  type = Type.DEPLOYMENT
+
+  vcs {
+    root(Kibana, "+:. => kibana")
+    checkoutDir = ""
+  }
+
+  params {
+    param("env.ES_SNAPSHOT_MANIFEST", "${ESSnapshotBuild.depParamRefs["env.ES_SNAPSHOT_MANIFEST"]}")
+    param("env.GOOGLE_APPLICATION_CREDENTIALS", "%teamcity.build.workingDir%/gcp-credentials.json")
+    password("env.GOOGLE_APPLICATION_CREDENTIALS_JSON", "credentialsJSON:6e0acb7c-f89c-4225-84b8-4fc102f1a5ef", display = ParameterDisplay.HIDDEN)
+  }
+
+  steps {
+    script {
+      name = "Setup Environment"
+      scriptContent =
+        """
+                #!/bin/bash
+                cd kibana
+                ./.ci/teamcity/setup_env.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Node and Yarn"
+      scriptContent =
+        """
+                #!/bin/bash
+                cd kibana
+                ./.ci/teamcity/setup_node.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Google Cloud Credentials"
+      scriptContent =
+        """#!/bin/bash
+          echo "${"$"}GOOGLE_APPLICATION_CREDENTIALS_JSON" > "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+          gcloud auth activate-service-account --key-file "${"$"}GOOGLE_APPLICATION_CREDENTIALS"
+        """.trimIndent()
+    }
+
+    script {
+      name = "Promote Snapshot Manifest"
+      scriptContent =
+        """#!/bin/bash
+          cd kibana
+          node ./.ci/teamcity/es_snapshots/promote_manifest.js "${"$"}ES_SNAPSHOT_MANIFEST"
+        """.trimIndent()
+    }
+  }
+
+  dependencies {
+    dependency(ESSnapshotBuild) {
+      snapshot {  }
+
+      // This is just here to allow build selection in the UI, the file isn't actually used
+      artifacts {
+        artifactRules = "manifest.json"
+      }
+    }
+  }
+
+  addSlackNotifications()
+})

.teamcity/src/builds/es_snapshots/Verify.kt

@@ -0,0 +1,96 @@
+package builds.es_snapshots
+
+import builds.default.DefaultBuild
+import builds.default.DefaultSecuritySolution
+import builds.default.defaultCiGroups
+import builds.oss.OssBuild
+import builds.oss.OssPluginFunctional
+import builds.oss.ossCiGroups
+import builds.test.ApiServerIntegration
+import builds.test.JestIntegration
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+
+val cloneForVerify = { build: BuildType ->
+  val newBuild = BuildType()
+  build.copyTo(newBuild)
+  newBuild.id = AbsoluteId(build.id?.toString() + "_ES_Snapshots")
+  newBuild.params {
+    param("env.ES_SNAPSHOT_MANIFEST", "${ESSnapshotBuild.depParamRefs["env.ES_SNAPSHOT_MANIFEST"]}")
+  }
+  newBuild.dependencies {
+    dependency(ESSnapshotBuild) {
+      snapshot {
+        onDependencyFailure = FailureAction.FAIL_TO_START
+        onDependencyCancel = FailureAction.FAIL_TO_START
+      }
+      // This is just here to allow us to select a build when manually triggering a build using the UI
+      artifacts {
+        artifactRules = "manifest.json"
+      }
+    }
+  }
+  newBuild.steps.items.removeIf { it.name == "Failed Test Reporter" }
+  newBuild
+}
+
+val ossBuildsToClone = listOf(
+  *ossCiGroups.toTypedArray(),
+  OssPluginFunctional
+)
+
+val ossCloned = ossBuildsToClone.map { cloneForVerify(it) }
+
+val defaultBuildsToClone = listOf(
+  *defaultCiGroups.toTypedArray(),
+  DefaultSecuritySolution
+)
+
+val defaultCloned = defaultBuildsToClone.map { cloneForVerify(it) }
+
+val integrationsBuildsToClone = listOf(
+  ApiServerIntegration,
+  JestIntegration
+)
+
+val integrationCloned = integrationsBuildsToClone.map { cloneForVerify(it) }
+
+object OssTests : BuildType({
+  id("ES_Snapshots_OSS_Tests_Composite")
+  name = "OSS Distro Tests"
+  type = Type.COMPOSITE
+
+  dependsOn(*ossCloned.toTypedArray())
+})
+
+object DefaultTests : BuildType({
+  id("ES_Snapshots_Default_Tests_Composite")
+  name = "Default Distro Tests"
+  type = Type.COMPOSITE
+
+  dependsOn(*defaultCloned.toTypedArray())
+})
+
+object IntegrationTests : BuildType({
+  id("ES_Snapshots_Integration_Tests_Composite")
+  name = "Integration Tests"
+  type = Type.COMPOSITE
+
+  dependsOn(*integrationCloned.toTypedArray())
+})
+
+object Verify : BuildType({
+  id("ES_Snapshots_Verify_Composite")
+  name = "Verify Snapshot"
+  description = "Run all Kibana functional and integration tests using a given Elasticsearch snapshot"
+  type = Type.COMPOSITE
+
+  dependsOn(
+    ESSnapshotBuild,
+    OssBuild,
+    DefaultBuild,
+    OssTests,
+    DefaultTests,
+    IntegrationTests
+  )
+})

.teamcity/src/builds/oss/OssAccessibility.kt

@@ -0,0 +1,13 @@
+package builds.oss
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import runbld
+
+object OssAccessibility : OssFunctionalBase({
+  id("OssAccessibility")
+  name = "Accessibility"
+
+  steps {
+    runbld("OSS Accessibility", "./.ci/teamcity/oss/accessibility.sh")
+  }
+})

.teamcity/src/builds/oss/OssBuild.kt

@@ -0,0 +1,41 @@
+package builds.oss
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.Dependencies
+import jetbrains.buildServer.configs.kotlin.v2019_2.FailureAction
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+
+object OssBuild : BuildType({
+  name = "Build OSS"
+  description = "Generates OSS Build Distribution artifact"
+
+  requirements {
+    startsWith("teamcity.agent.name", "kibana-c2-16-", "RQ_AGENT_NAME")
+  }
+
+  steps {
+    script {
+      name = "Build OSS Distribution"
+      scriptContent =
+        """
+          #!/bin/bash
+          ./.ci/teamcity/oss/build.sh
+        """.trimIndent()
+    }
+  }
+
+  artifactRules = "+:build/oss/kibana-build-oss/**/* => kibana-oss.tar.gz"
+})
+
+fun Dependencies.ossBuild(rules: String = "+:kibana-oss.tar.gz!** => ../build/kibana-build-oss") {
+  dependency(OssBuild) {
+    snapshot {
+      onDependencyFailure = FailureAction.FAIL_TO_START
+      onDependencyCancel = FailureAction.FAIL_TO_START
+    }
+
+    artifacts {
+      artifactRules = rules
+    }
+  }
+}

.teamcity/src/builds/oss/OssCiGroup.kt

@@ -0,0 +1,15 @@
+package builds.oss
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import runbld
+
+class OssCiGroup(val ciGroup: Int, init: BuildType.() -> Unit = {}) : OssFunctionalBase({
+  id("OssCiGroup_$ciGroup")
+  name = "CI Group $ciGroup"
+
+  steps {
+    runbld("OSS CI Group $ciGroup", "./.ci/teamcity/oss/ci_group.sh $ciGroup")
+  }
+
+  init()
+})

.teamcity/src/builds/oss/OssCiGroups.kt

@@ -0,0 +1,15 @@
+package builds.oss
+
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+
+const val OSS_CI_GROUP_COUNT = 12
+val ossCiGroups = (1..OSS_CI_GROUP_COUNT).map { OssCiGroup(it) }
+
+object OssCiGroups : BuildType({
+  id("OSS_CIGroups_Composite")
+  name = "CI Groups"
+  type = Type.COMPOSITE
+
+  dependsOn(*ossCiGroups.toTypedArray())
+})

.teamcity/src/builds/oss/OssFirefox.kt

@@ -0,0 +1,12 @@
+package builds.oss
+
+import runbld
+
+object OssFirefox : OssFunctionalBase({
+  id("OssFirefox")
+  name = "Firefox"
+
+  steps {
+    runbld("OSS Firefox", "./.ci/teamcity/oss/firefox.sh")
+  }
+})

.teamcity/src/builds/oss/OssFunctionalBase.kt

@@ -0,0 +1,18 @@
+package builds.oss
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+
+open class OssFunctionalBase(init: BuildType.() -> Unit = {}) : BuildType({
+  params {
+    param("env.KBN_NP_PLUGINS_BUILT", "true")
+  }
+
+  dependencies {
+    ossBuild()
+  }
+
+  init()
+
+  addTestSettings()
+})

.teamcity/src/builds/oss/OssPluginFunctional.kt

@@ -0,0 +1,29 @@
+package builds.oss
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+import runbld
+
+object OssPluginFunctional : OssFunctionalBase({
+  id("OssPluginFunctional")
+  name = "Plugin Functional"
+
+  steps {
+    script {
+      name = "Build OSS Plugins"
+      scriptContent =
+        """
+                #!/bin/bash
+                ./.ci/teamcity/oss/build_plugins.sh
+        """.trimIndent()
+    }
+
+    runbld("OSS Plugin Functional", "./.ci/teamcity/oss/plugin_functional.sh")
+  }
+
+  dependencies {
+    ossBuild()
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/test/AllTests.kt

@@ -0,0 +1,12 @@
+package builds.test
+
+import dependsOn
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+
+object AllTests : BuildType({
+  name = "All Tests"
+  description = "All Non-Functional Tests"
+  type = Type.COMPOSITE
+
+  dependsOn(QuickTests, Jest, XPackJest, JestIntegration, ApiServerIntegration)
+})

.teamcity/src/builds/test/ApiServerIntegration.kt

@@ -0,0 +1,17 @@
+package builds.test
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import runbld
+
+object ApiServerIntegration : BuildType({
+  name = "API/Server Integration"
+  description = "Executes API and Server Integration Tests"
+
+  steps {
+    runbld("API Integration", "yarn run grunt run:apiIntegrationTests")
+    runbld("Server Integration", "yarn run grunt run:serverIntegrationTests")
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/test/Jest.kt

@@ -0,0 +1,19 @@
+package builds.test
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import kibanaAgent
+import runbld
+
+object Jest : BuildType({
+  name = "Jest Unit"
+  description = "Executes Jest Unit Tests"
+
+  kibanaAgent(8)
+
+  steps {
+    runbld("Jest Unit", "yarn run grunt run:test_jest")
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/test/JestIntegration.kt

@@ -0,0 +1,16 @@
+package builds.test
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import runbld
+
+object JestIntegration : BuildType({
+  name = "Jest Integration"
+  description = "Executes Jest Integration Tests"
+
+  steps {
+    runbld("Jest Integration", "yarn run grunt run:test_jest_integration")
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/test/QuickTests.kt

@@ -0,0 +1,29 @@
+package builds.test
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import kibanaAgent
+import runbld
+
+object QuickTests : BuildType({
+  name = "Quick Tests"
+  description = "Executes Quick Tests"
+
+  kibanaAgent(2)
+
+  val testScripts = mapOf(
+    "Test Hardening" to ".ci/teamcity/tests/test_hardening.sh",
+    "X-Pack List cyclic dependency" to ".ci/teamcity/tests/xpack_list_cyclic_dependency.sh",
+    "X-Pack SIEM cyclic dependency" to ".ci/teamcity/tests/xpack_siem_cyclic_dependency.sh",
+    "Test Projects" to ".ci/teamcity/tests/test_projects.sh",
+    "Mocha Tests" to ".ci/teamcity/tests/mocha.sh"
+  )
+
+  steps {
+    for (testScript in testScripts) {
+      runbld(testScript.key, testScript.value)
+    }
+  }
+
+  addTestSettings()
+})

.teamcity/src/builds/test/XPackJest.kt

@@ -0,0 +1,22 @@
+package builds.test
+
+import addTestSettings
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import kibanaAgent
+import runbld
+
+object XPackJest : BuildType({
+  name = "X-Pack Jest Unit"
+  description = "Executes X-Pack Jest Unit Tests"
+
+  kibanaAgent(16)
+
+  steps {
+      runbld("X-Pack Jest Unit", """
+        cd x-pack
+        node --max-old-space-size=6144 scripts/jest --ci --verbose --maxWorkers=6
+      """.trimIndent())
+  }
+
+  addTestSettings()
+})

.teamcity/src/projects/EsSnapshots.kt

@@ -0,0 +1,55 @@
+package projects
+
+import builds.es_snapshots.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import templates.KibanaTemplate
+
+object EsSnapshotsProject : Project({
+  id("ES_Snapshots")
+  name = "ES Snapshots"
+
+  subProject {
+    id("ES_Snapshot_Tests")
+    name = "Tests"
+
+    defaultTemplate = KibanaTemplate
+
+    subProject {
+      id("ES_Snapshot_Tests_OSS")
+      name = "OSS Distro Tests"
+
+      ossCloned.forEach {
+        buildType(it)
+      }
+
+      buildType(OssTests)
+    }
+
+    subProject {
+      id("ES_Snapshot_Tests_Default")
+      name = "Default Distro Tests"
+
+      defaultCloned.forEach {
+        buildType(it)
+      }
+
+      buildType(DefaultTests)
+    }
+
+    subProject {
+      id("ES_Snapshot_Tests_Integration")
+      name = "Integration Tests"
+
+      integrationCloned.forEach {
+        buildType(it)
+      }
+
+      buildType(IntegrationTests)
+    }
+  }
+
+  buildType(ESSnapshotBuild)
+  buildType(ESSnapshotPromote)
+  buildType(ESSnapshotPromoteImmediate)
+  buildType(Verify)
+})

.teamcity/src/projects/Kibana.kt

@@ -0,0 +1,171 @@
+package projects
+
+import vcs.Kibana
+import builds.*
+import builds.default.*
+import builds.oss.*
+import builds.test.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.projectFeatures.slackConnection
+import kibanaAgent
+import templates.KibanaTemplate
+import templates.DefaultTemplate
+import vcs.Elasticsearch
+
+class KibanaConfiguration() {
+  var agentNetwork: String = "teamcity"
+  var agentSubnet: String = "teamcity"
+
+  constructor(init: KibanaConfiguration.() -> Unit) : this() {
+    init()
+  }
+}
+
+var kibanaConfiguration = KibanaConfiguration()
+
+fun Kibana(config: KibanaConfiguration = KibanaConfiguration()) : Project {
+  kibanaConfiguration = config
+
+  return Project {
+    params {
+      param("teamcity.ui.settings.readOnly", "true")
+
+      // https://github.com/JetBrains/teamcity-webhooks
+      param("teamcity.internal.webhooks.enable", "true")
+      param("teamcity.internal.webhooks.events", "BUILD_STARTED;BUILD_FINISHED;BUILD_INTERRUPTED;CHANGES_LOADED;BUILD_TYPE_ADDED_TO_QUEUE;BUILD_PROBLEMS_CHANGED")
+      param("teamcity.internal.webhooks.url", "https://ci-stats.kibana.dev/_teamcity_webhook")
+      param("teamcity.internal.webhooks.username", "automation")
+      password("teamcity.internal.webhooks.password", "credentialsJSON:b2ee34c5-fc89-4596-9b47-ecdeb68e4e7a", display = ParameterDisplay.HIDDEN)
+    }
+
+    vcsRoot(Kibana)
+    vcsRoot(Elasticsearch)
+
+    template(DefaultTemplate)
+    template(KibanaTemplate)
+
+    defaultTemplate = DefaultTemplate
+
+    features {
+      val sizes = listOf("2", "4", "8", "16")
+      for (size in sizes) {
+        kibanaAgent(size)
+      }
+
+      kibanaAgent {
+        id = "KIBANA_C2_16"
+        param("source-id", "kibana-c2-16-")
+        param("machineType", "c2-standard-16")
+      }
+
+      feature {
+        id = "kibana"
+        type = "CloudProfile"
+        param("agentPushPreset", "")
+        param("profileId", "kibana")
+        param("profileServerUrl", "")
+        param("name", "kibana")
+        param("total-work-time", "")
+        param("credentialsType", "key")
+        param("description", "")
+        param("next-hour", "")
+        param("cloud-code", "google")
+        param("terminate-after-build", "true")
+        param("terminate-idle-time", "30")
+        param("enabled", "true")
+        param("secure:accessKey", "credentialsJSON:447fdd4d-7129-46b7-9822-2e57658c7422")
+      }
+
+      slackConnection {
+        id = "KIBANA_SLACK"
+        displayName = "Kibana Slack"
+        botToken = "credentialsJSON:39eafcfc-97a6-4877-82c1-115f1f10d14b"
+        clientId = "12985172978.1291178427153"
+        clientSecret = "credentialsJSON:8b5901fb-fd2c-4e45-8aff-fdd86dc68f67"
+      }
+    }
+
+    subProject {
+      id("CI")
+      name = "CI"
+      defaultTemplate = KibanaTemplate
+
+      buildType(Lint)
+      buildType(Checks)
+
+      subProject {
+        id("Test")
+        name = "Test"
+
+        subProject {
+          id("Jest")
+          name = "Jest"
+
+          buildType(Jest)
+          buildType(XPackJest)
+          buildType(JestIntegration)
+        }
+
+        buildType(ApiServerIntegration)
+        buildType(QuickTests)
+        buildType(AllTests)
+      }
+
+      subProject {
+        id("OSS")
+        name = "OSS Distro"
+
+        buildType(OssBuild)
+
+        subProject {
+          id("OSS_Functional")
+          name = "Functional"
+
+          buildType(OssCiGroups)
+          buildType(OssFirefox)
+          buildType(OssAccessibility)
+          buildType(OssPluginFunctional)
+
+          subProject {
+            id("CIGroups")
+            name = "CI Groups"
+
+            ossCiGroups.forEach { buildType(it) }
+          }
+        }
+      }
+
+      subProject {
+        id("Default")
+        name = "Default Distro"
+
+        buildType(DefaultBuild)
+
+        subProject {
+          id("Default_Functional")
+          name = "Functional"
+
+          buildType(DefaultCiGroups)
+          buildType(DefaultFirefox)
+          buildType(DefaultAccessibility)
+          buildType(DefaultSecuritySolution)
+          buildType(DefaultSavedObjectFieldMetrics)
+
+          subProject {
+            id("Default_CIGroups")
+            name = "CI Groups"
+
+            defaultCiGroups.forEach { buildType(it) }
+          }
+        }
+      }
+
+      buildType(FullCi)
+      buildType(BaselineCi)
+      buildType(HourlyCi)
+      buildType(PullRequestCi)
+    }
+
+    subProject(EsSnapshotsProject)
+  }
+}

.teamcity/src/templates/DefaultTemplate.kt

@@ -0,0 +1,25 @@
+package templates
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.Template
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.perfmon
+
+object DefaultTemplate : Template({
+  name = "Default Template"
+
+  requirements {
+    equals("system.cloud.profile_id", "kibana", "RQ_CLOUD_PROFILE_ID")
+    startsWith("teamcity.agent.name", "kibana-standard-2-", "RQ_AGENT_NAME")
+  }
+
+  params {
+    param("env.HOME", "/var/lib/jenkins") // TODO once the agent images are sorted out
+  }
+
+  features {
+    perfmon {  }
+  }
+
+  failureConditions {
+    executionTimeoutMin = 120
+  }
+})

.teamcity/src/templates/KibanaTemplate.kt

@@ -0,0 +1,141 @@
+package templates
+
+import vcs.Kibana
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildStep
+import jetbrains.buildServer.configs.kotlin.v2019_2.ParameterDisplay
+import jetbrains.buildServer.configs.kotlin.v2019_2.Template
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.PullRequests
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.perfmon
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.pullRequests
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.placeholder
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
+
+object KibanaTemplate : Template({
+  name = "Kibana Template"
+  description = "For builds that need to check out kibana and execute against the repo using node"
+
+  vcs {
+    root(Kibana)
+
+    checkoutDir = "kibana"
+//    checkoutDir = "/dev/shm/%system.teamcity.buildType.id%/%system.build.number%/kibana"
+  }
+
+  requirements {
+    equals("system.cloud.profile_id", "kibana", "RQ_CLOUD_PROFILE_ID")
+    startsWith("teamcity.agent.name", "kibana-standard-2-", "RQ_AGENT_NAME")
+  }
+
+  features {
+    perfmon {  }
+    pullRequests {
+      vcsRootExtId = "${Kibana.id}"
+      provider = github {
+        authType = token {
+          token = "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b"
+        }
+        filterTargetBranch = "refs/heads/master_teamcity"
+        filterAuthorRole = PullRequests.GitHubRoleFilter.MEMBER
+      }
+    }
+  }
+
+  failureConditions {
+    executionTimeoutMin = 120
+    testFailure = false
+  }
+
+  params {
+    param("env.CI", "true")
+    param("env.TEAMCITY_CI", "true")
+    param("env.HOME", "/var/lib/jenkins") // TODO once the agent images are sorted out
+
+    // TODO remove these
+    param("env.GCS_UPLOAD_PREFIX", "INVALID_PREFIX")
+    param("env.CI_PARALLEL_PROCESS_NUMBER", "1")
+
+    param("env.TEAMCITY_URL", "%teamcity.serverUrl%")
+    param("env.TEAMCITY_BUILD_URL", "%teamcity.serverUrl%/build/%teamcity.build.id%")
+    param("env.TEAMCITY_JOB_ID", "%system.teamcity.buildType.id%")
+    param("env.TEAMCITY_BUILD_ID", "%build.number%")
+    param("env.TEAMCITY_BUILD_NUMBER", "%teamcity.build.id%")
+
+    param("env.GIT_BRANCH", "%vcsroot.branch%")
+    param("env.GIT_COMMIT", "%build.vcs.number%")
+    param("env.branch_specifier", "%vcsroot.branch%")
+
+    password("env.KIBANA_CI_STATS_CONFIG", "", display = ParameterDisplay.HIDDEN)
+    password("env.CI_STATS_TOKEN", "credentialsJSON:ea975068-ca68-4da5-8189-ce90f4286bc0", display = ParameterDisplay.HIDDEN)
+    password("env.CI_STATS_HOST", "credentialsJSON:933ba93e-4b06-44c1-8724-8c536651f2b6", display = ParameterDisplay.HIDDEN)
+
+    // TODO move these to vault once the configuration is finalized
+    // password("env.CI_STATS_TOKEN", "%vault:kibana-issues:secret/kibana-issues/dev/kibana_ci_stats!/api_token%", display = ParameterDisplay.HIDDEN)
+    // password("env.CI_STATS_HOST", "%vault:kibana-issues:secret/kibana-issues/dev/kibana_ci_stats!/api_host%", display = ParameterDisplay.HIDDEN)
+
+    // TODO remove this once we are able to pull it out of vault and put it closer to the things that require it
+    password("env.GITHUB_TOKEN", "credentialsJSON:07d22002-12de-4627-91c3-672bdb23b55b", display = ParameterDisplay.HIDDEN)
+    password("env.KIBANA_CI_REPORTER_KEY", "", display = ParameterDisplay.HIDDEN)
+    password("env.KIBANA_CI_REPORTER_KEY_BASE64", "credentialsJSON:86878779-4cf7-4434-82af-5164a1b992fb", display = ParameterDisplay.HIDDEN)
+
+  }
+
+  steps {
+    script {
+      name = "Setup Environment"
+      scriptContent =
+        """
+                #!/bin/bash
+                ./.ci/teamcity/setup_env.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup Node and Yarn"
+      scriptContent =
+        """
+                #!/bin/bash
+                ./.ci/teamcity/setup_node.sh
+        """.trimIndent()
+    }
+
+    script {
+      name = "Setup CI Stats"
+      scriptContent =
+        """
+                #!/bin/bash
+                node .ci/teamcity/setup_ci_stats.js
+        """.trimIndent()
+    }
+
+    script {
+      name = "Bootstrap"
+      scriptContent =
+        """
+                #!/bin/bash
+                ./.ci/teamcity/bootstrap.sh
+        """.trimIndent()
+    }
+
+    placeholder {}
+
+    script {
+      name = "Set Build Status Success"
+      scriptContent =
+        """
+                #!/bin/bash
+                echo "##teamcity[setParameter name='env.BUILD_STATUS' value='SUCCESS']"
+        """.trimIndent()
+      executionMode = BuildStep.ExecutionMode.RUN_ON_SUCCESS
+    }
+
+    script {
+      name = "CI Stats Complete"
+      scriptContent =
+        """
+                #!/bin/bash
+                node .ci/teamcity/ci_stats_complete.js
+        """.trimIndent()
+      executionMode = BuildStep.ExecutionMode.RUN_ON_FAILURE
+    }
+  }
+})

.teamcity/src/vcs/Elasticsearch.kt

@@ -0,0 +1,11 @@
+package vcs
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
+
+object Elasticsearch : GitVcsRoot({
+  id("elasticsearch_master")
+
+  name = "elasticsearch / master"
+  url = "https://github.com/elastic/elasticsearch.git"
+  branch = "refs/heads/master"
+})

.teamcity/src/vcs/Kibana.kt

@@ -0,0 +1,11 @@
+package vcs
+
+import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot
+
+object Kibana : GitVcsRoot({
+  id("kibana_master")
+
+  name = "kibana / master"
+  url = "https://github.com/elastic/kibana.git"
+  branch = "refs/heads/master_teamcity"
+})

.teamcity/tests/projects/KibanaTest.kt

@@ -0,0 +1,27 @@
+package projects
+
+import org.junit.Assert.*
+import org.junit.Test
+
+val TestConfig = KibanaConfiguration {
+  agentNetwork = "network"
+  agentSubnet = "subnet"
+}
+
+class KibanaTest {
+  @Test
+  fun test_Default_Configuration_Exists() {
+    assertNotNull(kibanaConfiguration)
+    Kibana()
+    assertEquals("teamcity", kibanaConfiguration.agentNetwork)
+  }
+
+  @Test
+  fun test_CloudImages_Exist() {
+    val project = Kibana(TestConfig)
+
+    assertTrue(project.features.items.any {
+      it.type == "CloudImage" && it.params.any { param -> param.name == "network" && param.value == "network"}
+    })
+  }
+}

packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts

@@ -19,7 +19,7 @@
 
 import dedent from 'dedent';
 
-import { createFailureIssue, updateFailureIssue } from './report_failure';
+import { createFailureIssue, getCiType, updateFailureIssue } from './report_failure';
 
 jest.mock('./github_api');
 const { GithubApi } = jest.requireMock('./github_api');
@@ -51,7 +51,7 @@ describe('createFailureIssue()', () => {
       this is the failure text
       \`\`\`
 
-      First failure: [Jenkins Build](https://build-url)
+      First failure: [${getCiType()} Build](https://build-url)
 
       <!-- kibanaCiData = {\\"failed-test\\":{\\"test.class\\":\\"some.classname\\",\\"test.name\\":\\"test name\\",\\"test.failCount\\":1}} -->",
             Array [
@@ -111,7 +111,7 @@ describe('updateFailureIssue()', () => {
         "calls": Array [
           Array [
             1234,
-            "New failure: [Jenkins Build](https://build-url)",
+            "New failure: [${getCiType()} Build](https://build-url)",
           ],
         ],
         "results": Array [

packages/kbn-test/src/failed_tests_reporter/report_failure.ts

@@ -21,6 +21,10 @@ import { TestFailure } from './get_failures';
 import { GithubIssueMini, GithubApi } from './github_api';
 import { getIssueMetadata, updateIssueMetadata } from './issue_metadata';
 
+export function getCiType() {
+  return process.env.TEAMCITY_CI ? 'TeamCity' : 'Jenkins';
+}
+
 export async function createFailureIssue(buildUrl: string, failure: TestFailure, api: GithubApi) {
   const title = `Failing test: ${failure.classname} - ${failure.name}`;
 
@@ -32,7 +36,7 @@ export async function createFailureIssue(buildUrl: string, failure: TestFailure,
       failure.failure,
       '```',
       '',
-      `First failure: [Jenkins Build](${buildUrl})`,
+      `First failure: [${getCiType()} Build](${buildUrl})`,
     ].join('\n'),
     {
       'test.class': failure.classname,
@@ -52,7 +56,7 @@ export async function updateFailureIssue(buildUrl: string, issue: GithubIssueMin
   });
 
   await api.editIssueBodyAndEnsureOpen(issue.number, newBody);
-  await api.addIssueComment(issue.number, `New failure: [Jenkins Build](${buildUrl})`);
+  await api.addIssueComment(issue.number, `New failure: [${getCiType()} Build](${buildUrl})`);
 
   return newCount;
 }

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -33,6 +33,17 @@ import { getReportMessageIter } from './report_metadata';
 
 const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
 
+const getBranch = () => {
+  if (process.env.TEAMCITY_CI) {
+    return (process.env.GIT_BRANCH || '').replace(/^refs\/heads\//, '');
+  } else {
+    // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
+    const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
+    const branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
+    return branch;
+  }
+};
+
 export function runFailedTestsReporterCli() {
   run(
     async ({ log, flags }) => {
@@ -44,16 +55,15 @@ export function runFailedTestsReporterCli() {
       }
 
       if (updateGithub) {
-        // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
-        const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//);
-        const branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH;
+        const branch = getBranch();
         if (!branch) {
           throw createFailError(
             'Unable to determine originating branch from job name or other environment variables'
           );
         }
 
-        const isPr = !!process.env.ghprbPullId;
+        // ghprbPullId check can be removed once there are no PR jobs running on Jenkins
+        const isPr = !!process.env.GITHUB_PR_NUMBER || !!process.env.ghprbPullId;
         const isMasterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
         if (!isMasterOrVersion || isPr) {
           log.info('Failure issues only created on master/version branch jobs');
@@ -69,7 +79,9 @@ export function runFailedTestsReporterCli() {
 
       const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl');
       if (typeof buildUrl !== 'string' || !buildUrl) {
-        throw createFlagError('Missing --build-url or process.env.BUILD_URL');
+        throw createFlagError(
+          'Missing --build-url, process.env.TEAMCITY_BUILD_URL, or process.env.BUILD_URL'
+        );
       }
 
       const patterns = flags._.length ? flags._ : DEFAULT_PATTERNS;
@@ -161,12 +173,12 @@ export function runFailedTestsReporterCli() {
         default: {
           'github-update': true,
           'report-update': true,
-          'build-url': process.env.BUILD_URL,
+          'build-url': process.env.TEAMCITY_BUILD_URL || process.env.BUILD_URL,
         },
         help: `
           --no-github-update Execute the CLI without writing to Github
           --no-report-update Execute the CLI without writing to the JUnit reports
-          --build-url        URL of the failed build, defaults to process.env.BUILD_URL
+          --build-url        URL of the failed build, defaults to process.env.TEAMCITY_BUILD_URL or process.env.BUILD_URL
         `,
       },
     }

packages/kbn-test/src/mocha/__tests__/junit_report_generation.js

@@ -67,6 +67,7 @@ describe('dev/mocha/junit report generation', () => {
     expect(testsuite).to.eql({
       $: {
         failures: '2',
+        name: 'test',
         skipped: '1',
         tests: '4',
         time: testsuite.$.time,

packages/kbn-test/src/mocha/junit_report_generation.js

@@ -108,6 +108,7 @@ export function setupJUnitReportGeneration(runner, options = {}) {
     );
 
     const testsuitesEl = builder.ele('testsuite', {
+      name: reportName,
       timestamp: new Date(stats.startTime).toISOString().slice(0, -5),
       time: getDuration(stats),
       tests: allTests.length + failedHooks.length,

src/dev/precommit_hook/casing_check_config.js

@@ -70,8 +70,11 @@ export const IGNORE_FILE_GLOBS = [
   'x-pack/plugins/apm/e2e/**/*',
 
   'x-pack/plugins/maps/server/fonts/**/*',
+
   // packages for the ingest manager's api integration tests could be valid semver which has dashes
   'x-pack/test/fleet_api_integration/apis/fixtures/test_packages/**/*',
+
+  '.teamcity/**/*',
 ];
 
 /**

x-pack/test/api_integration/apis/security_solution/feature_controls.ts

@@ -82,10 +82,11 @@ export default function ({ getService }: FtrProviderContext) {
   };
 
   describe('feature controls', () => {
-    let isProd = false;
+    let isProdOrCi = false;
     before(() => {
       const kbnConfig = config.get('servers.kibana');
-      isProd = kbnConfig.hostname === 'localhost' && kbnConfig.port === 5620 ? false : true;
+      isProdOrCi =
+        !!process.env.CI || !(kbnConfig.hostname === 'localhost' && kbnConfig.port === 5620);
     });
     it(`APIs can't be accessed by user with no privileges`, async () => {
       const username = 'logstash_read';
@@ -135,7 +136,7 @@ export default function ({ getService }: FtrProviderContext) {
         expectGraphQLResponse(graphQLResult);
 
         const graphQLIResult = await executeGraphIQLRequest(username, password);
-        if (!isProd) {
+        if (!isProdOrCi) {
           expectGraphIQLResponse(graphQLIResult);
         } else {
           expectGraphIQL404(graphQLIResult);
@@ -234,7 +235,7 @@ export default function ({ getService }: FtrProviderContext) {
         expectGraphQLResponse(graphQLResult);
 
         const graphQLIResult = await executeGraphIQLRequest(username, password, space1Id);
-        if (!isProd) {
+        if (!isProdOrCi) {
           expectGraphIQLResponse(graphQLIResult);
         } else {
           expectGraphIQL404(graphQLIResult);