#!/bin/bash # ??? Should we migrate # x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space # ### Yes, it needs migration # ### Saved Object type(s) that we care about: # index-pattern # ### Test file(s) that use it: # x-pack/test/api_integration/apis/security_solution/timeline_migrations.ts # ### Config(s) that govern the test file(s): # x-pack/test/api_integration/config.ts # The other types it contains: # config # index-pattern # siem-ui-timeline # siem-ui-timeline-note # siem-ui-timeline-pinned-event # space standard_list="url,index-pattern,query,graph-workspace,tag,visualization,canvas-element,canvas-workpad,dashboard,search,lens,map,cases,uptime-dynamic-settings,osquery-saved-query,osquery-pack,infrastructure-ui-source,metrics-explorer-view,inventory-view,infrastructure-monitoring-log-view,apm-indices" orig_archive="src/platform/test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects" new_archive="x-pack/test/functional/fixtures/kbn_archiver/saved_objects_management/hidden_saved_objects" testFiles=("src/platform/test/plugin_functional/test_suites/saved_objects_management/scroll_count.ts") test_config="src/platform/test/plugin_functional/config.ts" list_stragglers() { echo "### OSS" while read -r x; do local a=$(grep -l '"index": ".kibana' "$x") if [ -n "$a" ]; then echo "${a%/mappings.json}" fi done <<<"$(find src/platform/test/functional/fixtures/es_archiver -name mappings.json)" echo echo echo "### X-PACK" while read -r y; do local b=$(grep -l '"index": ".kibana' "$y") if [ -n "$b" ]; then echo "${b%/mappings.json}" fi done <<<"$(find x-pack/test/functional/es_archives -name mappings.json)" } curl_so_count() { local so=${1:-search-session} local count count=$(curl -s -XGET "http://elastic:changeme@localhost:9220/.kibana/_count" -H "kbn-xsrf: archive-migration-functions" -H "Content-Type: application/json" -d' { "query": { "bool": { "should": [ { "match_phrase": { "type": "${so}" } } ] } } }' | jq '.count') echo " ### [${so}] count: ${count}" } watch_so_count() { local so=${1:-search-session} while true; do curl_so_count "$so" sleep 1 done } create_space() { # Ex: Id, Name, and Disabled Features. # create_space tre "Tre Space" apm,ml,canvas,dashboard,visualize,maps,monitoring,uptime # Ex: Id. Name is generated # create_space rashmi # Ex: Id and Name. No disabled features. # create_space another-space "Another Space" local id=${1:-sales} local upperCased upperCased=$(echo "${id:0:1}" | tr '[:lower:]' '[:upper:]')"${id:1}" local name=${2:-$upperCased} local disabledFeatures=${3:-} if [[ -n $3 ]]; then disabledFeatures="$3" fi # Use jq to create the i and n variables, then inject them. local payload payload=$(jq -n --arg i "$id" --arg n "$name" --arg df "$disabledFeatures" \ '{ "id": $i, "name": $n, "disabledFeatures": [$df] }') curl -H "Content-Type: application/json" -H "kbn-xsrf: archive-migration-functions" \ -X POST -d "$payload" \ --user elastic:changeme http://localhost:5620/api/spaces/space } delete_space() { local id=${1:?Need a space id.} curl -H "kbn-xsrf: archive-migration-functions" \ -X DELETE \ --user elastic:changeme http://localhost:5620/api/spaces/space/"$id" } # Just a note that this is using Gnu date. # On OSX if you don't install this, and instead use the native date you only get seconds. # With gdate you can something like nanoseconds. alias timestamp='while read line; do echo "[`gdate +%H:%M:%S.%N`] $line"; done' arrayify_csv() { local xs=${1} echo "$xs" | tr ',' '\n' | uniq | sort } intersection() { local containedTypesArr=("$@") local standardListArr standardListArr=($(arrayify_csv "$standard_list")) intersections=() for a in "${containedTypesArr[@]}"; do for b in "${standardListArr[@]}"; do if [[ "$a" == "$b" ]]; then intersections+=("$a") fi done done echo "${intersections[@]}" } is_zipped() { local archive=$1 local zip zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz' 2>/dev/null) if [[ "$zip" == "" ]]; then echo "" else echo "$zip" fi } unzip_and_keep() { local archive=$1 local zip zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz') gunzip --keep "$zip" } _guard() { local archive=$1 local failures=() local archiveJson="${archive}/data.json" if [[ ! -d "$archive" ]]; then failures+=("Archive does not exist") fi if [ "${#failures[@]}" -eq 0 ]; then local zipped zipped=$(is_zipped "$archive") if [[ "$zipped" != "" && ! -f "$archiveJson" ]]; then unzip_and_keep "$archive" fi local containsDotKibanaRefs containsDotKibanaRefs=$(grep -is '"index": ".kibana' "$archiveJson") if [[ "$containsDotKibanaRefs" == "" ]]; then failures+=(".kibana is not mentioned within the archive") fi fi echo "${failures[@]}" } _types() { local archive=${1:-$orig_archive} local archiveJson="${archive}/data.json" if [[ -f "${archive}/data.json" ]]; then local typesList typesList=$(jq 'select(.value.source.type | length > 0) | .value.source.type' "$archiveJson") echo "$typesList" | sed 's/^.//;s/.$//' | sort | uniq # The sed command drops the first and last characters. fi } _find_config() { local test_file=${1} local current local parent local grandParent local greatGrand current=$(dirname "$test_file") parent=$(dirname "$current") grandParent=$(dirname "$parent") greatGrand=$(dirname "$grandParent") local dirs=("$current" "$parent" "$grandParent" "$greatGrand") local configs=() local config for x in "${dirs[@]}"; do config=$(find "$x" -maxdepth 1 -type f -name '*config.js' -or -name '*config.ts') if [ -n "$config" ]; then configs+=("$config") fi done echo "${configs[@]}" } uniqueify() { local xs=("$@") local xss=() for x in $(printf "%s\n" "${xs[@]}" | sort -u); do if [[ -n "$x" ]]; then xss+=("$x") fi done echo "${xss[@]}" } find_configs() { local testFiles=("$@") local configs=() local uniqConfigs=() for testFile in "${testFiles[@]}"; do configs+=($(_find_config "$testFile")) done uniqConfigs=($(uniqueify "${configs[@]}")) echo "${uniqConfigs[@]}" } usages_list() { local archive=${1:-$orig_archive} grep -q "x-pack" <<<"$archive" isInXpack=$? local found_usages=() if [[ $isInXpack = 0 ]]; then found_usages+=($(find x-pack/test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq)) else found_usages+=($(find test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq)) fi echo "${found_usages[@]}" } are_enabled() { local testConfigs=("$@") local enabledList=($(node scripts/enabled_ftr_configs.js)) local needsMigration=() for enabled in "${enabledList[@]}"; do for config in "${testConfigs[@]}"; do if [[ "$enabled" == "$config" ]]; then needsMigration+=("$config") fi done done echo "${needsMigration[@]}" } print_list() { local xs=("$@") for x in "${xs[@]}"; do echo " ${x}" done } clean_up() { local xs xs=$(git status -s | grep "??" | grep "data.json" | tr '??' ' ' | xargs) for x in "${xs[@]}"; do rm -f "$x" done } should_migrate() { local archive=${1:-$orig_archive} local archiveJson="${archive}/data.json" printf "\n??? Should we migrate \n %s\n" "$archive" local guardResults=($(_guard "$archive")) if [ "${#guardResults[@]}" -eq 0 ]; then local containedTypes=($(_types "$archive")) local intersections=($(intersection "${containedTypes[@]}")) local intersectionsLength="${#intersections[@]}" if [ "${intersectionsLength}" -gt 0 ]; then local foundTestFiles=($(usages_list "$archive")) local configs=($(find_configs "${foundTestFiles[@]}")) local needsMigration=($(are_enabled "${configs[@]}")) if [ "${#needsMigration[@]}" -gt 0 ]; then echo "### Yes, it needs migration" echo " ### Saved Object type(s) that we care about:" print_list "${intersections[@]}" echo " ### Test file(s) that use it:" print_list "${foundTestFiles[@]}" echo " ### Config(s) that govern the test file(s):" print_list "${configs[@]}" else echo "!!! No, do not migrate, it's config(s) is / are not enabled." echo "!!! It contains the following saved object(s)" print_list "${containedTypes[@]}" echo " !!! For these files:" print_list "${foundTestFiles[@]}" echo " !!! Config(s) that govern the test file(s):" print_list "${configs[@]}" fi else echo "!!! No, we've found $intersectionsLength saved objects that are listed in the standard_list" echo "!!! It contains the following saved object(s)" print_list "${containedTypes[@]}" fi else echo "!!! No, it failed the guard(s):" echo "${guardResults[@]}" fi trap clean_up EXIT } migrate() { set -x node scripts/es_archiver.js load "$orig_archive" --config "$test_config" node scripts/kbn_archiver.js save "$new_archive" --config "$test_config" --type "$standard_list" node scripts/es_archiver.js unload "$orig_archive" set +x } load_logstash() { set -x node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --config "$test_config" set +x } load_es() { set -x node scripts/es_archiver.js load "$orig_archive" --config "$test_config" set +x } unload_es() { set -x node scripts/es_archiver.js unload "$orig_archive" set +x } save_es_archive_without_kibana_objects() { local indexName=${1:?"You must pass the name of the index!"} set -x node scripts/es_archiver.js save "$orig_archive" --config "$test_config" "$indexName" set +x } save_kbn() { local space=${1:-default} set -x node scripts/kbn_archiver.js --config "$test_config" save "$new_archive" --type $standard_list --space "$space" set +x # node scripts/kbn_archiver.js --config x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts save x-pack/test/functional/fixtures/kbn_archiver/saved_objects/default_space --type search,index-pattern,visualization,dashboard,lens,map,graph-workspace,query,tag,url,canvas-workpad } load_kbn() { local space=${1:-default} local archive=${2:-${new_archive}} set -x node scripts/kbn_archiver.js --config "$test_config" load "$archive" --space "$space" set +x } load_kbn_list() { local space=${1:-default} local archive=${2:-${new_archive}} local newArchives=("${:-${archive}}") for x in "${newArchives[@]}"; do load_kbn default "${x}" done } print_so_types() { set -x node scripts/saved_objs_info.js --esUrl http://elastic:changeme@localhost:9220 set +x } load_created_kbn_archive() { set -x node scripts/kbn_archiver.js --config "$test_config" load "$new_archive" set +x } unload_kbn() { local archive=${1:-${new_archive}} set -x node scripts/kbn_archiver.js --config "$test_config" unload "$archive" set +x } unload_kbns() { local space=${1:-default} for x in "${newArchives[@]}"; do set -x node scripts/kbn_archiver.js --config "$test_config" unload "$x" set +x done } ping_server() { set -x server_addr="http://elastic:changeme@localhost:9220" curl -k "${server_addr}/_cat/health?v=true" set +x } server() { local config=${1:-$test_config} set -x node scripts/functional_tests_server.js --config "$config" set +x } run_test() { local config=${1:-$test_config} set -x node scripts/functional_test_runner --config "$config" set +x } run_test_extra() { set -x node scripts/functional_test_runner --config "$test_config" \ --grep="CSV Generation from SearchSource" \ --include-tag "ciGroup2" \ --updateSnapshots set +x } allowed_types='{ "types": [ { "name": "config", "namespaceType": "single", "hidden": false, "displayName": "config" }, { "name": "url", "namespaceType": "single", "hidden": false, "displayName": "url" }, { "name": "index-pattern", "namespaceType": "multiple", "hidden": false, "displayName": "data view" }, { "name": "action", "namespaceType": "multiple-isolated", "hidden": true, "displayName": "connector" }, { "name": "query", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "query" }, { "name": "alert", "namespaceType": "multiple-isolated", "hidden": true, "displayName": "rule" }, { "name": "graph-workspace", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "graph-workspace" }, { "name": "tag", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "tag" }, { "name": "visualization", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "visualization" }, { "name": "canvas-element", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "canvas-element" }, { "name": "canvas-workpad", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "canvas-workpad" }, { "name": "dashboard", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "dashboard" }, { "name": "search", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "search" }, { "name": "lens", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "lens" }, { "name": "map", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "map" }, { "name": "cases", "namespaceType": "multiple-isolated", "hidden": true, "displayName": "cases" }, { "name": "uptime-dynamic-settings", "namespaceType": "single", "hidden": false, "displayName": "uptime-dynamic-settings" }, { "name": "osquery-saved-query", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "osquery-saved-query" }, { "name": "osquery-pack", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "osquery-pack" }, { "name": "infrastructure-ui-source", "namespaceType": "single", "hidden": false, "displayName": "infrastructure-ui-source" }, { "name": "metrics-explorer-view", "namespaceType": "single", "hidden": false, "displayName": "metrics-explorer-view" }, { "name": "inventory-view", "namespaceType": "single", "hidden": false, "displayName": "inventory-view" }, { "name": "infrastructure-monitoring-log-view", "namespaceType": "multiple-isolated", "hidden": false, "displayName": "log view" }, { "name": "apm-indices", "namespaceType": "single", "hidden": false, "displayName": "apm-indices" }, { "name": "csp_rule", "namespaceType": "agnostic", "hidden": false, "displayName": "csp_rule" }, { "name": "csp-rule-template", "namespaceType": "agnostic", "hidden": false, "displayName": "csp-rule-template" } ] }' _allowed_types() { echo "$allowed_types" | jq '.types | .[].name' | sort }