mirror of
https://github.com/elastic/kibana.git
synced 2025-04-25 02:09:32 -04:00
7b5819c940
* [Archive Migrations] x-pack dashboard/session_in_spaces Replaces the old es archive with kbn archive. Change test to use new archive. Helps with: https://github.com/elastic/kibana/issues/102552 * fix deleting sessions * improve Co-authored-by: Anton Dosov <anton.dosov@elastic.co>
561 lines
14 KiB
Bash
561 lines
14 KiB
Bash
#!/bin/bash
|
|
|
|
standard_list="url,index-pattern,query,graph-workspace,tag,visualization,canvas-element,canvas-workpad,dashboard,search,lens,map,cases,uptime-dynamic-settings,osquery-saved-query,osquery-pack,infrastructure-ui-source,metrics-explorer-view,inventory-view,infrastructure-monitoring-log-view,apm-indices"
|
|
|
|
orig_archive="x-pack/test/functional/es_archives/dashboard/session_in_space"
|
|
new_archive="x-pack/test/functional/fixtures/kbn_archiver/dashboard/session_in_space"
|
|
newArchives=("x-pack/test/functional/fixtures/kbn_archiver/dashboard/session_in_space")
|
|
newArchives+=("x-pack/test/functional/fixtures/kbn_archiver/dashboard/session_in_another_space")
|
|
test_config="x-pack/test/search_sessions_integration/config.ts"
|
|
|
|
curl_so_count() {
|
|
local so=${1:-search-session}
|
|
local count
|
|
count=$(curl -s -XGET "http://elastic:changeme@localhost:9220/.kibana/_count" -H "kbn-xsrf: archive-migration-functions" -H "Content-Type: application/json" -d'
|
|
{
|
|
"query": {
|
|
"bool": {
|
|
"should": [
|
|
{
|
|
"match_phrase": {
|
|
"type": "${so}"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}' | jq '.count')
|
|
|
|
echo " ### [${so}] count: ${count}"
|
|
}
|
|
|
|
watch_so_count() {
|
|
local so=${1:-search-session}
|
|
|
|
while true; do
|
|
curl_so_count "$so"
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
create_space() {
|
|
# Ex: Id, Name, and Disabled Features.
|
|
# create_space tre "Tre Space" apm,ml,canvas,dashboard,visualize,maps,monitoring,uptime
|
|
# Ex: Id. Name is generated
|
|
# create_space rashmi
|
|
# Ex: Id and Name. No disabled features.
|
|
# create_space another-space "Another Space"
|
|
local id=${1:-sales}
|
|
local upperCased
|
|
upperCased=$(echo "${id:0:1}" | tr '[:lower:]' '[:upper:]')"${id:1}"
|
|
local name=${2:-$upperCased}
|
|
local disabledFeatures=${3:-}
|
|
|
|
if [[ -n $3 ]]; then
|
|
disabledFeatures="$3"
|
|
fi
|
|
|
|
# Use jq to create the i and n variables, then inject them.
|
|
local payload
|
|
payload=$(jq -n --arg i "$id" --arg n "$name" --arg df "$disabledFeatures" \
|
|
'{ "id": $i, "name": $n, "disabledFeatures": [$df] }')
|
|
|
|
curl -H "Content-Type: application/json" -H "kbn-xsrf: archive-migration-functions" \
|
|
-X POST -d "$payload" \
|
|
--user elastic:changeme http://localhost:5620/api/spaces/space
|
|
}
|
|
|
|
delete_space() {
|
|
local id=${1:?Need a space id.}
|
|
|
|
curl -H "kbn-xsrf: archive-migration-functions" \
|
|
-X DELETE \
|
|
--user elastic:changeme http://localhost:5620/api/spaces/space/"$id"
|
|
}
|
|
|
|
arrayify_csv() {
|
|
local xs=${1}
|
|
echo "$xs" | tr ',' '\n' | uniq | sort
|
|
}
|
|
|
|
intersection() {
|
|
local containedTypesArr=("$@")
|
|
local standardListArr
|
|
standardListArr=($(arrayify_csv "$standard_list"))
|
|
|
|
intersections=()
|
|
|
|
for a in "${containedTypesArr[@]}"; do
|
|
for b in "${standardListArr[@]}"; do
|
|
if [[ "$a" == "$b" ]]; then
|
|
intersections+=("$a")
|
|
fi
|
|
done
|
|
done
|
|
|
|
echo "${intersections[@]}"
|
|
}
|
|
|
|
is_zipped() {
|
|
local archive=$1
|
|
|
|
local zip
|
|
zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz' 2>/dev/null)
|
|
if [[ "$zip" == "" ]]; then
|
|
echo ""
|
|
else
|
|
echo "$zip"
|
|
fi
|
|
}
|
|
|
|
unzip_and_keep() {
|
|
local archive=$1
|
|
|
|
local zip
|
|
zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz')
|
|
gunzip --keep "$zip"
|
|
}
|
|
|
|
_guard() {
|
|
local archive=$1
|
|
local failures=()
|
|
local archiveJson="${archive}/data.json"
|
|
|
|
if [[ ! -d "$archive" ]]; then
|
|
failures+=("Archive does not exist")
|
|
fi
|
|
|
|
if [ "${#failures[@]}" -eq 0 ]; then
|
|
local zipped
|
|
zipped=$(is_zipped "$archive")
|
|
if [[ "$zipped" != "" && ! -f "$archiveJson" ]]; then
|
|
unzip_and_keep "$archive"
|
|
fi
|
|
|
|
local containsDotKibanaRefs
|
|
containsDotKibanaRefs=$(grep -is '"index": ".kibana' "$archiveJson")
|
|
if [[ "$containsDotKibanaRefs" == "" ]]; then
|
|
failures+=(".kibana is not mentioned within the archive")
|
|
fi
|
|
fi
|
|
|
|
echo "${failures[@]}"
|
|
}
|
|
|
|
_types() {
|
|
local archive=${1:-$orig_archive}
|
|
local archiveJson="${archive}/data.json"
|
|
|
|
if [[ -f "${archive}/data.json" ]]; then
|
|
local typesList
|
|
typesList=$(jq 'select(.value.source.type | length > 0) | .value.source.type' "$archiveJson")
|
|
echo "$typesList" | sed 's/^.//;s/.$//' | sort | uniq # The sed command drops the first and last characters.
|
|
fi
|
|
}
|
|
|
|
_find_config() {
|
|
local test_file=${1}
|
|
|
|
local current
|
|
local parent
|
|
local grandParent
|
|
local greatGrand
|
|
current=$(dirname "$test_file")
|
|
parent=$(dirname "$current")
|
|
grandParent=$(dirname "$parent")
|
|
greatGrand=$(dirname "$grandParent")
|
|
|
|
local dirs=("$current" "$parent" "$grandParent" "$greatGrand")
|
|
|
|
local configs=()
|
|
local config
|
|
for x in "${dirs[@]}"; do
|
|
config=$(find "$x" -maxdepth 1 -type f -name '*config.js' -or -name '*config.ts')
|
|
if [ -n "$config" ]; then
|
|
configs+=("$config")
|
|
fi
|
|
done
|
|
|
|
echo "${configs[@]}"
|
|
}
|
|
|
|
uniqueify() {
|
|
local xs=("$@")
|
|
local xss=()
|
|
for x in $(printf "%s\n" "${xs[@]}" | sort -u); do
|
|
if [[ -n "$x" ]]; then
|
|
xss+=("$x")
|
|
fi
|
|
done
|
|
echo "${xss[@]}"
|
|
}
|
|
|
|
find_configs() {
|
|
local testFiles=("$@")
|
|
local configs=()
|
|
local uniqConfigs=()
|
|
|
|
for testFile in "${testFiles[@]}"; do
|
|
configs+=($(_find_config "$testFile"))
|
|
done
|
|
|
|
uniqConfigs=($(uniqueify "${configs[@]}"))
|
|
echo "${uniqConfigs[@]}"
|
|
}
|
|
|
|
usages_list() {
|
|
local archive=${1:-$orig_archive}
|
|
|
|
grep -q "x-pack" <<<"$archive"
|
|
isInXpack=$?
|
|
|
|
local found_usages=()
|
|
|
|
if [[ $isInXpack = 0 ]]; then
|
|
found_usages+=($(find x-pack/test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq))
|
|
else
|
|
found_usages+=($(find test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq))
|
|
fi
|
|
|
|
echo "${found_usages[@]}"
|
|
}
|
|
|
|
are_enabled() {
|
|
local testConfigs=("$@")
|
|
local enabledList=($(node scripts/enabled_ftr_configs.js))
|
|
local needsMigration=()
|
|
|
|
for enabled in "${enabledList[@]}"; do
|
|
for config in "${testConfigs[@]}"; do
|
|
if [[ "$enabled" == "$config" ]]; then
|
|
needsMigration+=("$config")
|
|
fi
|
|
done
|
|
done
|
|
echo "${needsMigration[@]}"
|
|
}
|
|
|
|
print_list() {
|
|
local xs=("$@")
|
|
for x in "${xs[@]}"; do
|
|
echo " ${x}"
|
|
done
|
|
}
|
|
|
|
clean_up() {
|
|
local xs
|
|
xs=$(git status -s | grep "??" | grep "data.json" | tr '??' ' ' | xargs)
|
|
for x in "${xs[@]}"; do
|
|
rm -f "$x"
|
|
done
|
|
}
|
|
|
|
should_migrate() {
|
|
local archive=${1:-$orig_archive}
|
|
local archiveJson="${archive}/data.json"
|
|
printf "\n??? Should we migrate \n %s\n" "$archive"
|
|
|
|
local guardResults=($(_guard "$archive"))
|
|
|
|
if [ "${#guardResults[@]}" -eq 0 ]; then
|
|
local containedTypes=($(_types "$archive"))
|
|
local intersections=($(intersection "${containedTypes[@]}"))
|
|
local intersectionsLength="${#intersections[@]}"
|
|
if [ "${intersectionsLength}" -gt 0 ]; then
|
|
local foundTestFiles=($(usages_list "$archive"))
|
|
local configs=($(find_configs "${foundTestFiles[@]}"))
|
|
local needsMigration=($(are_enabled "${configs[@]}"))
|
|
if [ "${#needsMigration[@]}" -gt 0 ]; then
|
|
echo "### Yes, it needs migration"
|
|
echo " ### Saved Object type(s) that we care about:"
|
|
print_list "${intersections[@]}"
|
|
echo " ### Test file(s) that use it:"
|
|
print_list "${foundTestFiles[@]}"
|
|
echo " ### Config(s) that govern the test file(s):"
|
|
print_list "${configs[@]}"
|
|
else
|
|
echo "!!! No, do not migrate, it's config(s) is / are not enabled."
|
|
echo "!!! It contains the following saved object(s)"
|
|
print_list "${containedTypes[@]}"
|
|
echo " !!! For these files:"
|
|
print_list "${foundTestFiles[@]}"
|
|
echo " !!! Config(s) that govern the test file(s):"
|
|
print_list "${configs[@]}"
|
|
fi
|
|
else
|
|
echo "!!! No, we've found $intersectionsLength saved objects that are listed in the standard_list"
|
|
echo "!!! It contains the following saved object(s)"
|
|
print_list "${containedTypes[@]}"
|
|
fi
|
|
else
|
|
echo "!!! No, it failed the guard(s):"
|
|
echo "${guardResults[@]}"
|
|
fi
|
|
|
|
trap clean_up EXIT
|
|
}
|
|
|
|
migrate() {
|
|
set -x
|
|
node scripts/es_archiver.js load "$orig_archive" --config "$test_config"
|
|
node scripts/kbn_archiver.js save "$new_archive" --config "$test_config" --type "$standard_list"
|
|
node scripts/es_archiver.js unload "$orig_archive"
|
|
set +x
|
|
}
|
|
|
|
load_logstash() {
|
|
set -x
|
|
node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --config "$test_config"
|
|
set +x
|
|
}
|
|
|
|
load_es() {
|
|
set -x
|
|
node scripts/es_archiver.js load "$orig_archive" --config "$test_config"
|
|
set +x
|
|
}
|
|
|
|
unload_es() {
|
|
set -x
|
|
node scripts/es_archiver.js unload "$orig_archive"
|
|
set +x
|
|
}
|
|
|
|
save_es_archive_without_kibana_objects() {
|
|
local indexName=${1:?"You must pass the name of the index!"}
|
|
set -x
|
|
node scripts/es_archiver.js save "$orig_archive" --config "$test_config" "$indexName"
|
|
set +x
|
|
}
|
|
|
|
save_kbn() {
|
|
local space=${1:-default}
|
|
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" save "$new_archive" --type $standard_list --space "$space"
|
|
set +x
|
|
# node scripts/kbn_archiver.js --config x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts save x-pack/test/functional/fixtures/kbn_archiver/saved_objects/default_space --type search,index-pattern,visualization,dashboard,lens,map,graph-workspace,query,tag,url,canvas-workpad
|
|
}
|
|
|
|
load_kbn() {
|
|
local space=${1:-default}
|
|
local archive=${2:-${new_archive}}
|
|
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" load "$archive" --space "$space"
|
|
set +x
|
|
}
|
|
|
|
unload_kbn() {
|
|
local archive=${1:-${new_archive}}
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" unload "$archive"
|
|
set +x
|
|
}
|
|
|
|
load_kbn_custom() {
|
|
load_kbn default "${newArchives[1]}"
|
|
create_space another-space "Another Space"
|
|
load_kbn another-space "${newArchives[2]}"
|
|
}
|
|
|
|
unload_kbn_custom() {
|
|
unload_kbn "${newArchives[1]}"
|
|
delete_space another-space
|
|
}
|
|
|
|
ping_server() {
|
|
set -x
|
|
server_addr="http://elastic:changeme@localhost:9220"
|
|
curl -k "${server_addr}/_cat/health?v=true"
|
|
set +x
|
|
}
|
|
|
|
server() {
|
|
local config=${1:-$test_config}
|
|
|
|
set -x
|
|
node scripts/functional_tests_server.js --config "$config"
|
|
set +x
|
|
}
|
|
|
|
run_test() {
|
|
local config=${1:-$test_config}
|
|
|
|
set -x
|
|
node scripts/functional_test_runner --config "$config"
|
|
set +x
|
|
}
|
|
|
|
run_test_extra() {
|
|
set -x
|
|
node scripts/functional_test_runner --config "$test_config" \
|
|
--grep="CSV Generation from SearchSource" \
|
|
--include-tag "ciGroup2" \
|
|
--updateSnapshots
|
|
set +x
|
|
}
|
|
|
|
allowed_types='{
|
|
"types": [
|
|
{
|
|
"name": "config",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "config"
|
|
},
|
|
{
|
|
"name": "url",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "url"
|
|
},
|
|
{
|
|
"name": "index-pattern",
|
|
"namespaceType": "multiple",
|
|
"hidden": false,
|
|
"displayName": "data view"
|
|
},
|
|
{
|
|
"name": "action",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "connector"
|
|
},
|
|
{
|
|
"name": "query",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "query"
|
|
},
|
|
{
|
|
"name": "alert",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "rule"
|
|
},
|
|
{
|
|
"name": "graph-workspace",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "graph-workspace"
|
|
},
|
|
{
|
|
"name": "tag",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "tag"
|
|
},
|
|
{
|
|
"name": "visualization",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "visualization"
|
|
},
|
|
{
|
|
"name": "canvas-element",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "canvas-element"
|
|
},
|
|
{
|
|
"name": "canvas-workpad",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "canvas-workpad"
|
|
},
|
|
{
|
|
"name": "dashboard",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "dashboard"
|
|
},
|
|
{
|
|
"name": "search",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "search"
|
|
},
|
|
{
|
|
"name": "lens",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "lens"
|
|
},
|
|
{
|
|
"name": "map",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "map"
|
|
},
|
|
{
|
|
"name": "cases",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "cases"
|
|
},
|
|
{
|
|
"name": "uptime-dynamic-settings",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "uptime-dynamic-settings"
|
|
},
|
|
{
|
|
"name": "osquery-saved-query",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "osquery-saved-query"
|
|
},
|
|
{
|
|
"name": "osquery-pack",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "osquery-pack"
|
|
},
|
|
{
|
|
"name": "infrastructure-ui-source",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "infrastructure-ui-source"
|
|
},
|
|
{
|
|
"name": "metrics-explorer-view",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "metrics-explorer-view"
|
|
},
|
|
{
|
|
"name": "inventory-view",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "inventory-view"
|
|
},
|
|
{
|
|
"name": "infrastructure-monitoring-log-view",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "log view"
|
|
},
|
|
{
|
|
"name": "apm-indices",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "apm-indices"
|
|
},
|
|
{
|
|
"name": "csp_rule",
|
|
"namespaceType": "agnostic",
|
|
"hidden": false,
|
|
"displayName": "csp_rule"
|
|
},
|
|
{
|
|
"name": "csp-rule-template",
|
|
"namespaceType": "agnostic",
|
|
"hidden": false,
|
|
"displayName": "csp-rule-template"
|
|
}
|
|
]
|
|
}'
|
|
_allowed_types() {
|
|
echo "$allowed_types" | jq '.types | .[].name' | sort
|
|
}
|