mirror of
https://github.com/elastic/kibana.git
synced 2025-04-25 10:23:14 -04:00
d1feb43722
## Summary Resolves https://github.com/elastic/kibana/issues/136856 - Add async keyword where it should have been. - Split archive into kbn and es archives... instigated by Pierre's comment: https://github.com/elastic/kibana/issues/136856#issuecomment-1253609280 - Note: Had to use both archive types as one archive is "hidden", and cannot be accessed via the kbn client. - Unload via Pierre's "new" clean method. - Add shell fn to print out server's currently loaded so's, with user and pass hardcoded for local dev.
627 lines
15 KiB
Bash
Executable file
627 lines
15 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
# ??? Should we migrate
|
|
# x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space
|
|
# ### Yes, it needs migration
|
|
# ### Saved Object type(s) that we care about:
|
|
# index-pattern
|
|
# ### Test file(s) that use it:
|
|
# x-pack/test/api_integration/apis/security_solution/timeline_migrations.ts
|
|
# ### Config(s) that govern the test file(s):
|
|
# x-pack/test/api_integration/config.ts
|
|
# The other types it contains:
|
|
# config
|
|
# index-pattern
|
|
# siem-ui-timeline
|
|
# siem-ui-timeline-note
|
|
# siem-ui-timeline-pinned-event
|
|
# space
|
|
|
|
standard_list="url,index-pattern,query,graph-workspace,tag,visualization,canvas-element,canvas-workpad,dashboard,search,lens,map,cases,uptime-dynamic-settings,osquery-saved-query,osquery-pack,infrastructure-ui-source,metrics-explorer-view,inventory-view,infrastructure-monitoring-log-view,apm-indices"
|
|
|
|
orig_archive="test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects"
|
|
new_archive="x-pack/test/functional/fixtures/kbn_archiver/saved_objects_management/hidden_saved_objects"
|
|
testFiles=("test/plugin_functional/test_suites/saved_objects_management/scroll_count.ts")
|
|
|
|
test_config="test/plugin_functional/config.ts"
|
|
|
|
list_stragglers() {
|
|
|
|
echo "### OSS"
|
|
while read -r x; do
|
|
local a=$(grep -l '"index": ".kibana' "$x")
|
|
if [ -n "$a" ]; then
|
|
echo "${a%/mappings.json}"
|
|
fi
|
|
done <<<"$(find test/functional/fixtures/es_archiver -name mappings.json)"
|
|
|
|
echo
|
|
echo
|
|
|
|
echo "### X-PACK"
|
|
while read -r y; do
|
|
local b=$(grep -l '"index": ".kibana' "$y")
|
|
if [ -n "$b" ]; then
|
|
echo "${b%/mappings.json}"
|
|
fi
|
|
done <<<"$(find x-pack/test/functional/es_archives -name mappings.json)"
|
|
|
|
}
|
|
|
|
curl_so_count() {
|
|
local so=${1:-search-session}
|
|
local count
|
|
count=$(curl -s -XGET "http://elastic:changeme@localhost:9220/.kibana/_count" -H "kbn-xsrf: archive-migration-functions" -H "Content-Type: application/json" -d'
|
|
{
|
|
"query": {
|
|
"bool": {
|
|
"should": [
|
|
{
|
|
"match_phrase": {
|
|
"type": "${so}"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}' | jq '.count')
|
|
|
|
echo " ### [${so}] count: ${count}"
|
|
}
|
|
|
|
watch_so_count() {
|
|
local so=${1:-search-session}
|
|
|
|
while true; do
|
|
curl_so_count "$so"
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
create_space() {
|
|
# Ex: Id, Name, and Disabled Features.
|
|
# create_space tre "Tre Space" apm,ml,canvas,dashboard,visualize,maps,monitoring,uptime
|
|
# Ex: Id. Name is generated
|
|
# create_space rashmi
|
|
# Ex: Id and Name. No disabled features.
|
|
# create_space another-space "Another Space"
|
|
local id=${1:-sales}
|
|
local upperCased
|
|
upperCased=$(echo "${id:0:1}" | tr '[:lower:]' '[:upper:]')"${id:1}"
|
|
local name=${2:-$upperCased}
|
|
local disabledFeatures=${3:-}
|
|
|
|
if [[ -n $3 ]]; then
|
|
disabledFeatures="$3"
|
|
fi
|
|
|
|
# Use jq to create the i and n variables, then inject them.
|
|
local payload
|
|
payload=$(jq -n --arg i "$id" --arg n "$name" --arg df "$disabledFeatures" \
|
|
'{ "id": $i, "name": $n, "disabledFeatures": [$df] }')
|
|
|
|
curl -H "Content-Type: application/json" -H "kbn-xsrf: archive-migration-functions" \
|
|
-X POST -d "$payload" \
|
|
--user elastic:changeme http://localhost:5620/api/spaces/space
|
|
}
|
|
|
|
delete_space() {
|
|
local id=${1:?Need a space id.}
|
|
|
|
curl -H "kbn-xsrf: archive-migration-functions" \
|
|
-X DELETE \
|
|
--user elastic:changeme http://localhost:5620/api/spaces/space/"$id"
|
|
}
|
|
|
|
# Just a note that this is using Gnu date.
|
|
# On OSX if you don't install this, and instead use the native date you only get seconds.
|
|
# With gdate you can something like nanoseconds.
|
|
alias timestamp='while read line; do echo "[`gdate +%H:%M:%S.%N`] $line"; done'
|
|
|
|
arrayify_csv() {
|
|
local xs=${1}
|
|
echo "$xs" | tr ',' '\n' | uniq | sort
|
|
}
|
|
|
|
intersection() {
|
|
local containedTypesArr=("$@")
|
|
local standardListArr
|
|
standardListArr=($(arrayify_csv "$standard_list"))
|
|
|
|
intersections=()
|
|
|
|
for a in "${containedTypesArr[@]}"; do
|
|
for b in "${standardListArr[@]}"; do
|
|
if [[ "$a" == "$b" ]]; then
|
|
intersections+=("$a")
|
|
fi
|
|
done
|
|
done
|
|
|
|
echo "${intersections[@]}"
|
|
}
|
|
|
|
is_zipped() {
|
|
local archive=$1
|
|
|
|
local zip
|
|
zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz' 2>/dev/null)
|
|
if [[ "$zip" == "" ]]; then
|
|
echo ""
|
|
else
|
|
echo "$zip"
|
|
fi
|
|
}
|
|
|
|
unzip_and_keep() {
|
|
local archive=$1
|
|
|
|
local zip
|
|
zip=$(find "$archive" -maxdepth 1 -type f -name '*.gz')
|
|
gunzip --keep "$zip"
|
|
}
|
|
|
|
_guard() {
|
|
local archive=$1
|
|
local failures=()
|
|
local archiveJson="${archive}/data.json"
|
|
|
|
if [[ ! -d "$archive" ]]; then
|
|
failures+=("Archive does not exist")
|
|
fi
|
|
|
|
if [ "${#failures[@]}" -eq 0 ]; then
|
|
local zipped
|
|
zipped=$(is_zipped "$archive")
|
|
if [[ "$zipped" != "" && ! -f "$archiveJson" ]]; then
|
|
unzip_and_keep "$archive"
|
|
fi
|
|
|
|
local containsDotKibanaRefs
|
|
containsDotKibanaRefs=$(grep -is '"index": ".kibana' "$archiveJson")
|
|
if [[ "$containsDotKibanaRefs" == "" ]]; then
|
|
failures+=(".kibana is not mentioned within the archive")
|
|
fi
|
|
fi
|
|
|
|
echo "${failures[@]}"
|
|
}
|
|
|
|
_types() {
|
|
local archive=${1:-$orig_archive}
|
|
local archiveJson="${archive}/data.json"
|
|
|
|
if [[ -f "${archive}/data.json" ]]; then
|
|
local typesList
|
|
typesList=$(jq 'select(.value.source.type | length > 0) | .value.source.type' "$archiveJson")
|
|
echo "$typesList" | sed 's/^.//;s/.$//' | sort | uniq # The sed command drops the first and last characters.
|
|
fi
|
|
}
|
|
|
|
_find_config() {
|
|
local test_file=${1}
|
|
|
|
local current
|
|
local parent
|
|
local grandParent
|
|
local greatGrand
|
|
current=$(dirname "$test_file")
|
|
parent=$(dirname "$current")
|
|
grandParent=$(dirname "$parent")
|
|
greatGrand=$(dirname "$grandParent")
|
|
|
|
local dirs=("$current" "$parent" "$grandParent" "$greatGrand")
|
|
|
|
local configs=()
|
|
local config
|
|
for x in "${dirs[@]}"; do
|
|
config=$(find "$x" -maxdepth 1 -type f -name '*config.js' -or -name '*config.ts')
|
|
if [ -n "$config" ]; then
|
|
configs+=("$config")
|
|
fi
|
|
done
|
|
|
|
echo "${configs[@]}"
|
|
}
|
|
|
|
uniqueify() {
|
|
local xs=("$@")
|
|
local xss=()
|
|
for x in $(printf "%s\n" "${xs[@]}" | sort -u); do
|
|
if [[ -n "$x" ]]; then
|
|
xss+=("$x")
|
|
fi
|
|
done
|
|
echo "${xss[@]}"
|
|
}
|
|
|
|
find_configs() {
|
|
local testFiles=("$@")
|
|
local configs=()
|
|
local uniqConfigs=()
|
|
|
|
for testFile in "${testFiles[@]}"; do
|
|
configs+=($(_find_config "$testFile"))
|
|
done
|
|
|
|
uniqConfigs=($(uniqueify "${configs[@]}"))
|
|
echo "${uniqConfigs[@]}"
|
|
}
|
|
|
|
usages_list() {
|
|
local archive=${1:-$orig_archive}
|
|
|
|
grep -q "x-pack" <<<"$archive"
|
|
isInXpack=$?
|
|
|
|
local found_usages=()
|
|
|
|
if [[ $isInXpack = 0 ]]; then
|
|
found_usages+=($(find x-pack/test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq))
|
|
else
|
|
found_usages+=($(find test -type f -print0 | xargs -0 grep -n "$archive" | cut -d ':' -f 1 | uniq))
|
|
fi
|
|
|
|
echo "${found_usages[@]}"
|
|
}
|
|
|
|
are_enabled() {
|
|
local testConfigs=("$@")
|
|
local enabledList=($(node scripts/enabled_ftr_configs.js))
|
|
local needsMigration=()
|
|
|
|
for enabled in "${enabledList[@]}"; do
|
|
for config in "${testConfigs[@]}"; do
|
|
if [[ "$enabled" == "$config" ]]; then
|
|
needsMigration+=("$config")
|
|
fi
|
|
done
|
|
done
|
|
echo "${needsMigration[@]}"
|
|
}
|
|
|
|
print_list() {
|
|
local xs=("$@")
|
|
for x in "${xs[@]}"; do
|
|
echo " ${x}"
|
|
done
|
|
}
|
|
|
|
clean_up() {
|
|
local xs
|
|
xs=$(git status -s | grep "??" | grep "data.json" | tr '??' ' ' | xargs)
|
|
for x in "${xs[@]}"; do
|
|
rm -f "$x"
|
|
done
|
|
}
|
|
|
|
should_migrate() {
|
|
local archive=${1:-$orig_archive}
|
|
local archiveJson="${archive}/data.json"
|
|
printf "\n??? Should we migrate \n %s\n" "$archive"
|
|
|
|
local guardResults=($(_guard "$archive"))
|
|
|
|
if [ "${#guardResults[@]}" -eq 0 ]; then
|
|
local containedTypes=($(_types "$archive"))
|
|
local intersections=($(intersection "${containedTypes[@]}"))
|
|
local intersectionsLength="${#intersections[@]}"
|
|
if [ "${intersectionsLength}" -gt 0 ]; then
|
|
local foundTestFiles=($(usages_list "$archive"))
|
|
local configs=($(find_configs "${foundTestFiles[@]}"))
|
|
local needsMigration=($(are_enabled "${configs[@]}"))
|
|
if [ "${#needsMigration[@]}" -gt 0 ]; then
|
|
echo "### Yes, it needs migration"
|
|
echo " ### Saved Object type(s) that we care about:"
|
|
print_list "${intersections[@]}"
|
|
echo " ### Test file(s) that use it:"
|
|
print_list "${foundTestFiles[@]}"
|
|
echo " ### Config(s) that govern the test file(s):"
|
|
print_list "${configs[@]}"
|
|
else
|
|
echo "!!! No, do not migrate, it's config(s) is / are not enabled."
|
|
echo "!!! It contains the following saved object(s)"
|
|
print_list "${containedTypes[@]}"
|
|
echo " !!! For these files:"
|
|
print_list "${foundTestFiles[@]}"
|
|
echo " !!! Config(s) that govern the test file(s):"
|
|
print_list "${configs[@]}"
|
|
fi
|
|
else
|
|
echo "!!! No, we've found $intersectionsLength saved objects that are listed in the standard_list"
|
|
echo "!!! It contains the following saved object(s)"
|
|
print_list "${containedTypes[@]}"
|
|
fi
|
|
else
|
|
echo "!!! No, it failed the guard(s):"
|
|
echo "${guardResults[@]}"
|
|
fi
|
|
|
|
trap clean_up EXIT
|
|
}
|
|
|
|
migrate() {
|
|
set -x
|
|
node scripts/es_archiver.js load "$orig_archive" --config "$test_config"
|
|
node scripts/kbn_archiver.js save "$new_archive" --config "$test_config" --type "$standard_list"
|
|
node scripts/es_archiver.js unload "$orig_archive"
|
|
set +x
|
|
}
|
|
|
|
load_logstash() {
|
|
set -x
|
|
node scripts/es_archiver.js load x-pack/test/functional/es_archives/logstash_functional --config "$test_config"
|
|
set +x
|
|
}
|
|
|
|
load_es() {
|
|
set -x
|
|
node scripts/es_archiver.js load "$orig_archive" --config "$test_config"
|
|
set +x
|
|
}
|
|
|
|
unload_es() {
|
|
set -x
|
|
node scripts/es_archiver.js unload "$orig_archive"
|
|
set +x
|
|
}
|
|
|
|
save_es_archive_without_kibana_objects() {
|
|
local indexName=${1:?"You must pass the name of the index!"}
|
|
set -x
|
|
node scripts/es_archiver.js save "$orig_archive" --config "$test_config" "$indexName"
|
|
set +x
|
|
}
|
|
|
|
save_kbn() {
|
|
local space=${1:-default}
|
|
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" save "$new_archive" --type $standard_list --space "$space"
|
|
set +x
|
|
# node scripts/kbn_archiver.js --config x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts save x-pack/test/functional/fixtures/kbn_archiver/saved_objects/default_space --type search,index-pattern,visualization,dashboard,lens,map,graph-workspace,query,tag,url,canvas-workpad
|
|
}
|
|
|
|
load_kbn() {
|
|
local space=${1:-default}
|
|
local archive=${2:-${new_archive}}
|
|
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" load "$archive" --space "$space"
|
|
set +x
|
|
}
|
|
|
|
load_kbn_list() {
|
|
local space=${1:-default}
|
|
local archive=${2:-${new_archive}}
|
|
local newArchives=("${:-${archive}}")
|
|
|
|
for x in "${newArchives[@]}"; do
|
|
load_kbn default "${x}"
|
|
done
|
|
}
|
|
|
|
print_so_types() {
|
|
set -x
|
|
node scripts/saved_objs_info.js --esUrl http://elastic:changeme@localhost:9220
|
|
set +x
|
|
}
|
|
|
|
load_created_kbn_archive() {
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" load "$new_archive"
|
|
set +x
|
|
}
|
|
|
|
unload_kbn() {
|
|
local archive=${1:-${new_archive}}
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" unload "$archive"
|
|
set +x
|
|
}
|
|
|
|
unload_kbns() {
|
|
local space=${1:-default}
|
|
|
|
for x in "${newArchives[@]}"; do
|
|
set -x
|
|
node scripts/kbn_archiver.js --config "$test_config" unload "$x"
|
|
set +x
|
|
done
|
|
}
|
|
|
|
ping_server() {
|
|
set -x
|
|
server_addr="http://elastic:changeme@localhost:9220"
|
|
curl -k "${server_addr}/_cat/health?v=true"
|
|
set +x
|
|
}
|
|
|
|
server() {
|
|
local config=${1:-$test_config}
|
|
|
|
set -x
|
|
node scripts/functional_tests_server.js --config "$config"
|
|
set +x
|
|
}
|
|
|
|
run_test() {
|
|
local config=${1:-$test_config}
|
|
|
|
set -x
|
|
node scripts/functional_test_runner --config "$config"
|
|
set +x
|
|
}
|
|
|
|
run_test_extra() {
|
|
set -x
|
|
node scripts/functional_test_runner --config "$test_config" \
|
|
--grep="CSV Generation from SearchSource" \
|
|
--include-tag "ciGroup2" \
|
|
--updateSnapshots
|
|
set +x
|
|
}
|
|
|
|
allowed_types='{
|
|
"types": [
|
|
{
|
|
"name": "config",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "config"
|
|
},
|
|
{
|
|
"name": "url",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "url"
|
|
},
|
|
{
|
|
"name": "index-pattern",
|
|
"namespaceType": "multiple",
|
|
"hidden": false,
|
|
"displayName": "data view"
|
|
},
|
|
{
|
|
"name": "action",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "connector"
|
|
},
|
|
{
|
|
"name": "query",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "query"
|
|
},
|
|
{
|
|
"name": "alert",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "rule"
|
|
},
|
|
{
|
|
"name": "graph-workspace",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "graph-workspace"
|
|
},
|
|
{
|
|
"name": "tag",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "tag"
|
|
},
|
|
{
|
|
"name": "visualization",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "visualization"
|
|
},
|
|
{
|
|
"name": "canvas-element",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "canvas-element"
|
|
},
|
|
{
|
|
"name": "canvas-workpad",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "canvas-workpad"
|
|
},
|
|
{
|
|
"name": "dashboard",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "dashboard"
|
|
},
|
|
{
|
|
"name": "search",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "search"
|
|
},
|
|
{
|
|
"name": "lens",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "lens"
|
|
},
|
|
{
|
|
"name": "map",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "map"
|
|
},
|
|
{
|
|
"name": "cases",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": true,
|
|
"displayName": "cases"
|
|
},
|
|
{
|
|
"name": "uptime-dynamic-settings",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "uptime-dynamic-settings"
|
|
},
|
|
{
|
|
"name": "osquery-saved-query",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "osquery-saved-query"
|
|
},
|
|
{
|
|
"name": "osquery-pack",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "osquery-pack"
|
|
},
|
|
{
|
|
"name": "infrastructure-ui-source",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "infrastructure-ui-source"
|
|
},
|
|
{
|
|
"name": "metrics-explorer-view",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "metrics-explorer-view"
|
|
},
|
|
{
|
|
"name": "inventory-view",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "inventory-view"
|
|
},
|
|
{
|
|
"name": "infrastructure-monitoring-log-view",
|
|
"namespaceType": "multiple-isolated",
|
|
"hidden": false,
|
|
"displayName": "log view"
|
|
},
|
|
{
|
|
"name": "apm-indices",
|
|
"namespaceType": "single",
|
|
"hidden": false,
|
|
"displayName": "apm-indices"
|
|
},
|
|
{
|
|
"name": "csp_rule",
|
|
"namespaceType": "agnostic",
|
|
"hidden": false,
|
|
"displayName": "csp_rule"
|
|
},
|
|
{
|
|
"name": "csp-rule-template",
|
|
"namespaceType": "agnostic",
|
|
"hidden": false,
|
|
"displayName": "csp-rule-template"
|
|
}
|
|
]
|
|
}'
|
|
_allowed_types() {
|
|
echo "$allowed_types" | jq '.types | .[].name' | sort
|
|
}
|