From fecfc7c6021ea55d42dc85371b968f29b982188c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Fri, 28 Feb 2025 11:34:06 +0000 Subject: [PATCH] add env2yaml source files to build context tarball (#17151) * build full docker image from dockerfiles during docker acceptance tests --- ci/docker_acceptance_tests.sh | 2 +- docker/Makefile | 18 +++++++++++--- rakelib/artifacts.rake | 45 +++++++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index 86a721a15..b1e62de2c 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -48,7 +48,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:docker + rake artifact:build_docker_full echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/docker/Makefile b/docker/Makefile index 8a8fc4b70..0fe30b36f 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -60,6 +60,7 @@ build-from-local-wolfi-artifacts: dockerfile COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties +COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/docker/env2yaml/go.mod $(ARTIFACTS_DIR)/docker/env2yaml/go.sum COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml @@ -69,6 +70,9 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint +$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go +$(ARTIFACTS_DIR)/docker/env2yaml/go.mod: data/logstash/env2yaml/go.mod +$(ARTIFACTS_DIR)/docker/env2yaml/go.sum: data/logstash/env2yaml/go.sum $(ARTIFACTS_DIR)/docker/%: cp -f $< $@ @@ -77,6 +81,7 @@ docker_paths: mkdir -p $(ARTIFACTS_DIR)/docker/ mkdir -p $(ARTIFACTS_DIR)/docker/bin mkdir -p $(ARTIFACTS_DIR)/docker/config + mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml mkdir -p $(ARTIFACTS_DIR)/docker/pipeline COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml @@ -122,7 +127,14 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ - tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline + tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline + +build-from-dockerfiles_full: public-dockerfiles_full + cd $(ARTIFACTS_DIR)/docker && \ + mkdir -p dockerfile_build && cd dockerfile_build && \ + tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ + sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ + docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -136,7 +148,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ - tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline + tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -150,7 +162,7 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ - tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline + tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 2788debe0..a677fa1f8 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -156,8 +156,8 @@ namespace "artifact" do task "archives_docker" => ["prepare", "generate_build_metadata"] do license_details = ['ELASTIC-LICENSE'] @bundles_jdk = true - create_archive_pack(license_details, "x86_64", "linux", "darwin") - create_archive_pack(license_details, "arm64", "linux", "darwin") + create_archive_pack(license_details, "x86_64", "linux") + create_archive_pack(license_details, "arm64", "linux") safe_system("./gradlew bootstrap") # force the build of Logstash jars end @@ -222,8 +222,8 @@ namespace "artifact" do #with bundled JDKs @bundles_jdk = true license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths] - create_archive_pack(license_details, "x86_64", "linux", "darwin") - create_archive_pack(license_details, "arm64", "linux", "darwin") + create_archive_pack(license_details, "x86_64", "linux") + create_archive_pack(license_details, "arm64", "linux") safe_system("./gradlew bootstrap") # force the build of Logstash jars end @@ -346,18 +346,39 @@ namespace "artifact" do build_dockerfile('oss') end + namespace "dockerfile_oss" do + desc "Build Oss Docker image from Dockerfile context files" + task "docker" => ["archives_docker", "dockerfile_oss"] do + build_docker_from_dockerfiles('oss') + end + end + desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building full Dockerfiles") build_dockerfile('full') end + namespace "dockerfile_full" do + desc "Build Full Docker image from Dockerfile context files" + task "docker" => ["archives_docker", "dockerfile_full"] do + build_docker_from_dockerfiles('full') + end + end + desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end + namespace "dockerfile_wolfi" do + desc "Build Wolfi Docker image from Dockerfile context files" + task "docker" => ["archives_docker", "dockerfile_wolfi"] do + build_docker_from_dockerfiles('wolfi') + end + end + desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -386,16 +407,19 @@ namespace "artifact" do task "build_docker_full" => [:generate_build_metadata] do Rake::Task["artifact:docker"].invoke Rake::Task["artifact:dockerfile_full"].invoke + Rake::Task["artifact:dockerfile_full:docker"].invoke end task "build_docker_oss" => [:generate_build_metadata] do Rake::Task["artifact:docker_oss"].invoke Rake::Task["artifact:dockerfile_oss"].invoke + Rake::Task["artifact:dockerfile_oss:docker"].invoke end task "build_docker_wolfi" => [:generate_build_metadata] do Rake::Task["artifact:docker_wolfi"].invoke Rake::Task["artifact:dockerfile_wolfi"].invoke + Rake::Task["artifact:dockerfile_wolfi:docker"].invoke end task "generate_build_metadata" do @@ -783,6 +807,19 @@ namespace "artifact" do end end + def build_docker_from_dockerfiles(flavor) + env = { + "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), + "RELEASE" => ENV["RELEASE"], + "VERSION_QUALIFIER" => VERSION_QUALIFIER, + "BUILD_DATE" => BUILD_DATE, + "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + } + Dir.chdir("docker") do |dir| + safe_system(env, "make build-from-dockerfiles_#{flavor}") + end + end + def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),