add env2yaml source files to build context tarball (#17151)

* build full docker image from dockerfiles during docker acceptance tests

ci/docker_acceptance_tests.sh

@@ -48,7 +48,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then
 elif [[ $SELECTED_TEST_SUITE == "full" ]]; then
   echo "--- Building $SELECTED_TEST_SUITE docker images"
   cd $LS_HOME
-  rake artifact:docker
+  rake artifact:build_docker_full
   echo "--- Acceptance: Installing dependencies"
   cd $QA_DIR
   bundle install

docker/Makefile

@@ -60,6 +60,7 @@ build-from-local-wolfi-artifacts: dockerfile
 
 COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
+COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/docker/env2yaml/go.mod $(ARTIFACTS_DIR)/docker/env2yaml/go.sum
 COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint
 
 $(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml
@@ -69,6 +70,9 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4
 $(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties
 $(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf
 $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
+$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go: data/logstash/env2yaml/env2yaml.go
+$(ARTIFACTS_DIR)/docker/env2yaml/go.mod: data/logstash/env2yaml/go.mod
+$(ARTIFACTS_DIR)/docker/env2yaml/go.sum: data/logstash/env2yaml/go.sum
 
 $(ARTIFACTS_DIR)/docker/%:
 	cp -f $< $@
@@ -77,6 +81,7 @@ docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/docker/
 	mkdir -p $(ARTIFACTS_DIR)/docker/bin
 	mkdir -p $(ARTIFACTS_DIR)/docker/config
+	mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
 	mkdir -p $(ARTIFACTS_DIR)/docker/pipeline
 
 COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml
@@ -122,7 +127,14 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
-	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
+	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
+
+build-from-dockerfiles_full: public-dockerfiles_full
+	cd $(ARTIFACTS_DIR)/docker && \
+	mkdir -p dockerfile_build && cd dockerfile_build && \
+	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
+	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) .
 
 public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -136,7 +148,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
-	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
+	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
 public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -150,7 +162,7 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
-	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline
+	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
 public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\

rakelib/artifacts.rake

@@ -156,8 +156,8 @@ namespace "artifact" do
   task "archives_docker" => ["prepare", "generate_build_metadata"] do
     license_details = ['ELASTIC-LICENSE']
     @bundles_jdk = true
-    create_archive_pack(license_details, "x86_64", "linux", "darwin")
+    create_archive_pack(license_details, "x86_64", "linux")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
+    create_archive_pack(license_details, "arm64", "linux")
     safe_system("./gradlew bootstrap") # force the build of Logstash jars
   end
 
@@ -222,8 +222,8 @@ namespace "artifact" do
     #with bundled JDKs
     @bundles_jdk = true
     license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths]
-    create_archive_pack(license_details, "x86_64", "linux", "darwin")
+    create_archive_pack(license_details, "x86_64", "linux")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
+    create_archive_pack(license_details, "arm64", "linux")
     safe_system("./gradlew bootstrap") # force the build of Logstash jars
   end
 
@@ -346,18 +346,39 @@ namespace "artifact" do
     build_dockerfile('oss')
   end
 
+  namespace "dockerfile_oss" do
+    desc "Build Oss Docker image from Dockerfile context files"
+    task "docker" => ["archives_docker", "dockerfile_oss"]  do
+      build_docker_from_dockerfiles('oss')
+    end
+  end
+
   desc "Generate Dockerfile for full images"
   task "dockerfile_full" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building full Dockerfiles")
     build_dockerfile('full')
   end
 
+  namespace "dockerfile_full" do
+    desc "Build Full Docker image from Dockerfile context files"
+    task "docker" => ["archives_docker", "dockerfile_full"]  do
+      build_docker_from_dockerfiles('full')
+    end
+  end
+
   desc "Generate Dockerfile for wolfi images"
   task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building wolfi Dockerfiles")
     build_dockerfile('wolfi')
   end
 
+  namespace "dockerfile_wolfi" do
+    desc "Build Wolfi Docker image from Dockerfile context files"
+    task "docker" => ["archives_docker", "dockerfile_wolfi"]  do
+      build_docker_from_dockerfiles('wolfi')
+    end
+  end
+
   desc "Generate build context for ironbank"
   task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building ironbank Dockerfiles")
@@ -386,16 +407,19 @@ namespace "artifact" do
   task "build_docker_full" => [:generate_build_metadata] do
     Rake::Task["artifact:docker"].invoke
     Rake::Task["artifact:dockerfile_full"].invoke
+    Rake::Task["artifact:dockerfile_full:docker"].invoke
   end
 
   task "build_docker_oss" => [:generate_build_metadata] do
     Rake::Task["artifact:docker_oss"].invoke
     Rake::Task["artifact:dockerfile_oss"].invoke
+    Rake::Task["artifact:dockerfile_oss:docker"].invoke
   end
 
   task "build_docker_wolfi" => [:generate_build_metadata] do
     Rake::Task["artifact:docker_wolfi"].invoke
     Rake::Task["artifact:dockerfile_wolfi"].invoke
+    Rake::Task["artifact:dockerfile_wolfi:docker"].invoke
   end
 
   task "generate_build_metadata" do
@@ -783,6 +807,19 @@ namespace "artifact" do
     end
   end
 
+  def build_docker_from_dockerfiles(flavor)
+    env = {
+      "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
+      "RELEASE" => ENV["RELEASE"],
+      "VERSION_QUALIFIER" => VERSION_QUALIFIER,
+      "BUILD_DATE" => BUILD_DATE,
+      "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS
+    }
+    Dir.chdir("docker") do |dir|
+      safe_system(env, "make build-from-dockerfiles_#{flavor}")
+    end
+  end
+
   def build_dockerfile(flavor)
     env = {
       "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),