From cc7e731b0863a3b752caa83dbdb1776198f79d4e Mon Sep 17 00:00:00 2001 From: Rahul Raj D N Date: Thu, 8 May 2025 00:55:46 -0400 Subject: [PATCH] Created script for running apptainer; Added instructions in README with more build details --- .gitignore | 2 +- miscs/apptainer/README.md | 82 ++++++++++++++++++- miscs/apptainer/run_apptainer.sh | 20 +++++ .../apptainer/{vortex_fpga.def => vortex.def} | 39 +++++---- 4 files changed, 121 insertions(+), 22 deletions(-) create mode 100755 miscs/apptainer/run_apptainer.sh rename miscs/apptainer/{vortex_fpga.def => vortex.def} (70%) diff --git a/.gitignore b/.gitignore index 475634868..e188b4c5b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ /.vscode *.cache *.code-workspace -miscs/apptainer/vortex_fpga.sif +miscs/apptainer/vortex.sif diff --git a/miscs/apptainer/README.md b/miscs/apptainer/README.md index 6da64d62c..13511d6e3 100644 --- a/miscs/apptainer/README.md +++ b/miscs/apptainer/README.md @@ -11,29 +11,105 @@ Go to `apptainer` directory $ pwd vortex/miscs/apptainer -$ apptainer build --no-https vortex_fpga.sif vortex_fpga.def +$ apptainer build --no-https vortex.sif vortex.def ``` + + To start the apptainer, ``` -$ apptainer shell --fakeroot --cleanenv --writable-tmpfs --bind /opt/xilinx/:/opt/xilinx/ --bind /netscratch/rn84/devnull:/dev/null --bind /dev/bus/usb,/sys/bus/pci --bind /projects:/projects --bind /tools:/tools --bind /netscratch:/netscratch vortex_fpga.sif +$ chmod +x run_apptainer.sh +$ ./run_apptainer.sh ``` + +| Note: Set env variables in `run_apptainer.sh` accordingly. + + Inside the Apptainer, ``` +# should show devices connected to machine on which you are running this command Apptainer> lsusb -should show devices connected to machine on which you are running this command ``` +# Vortex Simulation inside Apptainer + +Go to bind of vortex repo, +``` +Apptainer> cd /home/vortex +Apptainer> mkdir build +Apptainer> cd build +Apptainer> ../configure --xlen=32 --tooldir=$HOME/tools + + +Skip the below 3 steps, If tools are already present in the $HOME/tools +Apptainer> sed -i 's/\btar /tar --no-same-owner /g' ci/toolchain_install.sh +Apptainer> ./ci/toolchain_install.sh --all +Apptainer> sed -i 's/\btar --no-same-owner /tar /g' ci/toolchain_install.sh + + +Apptainer> source ./ci/toolchain_env.sh +Apptainer> verilator --version +``` + + +### Running SIMX, RTLSIM and XRTSIM +``` +Compile the Vortex codebase +Apptainer> make -s + +Run the programs by specifying the appropriate driver as shown below: + +SIMX +Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=simx + +RTLSIM +Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=rtlsim + +XRTSIM +Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=xrt +``` + + +### Vortex Bitstream on FPGA + +Common Commands ``` Apptainer> source /opt/xilinx/xrt/setup.sh Apptainer> source /tools/reconfig/xilinx/Vitis/2023.1/settings64.sh Apptainer> platforminfo -l + ``` +##### Building Vortex Bitstream +``` +Apptainer> pwd +/home/vortex/build +Apptainer> source ci/toolchain_env.sh +Apptainer> verilator --version +Verilator 5.026 2024-06-15 rev v5.026-43-g065f36ab5 + +Apptainer> cd hw/syn/xilinx/xrt +Apptainer> PREFIX=test1 PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 TARGET=hw NUM_CORES=1 make > build_u50_hw_1c.log 2>&1 & +Creates ../test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin/vortex_afu.xclbin +``` + +##### Running Vortex Bitstream on FPGA +``` +Apptainer> cd ../../../../ +Apptainer> pwd +/home/vortex/build + +Apptainer> make -C runtime/ clean + +Apptainer> FPGA_BIN_DIR=hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin TARGET=hw PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 ./ci/blackbox.sh --driver=xrt --app=demo + +Verify following line being printed: +info: device name=xilinx_u50_gen3x16_xdma_base_5, memory_capacity=0x200000000 bytes, memory_banks=32. +``` \ No newline at end of file diff --git a/miscs/apptainer/run_apptainer.sh b/miscs/apptainer/run_apptainer.sh new file mode 100755 index 000000000..aea5c7e5d --- /dev/null +++ b/miscs/apptainer/run_apptainer.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Set environment variables with fallback default values. +# The :- means if the variable CONTAINER_IMAGE is not set or is empty, then use the default value vortex.sif +export GIT_REPO_PATH=${GIT_REPO_PATH:-~/USERSCRATCH/vortex} +export CONTAINER_IMAGE=${CONTAINER_IMAGE:-vortex.sif} +export DEVNULL_BIND=${DEVNULL_BIND:-~/USERSCRATCH/devnull} +export VORTEX_TOOLCHAIN_PATH=${VORTEX_TOOLCHAIN_PATH:-~/USERSCRATCH/tools} + +# Launch the Apptainer container with the bind mount +apptainer shell --fakeroot --cleanenv --writable-tmpfs \ + --bind /dev/bus/usb,/sys/bus/pci \ + --bind /projects:/projects \ + --bind /lib/firmware:/lib/firmware \ + --bind /opt/xilinx/:/opt/xilinx/ \ + --bind /tools:/tools \ + --bind /netscratch:/netscratch \ + --bind "$VORTEX_TOOLCHAIN_PATH":/home/tools \ + --bind "$GIT_REPO_PATH":/home/vortex \ + "$CONTAINER_IMAGE" \ No newline at end of file diff --git a/miscs/apptainer/vortex_fpga.def b/miscs/apptainer/vortex.def similarity index 70% rename from miscs/apptainer/vortex_fpga.def rename to miscs/apptainer/vortex.def index d35989690..29380fecb 100644 --- a/miscs/apptainer/vortex_fpga.def +++ b/miscs/apptainer/vortex.def @@ -1,7 +1,6 @@ Bootstrap: docker From: ubuntu:22.04 - %files install_boost_openssl.sh . openssl-package.deb . @@ -10,21 +9,19 @@ From: ubuntu:22.04 %post echo "Setting up the environment..." - touch /var/log/apt/term.log if [ -f /var/log/apt/term.log ]; then chown root:adm /var/log/apt/term.log || true fi - # Set environment variable to avoid interactive prompts during installation export DEBIAN_FRONTEND=noninteractive echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/no-sandbox mkdir -p /netscratch - # Update and install base dependencies + # Update and install base dependencies (including wget here) apt-get update - apt-get install -y libc6 fakeroot + apt-get install -y libc6 fakeroot wget git vim # Fix dependency issues apt-get install -f -y || true @@ -38,25 +35,23 @@ From: ubuntu:22.04 rm -f /var/lib/dpkg/lock rm -f /var/cache/apt/archives/lock - # mount -o remount,rw /usr - # mount -o remount,rw /var + # Adjust permissions for dbus helper chown -R root:root /usr/lib/dbus-1.0/dbus-daemon-launch-helper || true - # Install necessary packages apt-get install -y fontconfig libfontconfig1 libpangoft2-1.0-0 libbluray2 \ libpangocairo-1.0-0 libavformat58 libpango-1.0-0 \ openjdk-11-jre-headless libcairo2 librsvg2-2 librsvg2-common \ openjdk-11-jre-zero libtheora0 libavcodec58 libcairo-gobject2 \ ca-certificates-java libchromaprint1 software-properties-common perl-modules bzip2 \ - unzip zlib1g-dev libtinfo5 g++ usbutils gawk bison gcc make wget tar python3.9 locales zstd uuid-dev git || true + unzip zlib1g-dev libtinfo5 g++ usbutils gawk bison gcc make tar python3.9 locales zstd uuid-dev ccache || true ln -s /usr/bin/python3 /usr/bin/python locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 - # Run dpkg to ensure everything is properly configured + # Run dpkg to ensure configuration is complete dpkg --configure -a || true # Clean up apt cache @@ -65,10 +60,19 @@ From: ubuntu:22.04 echo "Setting up the environment..." - # Copy the installation script into the container + # Install CMake 3.26 + echo "Installing CMake 3.26..." + wget https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0-linux-x86_64.sh -O /tmp/cmake.sh + chmod +x /tmp/cmake.sh + mkdir -p /opt/cmake + /tmp/cmake.sh --skip-license --prefix=/opt/cmake + ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake + rm -f /tmp/cmake.sh + + # Copy the boost and openssl installation script into the container cp install_boost_openssl.sh /tmp/ - # Execute the script + # Execute the installation script chmod +x /tmp/install_boost_openssl.sh /tmp/install_boost_openssl.sh @@ -79,19 +83,18 @@ From: ubuntu:22.04 export LANG=en_US.UTF-8 export LANGUAGE=en_US.UTF-8 export LC_ALL=en_US.UTF-8 - export HOME=/home/developer - - + export HOME=/home + export LD_LIBRARY_PATH=/opt/boost-1.66/lib:/opt/openssl-1.1/lib:$LD_LIBRARY_PATH %runscript exec /bin/bash %labels - Author Rahul - Version 1.0 + Author Rahul, Udit + Version 2.0 %test - echo "Container test successful!" locale lsusb || echo "lsusb not available" ls -l /dev/null + echo "Container test successful!" \ No newline at end of file