Merge pull request #248 from rahul7rajdn/apptainer

Apptainer for Vortex

.gitignore

@@ -2,3 +2,4 @@
 /.vscode
 *.cache
 *.code-workspace
+miscs/apptainer/vortex.sif

docs/fpga_setup.md

@@ -63,6 +63,12 @@ $ source /opt/xilinx/xrt/setup.sh
 $ source /tools/reconfig/xilinx/Vitis/2023.1/settings64.sh
 ```
 
+If you are using an apptainer,
+```
+$ export LD_LIBRARY_PATH=/opt/boost-1.66/lib/:/opt/openssl-1.1/lib/:$LD_LIBRARY_PATH
+```
+
+
 ### Check Installed FPGA Platforms
 `platforminfo -l` which tells us the correct name of the platform installed on the current fpga node. It should be used for the `PLATFORM` variable below. Otherwise, if there is an error then there was an issue with the previous two commands.
 

miscs/apptainer/README.md

@@ -0,0 +1,115 @@
+# Apptainer Build Process
+
+Use the Slurm scheduler to request an interactive job on Flubber9
+```
+salloc -p rg-fpga --nodes=1 --ntasks-per-node=64 --mem=16G --nodelist flubber9 --time=01:00:00
+```
+
+Go to `apptainer` directory
+
+```
+$ pwd
+vortex/miscs/apptainer
+
+$  apptainer build --no-https vortex.sif vortex.def
+
+```
+
+
+To start the apptainer,
+```
+$ chmod +x run_apptainer.sh
+$ ./run_apptainer.sh
+```
+
+
+| Note: Set env variables in `run_apptainer.sh` accordingly.
+
+
+Inside the Apptainer,
+```
+# should show devices connected to machine on which you are running this command
+Apptainer> lsusb
+
+```
+
+
+# Vortex Simulation inside Apptainer
+
+Go to bind of vortex repo,
+```
+Apptainer> cd /home/vortex
+Apptainer> mkdir build
+Apptainer> cd build
+Apptainer> ../configure --xlen=32 --tooldir=$HOME/tools
+
+
+Skip the below 3 steps, If tools are already present in the $HOME/tools
+Apptainer> sed -i 's/\btar /tar --no-same-owner /g' ci/toolchain_install.sh
+Apptainer> ./ci/toolchain_install.sh --all
+Apptainer> sed -i 's/\btar --no-same-owner /tar /g' ci/toolchain_install.sh
+
+
+Apptainer> source ./ci/toolchain_env.sh
+Apptainer> verilator --version
+```
+
+
+### Running SIMX, RTLSIM and XRTSIM
+```
+Compile the Vortex codebase
+Apptainer> make -s
+
+Run the programs by specifying the appropriate driver as shown below:
+
+SIMX
+Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=simx
+
+RTLSIM
+Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=rtlsim
+
+XRTSIM
+Apptainer> ./ci/blackbox.sh --cores=2 --app=demo --driver=xrt
+```
+
+
+### Vortex Bitstream on FPGA
+
+Common Commands
+```
+Apptainer> source /opt/xilinx/xrt/setup.sh
+Apptainer> source /tools/reconfig/xilinx/Vitis/2023.1/settings64.sh
+
+Apptainer> platforminfo -l
+
+```
+
+##### Building Vortex Bitstream
+
+```
+Apptainer> pwd
+/home/vortex/build
+
+Apptainer> source ci/toolchain_env.sh
+
+Apptainer> verilator --version
+Verilator 5.026 2024-06-15 rev v5.026-43-g065f36ab5
+
+Apptainer> cd hw/syn/xilinx/xrt
+Apptainer> PREFIX=test1 PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 TARGET=hw NUM_CORES=1 make > build_u50_hw_1c.log 2>&1 &
+Creates ../test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin/vortex_afu.xclbin
+```
+
+##### Running Vortex Bitstream on FPGA
+```
+Apptainer> cd ../../../../
+Apptainer> pwd
+/home/vortex/build
+
+Apptainer> make -C runtime/ clean
+
+Apptainer> FPGA_BIN_DIR=hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin  TARGET=hw PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1  ./ci/blackbox.sh --driver=xrt --app=demo
+
+Verify following line being printed:
+info: device name=xilinx_u50_gen3x16_xdma_base_5, memory_capacity=0x200000000 bytes, memory_banks=32.
+```

miscs/apptainer/install_boost_openssl.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e  # Exit on error
+echo "Installing Boost and OpenSSL from .deb packages"
+
+# Extract and install Boost
+echo "Extracting Boost..."
+ar -x boost-package.deb
+ls
+mkdir -p /tmp/boost-extract
+tar --zstd --no-same-owner -xvf data.tar.zst -C /tmp/boost-extract
+ls -l /tmp/boost-extract/
+cp -r /tmp/boost-extract/opt/boost-1.66/ /opt/
+ls -l /opt/boost-1.66/
+rm -rf data.tar
+
+# Extract and install OpenSSL
+echo "Extracting OpenSSL..."
+ar -x openssl-package.deb
+mkdir -p /tmp/openssl-extract
+tar --zstd --no-same-owner -xvf data.tar.zst -C /tmp/openssl-extract
+ls -l /tmp/openssl-extract/
+cp -r /tmp/openssl-extract/opt/openssl-1.1/ /opt/
+ls -l /opt/openssl-1.1/
+rm -rf data.tar
+
+
+
+echo "Boost and OpenSSL installation complete!"

miscs/apptainer/run_apptainer.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Set environment variables with fallback default values.
+# The :- means if the variable CONTAINER_IMAGE is not set or is empty, then use the default value vortex.sif
+export GIT_REPO_PATH=${GIT_REPO_PATH:-~/USERSCRATCH/vortex}
+export CONTAINER_IMAGE=${CONTAINER_IMAGE:-vortex.sif}
+export DEVNULL_BIND=${DEVNULL_BIND:-~/USERSCRATCH/devnull}
+export VORTEX_TOOLCHAIN_PATH=${VORTEX_TOOLCHAIN_PATH:-~/USERSCRATCH/tools}
+
+# Launch the Apptainer container with the bind mount
+apptainer shell --fakeroot --cleanenv --writable-tmpfs \
+  --bind /dev/bus/usb,/sys/bus/pci \
+  --bind /projects:/projects \
+  --bind /lib/firmware:/lib/firmware \
+  --bind /opt/xilinx/:/opt/xilinx/ \
+  --bind /tools:/tools \
+  --bind /netscratch:/netscratch \
+  --bind "$VORTEX_TOOLCHAIN_PATH":/home/tools \
+  --bind "$GIT_REPO_PATH":/home/vortex \
+  "$CONTAINER_IMAGE"

miscs/apptainer/vortex.def

@@ -0,0 +1,100 @@
+Bootstrap: docker
+From: ubuntu:22.04
+
+%files
+    install_boost_openssl.sh .
+    openssl-package.deb .
+    boost-package.deb .
+
+%post
+    echo "Setting up the environment..."
+
+    touch /var/log/apt/term.log
+    if [ -f /var/log/apt/term.log ]; then
+        chown root:adm /var/log/apt/term.log || true
+    fi
+
+    # Set environment variable to avoid interactive prompts during installation
+    export DEBIAN_FRONTEND=noninteractive
+    echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/no-sandbox
+    mkdir -p /netscratch
+
+    # Update and install base dependencies (including wget here)
+    apt-get update
+    apt-get install -y libc6 fakeroot wget git vim
+
+    # Fix dependency issues
+    apt-get install -f -y || true
+    dpkg --configure -a || true
+
+    # Reinstall problematic packages
+    apt-get remove --purge -y dbus fontconfig-config || true
+    apt-get install -y dbus fontconfig-config || true
+
+    rm -f /var/lib/dpkg/lock-frontend
+    rm -f /var/lib/dpkg/lock
+    rm -f /var/cache/apt/archives/lock
+
+    # Adjust permissions for dbus helper
+    chown -R root:root /usr/lib/dbus-1.0/dbus-daemon-launch-helper || true
+
+    # Install necessary packages
+    apt-get install -y fontconfig libfontconfig1 libpangoft2-1.0-0 libbluray2 \
+                       libpangocairo-1.0-0 libavformat58 libpango-1.0-0 \
+                       openjdk-11-jre-headless libcairo2 librsvg2-2 librsvg2-common \
+                       openjdk-11-jre-zero libtheora0 libavcodec58 libcairo-gobject2 \
+                       ca-certificates-java libchromaprint1 software-properties-common perl-modules bzip2 \
+                       unzip zlib1g-dev libtinfo5 g++ usbutils gawk bison gcc make tar python3.9 locales zstd uuid-dev ccache || true
+
+    ln -s /usr/bin/python3 /usr/bin/python
+
+    locale-gen en_US.UTF-8
+    update-locale LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8
+
+    # Run dpkg to ensure configuration is complete
+    dpkg --configure -a || true
+
+    # Clean up apt cache
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+
+    echo "Setting up the environment..."
+
+    # Install CMake 3.26
+    echo "Installing CMake 3.26..."
+    wget https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0-linux-x86_64.sh -O /tmp/cmake.sh
+    chmod +x /tmp/cmake.sh
+    mkdir -p /opt/cmake
+    /tmp/cmake.sh --skip-license --prefix=/opt/cmake
+    ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake
+    rm -f /tmp/cmake.sh
+
+    # Copy the boost and openssl installation script into the container
+    cp install_boost_openssl.sh /tmp/
+
+    # Execute the installation script
+    chmod +x /tmp/install_boost_openssl.sh
+    /tmp/install_boost_openssl.sh
+
+    # Clean up
+    rm -f /tmp/install_boost_openssl.sh
+
+%environment
+    export LANG=en_US.UTF-8
+    export LANGUAGE=en_US.UTF-8
+    export LC_ALL=en_US.UTF-8
+    export HOME=/home
+    export LD_LIBRARY_PATH=/opt/boost-1.66/lib:/opt/openssl-1.1/lib:$LD_LIBRARY_PATH
+
+%runscript
+    exec /bin/bash
+
+%labels
+    Author Rahul, Udit
+    Version 2.0
+
+%test
+    locale
+    lsusb || echo "lsusb not available"
+    ls -l /dev/null
+    echo "Container test successful!"