Merge 91cc057f5e into 594ea976c9

apt-requirements.txt

@@ -0,0 +1,54 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# List of packages installed with apt on our reference Ubuntu platform.
+#
+# When updating this list, please keep the yum package requirements for
+# RHEL/CentOS 7 in sync. These were derived from the Ubuntu requirements
+# and are maintained in yum-requirements.txt.
+#
+# This list of packages is also included in the documentation at
+# doc/ug/install_instructions/index.md. When updating this file also check if
+# doc/ug/install_instructions/index.md needs to be updated as well.
+#
+# Keep it sorted.
+autoconf
+bison
+build-essential
+clang-format
+cmake
+curl
+device-tree-compiler
+doxygen
+flex
+g++
+git
+golang
+lcov
+libelf1
+libelf-dev
+libftdi1-2
+libftdi1-dev
+# A requirement of the prebuilt clang toolchain.
+libncursesw5
+libssl-dev
+libudev-dev
+libusb-1.0-0
+lld
+lsb-release
+make
+ninja-build
+perl
+pkgconf
+python3
+python3-pip
+python3-setuptools
+python3-urllib3
+python3-wheel
+shellcheck
+srecord
+tree
+xsltproc
+zlib1g-dev
+xz-utils

util/container/Dockerfile

@@ -0,0 +1,153 @@
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Docker container containing various hardware and software development tools
+# for Ibex.
+
+# Global configuration options.
+ARG VERILATOR_VERSION=4.210
+# The RISCV toolchain version should match the release tag used in GitHub.
+ARG RISCV_TOOLCHAIN_TAR_VERSION=20220210-1
+# This should match the version in ci/install-package-dependencies.sh
+ARG GCC_VERSION=9
+
+# Main container image.
+FROM ubuntu:20.04 AS ibex
+ARG VERILATOR_VERSION
+ARG RISCV_TOOLCHAIN_TAR_VERSION
+ARG GCC_VERSION
+
+LABEL version="0.1"
+LABEL description="Ibex development container."
+LABEL maintainer="ibex@lowrisc.org"
+
+# Use bash as default shell.
+RUN ln -sf /bin/bash /bin/sh
+
+# Add OBS repository to apt sources.
+RUN OBS_URL="https://download.opensuse.org/repositories"; \
+    OBS_PATH="/home:/phiwag:/edatools/xUbuntu_20.04"; \
+    REPO_URL="${OBS_URL}${OBS_PATH}"; \
+    \
+    EDATOOLS_REPO_KEY="${REPO_URL}/Release.key"; \
+    EDATOOLS_REPO="deb ${REPO_URL}/ /"; \
+    \
+    apt-get update && \
+    apt-get install -y curl && \
+    \
+    curl -f -sL -o "$TMPDIR/obs.asc" "$EDATOOLS_REPO_KEY" || { \
+        error "Failed to download repository key from ${REPO_URL}"; \
+    } && \
+    echo "$EDATOOLS_REPO" > "$TMPDIR/obs.list" && \
+    mv "$TMPDIR/obs.asc"  /etc/apt/trusted.gpg.d/obs.asc && \
+    mv "$TMPDIR/obs.list" /etc/apt/sources.list.d/edatools.list
+
+# Install system packages
+#
+# Install (and cleanup) required packages (from apt-requirements.txt).
+# Also add some additional packages for the use within this container and for
+# developer convenience:
+# - gosu and sudo are used by the scripting to make the image more convenient
+#   to use.
+# - locales and locales-all are required to set the locale.
+# - minicom and screen are useful to see UART communication.
+# - dc and time are requirements of Synopsys VCS.
+# - csh and ksh are requirements of Cadence Xcelium.
+# - software-properties-common is required to be able to install newer gcc versions.
+
+# Necessary to avoid user interaction with tzdata during install
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=UTC
+
+COPY apt-requirements.txt /tmp/apt-requirements.txt
+RUN echo "verilator-${VERILATOR_VERSION}" >>/tmp/apt-requirements.txt \
+    && sed -i -e '/^$/d' -e '/^#/d' -e 's/#.*//' /tmp/apt-requirements.txt \
+    && apt-get update \
+    && xargs apt-get install -y </tmp/apt-requirements.txt \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \
+        sudo \
+        gosu \
+        locales \
+        locales-all \
+        minicom \
+        screen \
+        dc \
+        time \
+        software-properties-common \
+        environment-modules \
+        tclsh \
+        csh \
+        ksh \
+    && apt-get clean; \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc/*
+
+# Install the CI version of gcc and g++
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y gcc-${GCC_VERSION} g++-${GCC_VERSION} \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 90 \
+    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 90
+
+# RISC-V device toolchain
+COPY util/get-toolchain.py /tmp/get-toolchain.py
+RUN /tmp/get-toolchain.py -r ${RISCV_TOOLCHAIN_TAR_VERSION} \
+    && rm -f /tmp/get-toolchain.py
+
+# Set Locale to utf-8 everywhere
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+
+# Scripting for use within this container.
+COPY util/container/start.sh /start.sh
+COPY util/container/sudoconf /etc/sudoers.d/dev
+
+# Add the development user (UID/GID to be replaced).
+RUN groupadd dev \
+    && useradd --create-home -g dev dev \
+    && usermod -p '*' dev \
+    && passwd -u dev
+
+# All subsequent steps are performed as user.
+USER dev:dev
+
+# Install Python plus packages.
+#
+# Explicitly updating pip and setuptools is required to have these tools
+# properly parse Python-version metadata, which some packages uses to
+# specify that an older version of a package must be used for a certain
+# Python version. If that information is not read, pip installs the latest
+# version, which then fails to run.
+ENV PATH "/home/dev/.local/bin:${PATH}"
+COPY --chown=dev:dev python-requirements.txt /tmp/python-requirements.txt
+RUN sudo mkdir -p vendor/google_riscv-dv
+COPY --chown=dev:dev vendor/google_riscv-dv/requirements.txt \
+                     /tmp/vendor/google_riscv-dv/requirements.txt
+
+RUN python3 -m pip install --user -U pip setuptools \
+    && python3 -m pip install --user -r /tmp/python-requirements.txt \
+        --no-warn-script-location \
+    && rm -f /tmp/python-requirements.txt \
+    && rm -rf /tmp/vendor
+
+# Set RISC-V-specific environment variables.
+ENV RISCV /tools/riscv
+ENV RISCV_TOOLCHAIN "${RISCV}"
+ENV RISCV_GCC "${RISCV}/bin/riscv32-unknown-elf-gcc"
+ENV RISCV_OBJCOPY "${RISCV}/bin/riscv32-unknown-elf-objcopy"
+
+# Build and install Spike.
+RUN mkdir /tmp/spike && cd /tmp/spike \
+    && git clone https://github.com/lowRISC/riscv-isa-sim && cd riscv-isa-sim \
+    && git checkout 15fbd5680e44da699f828c67db15345822a47ef6 \
+    && mkdir build && cd build \
+    && ../configure --prefix=$RISCV --enable-commitlog --enable-misaligned \
+    && make -j16 \
+    && sudo make install
+ENV SPIKE_PATH "${RISCV}/bin"
+ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${RISCV}/lib/pkgconfig"
+
+USER root:root
+
+ENTRYPOINT [ "/start.sh" ]

util/container/start.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Map the "dev" user to an UID passed in as environment variable to ensure
+# files are written by the same UID/GID into mounted volumes.
+DEV_UID=${DEV_UID:-1000}
+DEV_GID=${DEV_GID:-1000}
+groupmod -o -g "$DEV_GID" dev >/dev/null 2>&1
+usermod -o -u "$DEV_UID" dev >/dev/null 2>&1
+
+# Load user configuration.
+test -f "${USER_CONFIG}" && export BASH_ENV=${USER_CONFIG}
+
+cd /home/dev || exit
+exec gosu dev:dev /bin/bash -c "$@"

util/container/sudoconf

@@ -0,0 +1,2 @@
+# Give dev user account root permissions in container
+dev ALL=(ALL) NOPASSWD:ALL

util/get-toolchain.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+# Copyright lowRISC contributors.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import json
+import logging as log
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from urllib.request import urlopen, urlretrieve
+
+log.basicConfig(level=log.INFO, format="%(levelname)s: %(message)s")
+
+# the keys in this dictionary specify valid toolchain kinds
+ASSET_PREFIXES = {
+    # kind : prefix,
+    "combined": "lowrisc-toolchain-rv32imcb-",
+    "gcc-only": "lowrisc-toolchain-gcc-rv32imcb-",
+}
+ASSET_SUFFIX = ".tar.xz"
+RELEASES_URL_BASE = 'https://api.github.com/repos/lowRISC/lowrisc-toolchains/releases'
+
+INSTALL_DIR = '/tools/riscv'
+TOOLCHAIN_VERSION = 'latest'
+TOOLCHAIN_KIND = 'combined'
+
+FILE_PATTERNS_TO_REWRITE = [
+    "riscv32-unknown-elf-*.cmake",
+    "meson-riscv32-unknown-elf-*.txt",
+]
+
+
+def get_available_toolchain_info(version, kind):
+    assert kind in ASSET_PREFIXES
+
+    if version == 'latest':
+        releases_url = '%s/%s' % (RELEASES_URL_BASE, version)
+    else:
+        releases_url = '%s/tags/%s' % (RELEASES_URL_BASE, version)
+
+    with urlopen(releases_url) as f:
+        release_info = json.loads(f.read().decode('utf-8'))
+
+    for asset in release_info["assets"]:
+        if (asset["name"].startswith(ASSET_PREFIXES[kind]) and
+                asset["name"].endswith(ASSET_SUFFIX)):
+            return {
+                'download_url': asset['browser_download_url'],
+                'name': asset['name'],
+                'version': release_info['tag_name'],
+                'kind': kind
+            }
+
+    # No matching asset found for the toolchain kind requested
+    log.error("No available downloads found for %s toolchain version: %s",
+              kind, release_info['tag_name'])
+    raise SystemExit(1)
+
+
+def get_installed_toolchain_info(unpack_dir):
+
+    # Try new-style buildinfo.json first
+    try:
+        buildinfo = {}
+        with open(str(unpack_dir / 'buildinfo.json'), 'r') as f:
+            buildinfo = json.loads(f.read())
+
+        # Toolchains before 20200602-4 contained a `buildinfo.json` without a
+        # 'kind' field. Setting it to 'unknown' will ensure we never skip
+        # updating because we think it's the same as the existing toolchain.
+        if 'kind' not in buildinfo:
+            buildinfo['kind'] = 'unknown'
+
+        return buildinfo
+    except Exception as e:
+        # buildinfo.json might not exist in older builds
+        log.info("Unable to parse buildinfo.json: %s", str(e))
+        pass
+
+    # If that wasn't successful, try old-style plaintext buildinfo
+    version_re = r"(lowRISC toolchain version|Version):\s*\n?(?P<version>[^\n\s]+)"
+    buildinfo_txt_path = unpack_dir / 'buildinfo'
+    try:
+        with open(str(buildinfo_txt_path), 'r') as f:
+            match = re.match(version_re, f.read(), re.M)
+        if not match:
+            log.warning("Unable extract version from %s",
+                        str(buildinfo_txt_path))
+            return None
+        return {'version': match.group("version"), 'kind': 'unknown'}
+    except Exception as e:
+        log.error("Unable to read %s: %s", str(buildinfo_txt_path), str(e))
+        return None
+
+
+def download(url):
+    log.info("Downloading toolchain from %s", url)
+    tmpfile = tempfile.mkstemp()[1]
+    urlretrieve(url, tmpfile)
+    log.info("Download complete")
+    return Path(tmpfile)
+
+
+def install(archive_file, unpack_dir):
+    unpack_dir.mkdir(parents=True, exist_ok=True)
+
+    cmd = [
+        'tar',
+        '-x',
+        '-f',
+        str(archive_file),
+        '--strip-components=1',
+        '-C',
+        str(unpack_dir),
+    ]
+    subprocess.run(cmd, check=True)
+
+
+def postinstall_rewrite_configs(unpack_dir, install_dir):
+    """Rewrites the toolchain configuration files to point to install_dir.
+
+    'unpack_dir' is where the toolchain is unpacked by this script.
+    'install_dir' is where the toolchain is eventually invoked from. Typically,
+    these are the same, unless a staged installation is being performed by
+    supplying both, --install-dir and --dest-dir switches. Regardless, if the
+    'install_dir' is different from the default, the config files need to be
+    updated to reflect the correct paths.
+    """
+    if str(install_dir) == INSTALL_DIR:
+        return
+
+    for file_pattern in FILE_PATTERNS_TO_REWRITE:
+        for config_file_path in unpack_dir.glob(file_pattern):
+            # Rewrite INSTALL_DIR to the requested target dir.
+            log.info("Updating toolchain paths in %s", str(config_file_path))
+            with open(str(config_file_path)) as f:
+                original = f.read()
+            with open(str(config_file_path), "w") as f:
+                f.write(original.replace(INSTALL_DIR, str(install_dir)))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--install-dir',
+                        '-i',
+                        required=False,
+                        default=INSTALL_DIR,
+                        help="Installation directory (default: %(default)s)")
+    parser.add_argument('--dest-dir',
+                        '-d',
+                        required=False,
+                        help="""Destination directory if performing a staged
+                        installation. This is the staging directory where the
+                        toolchain is unpacked.""")
+    parser.add_argument('--release-version',
+                        '-r',
+                        required=False,
+                        default=TOOLCHAIN_VERSION,
+                        help="Toolchain version (default: %(default)s)")
+    parser.add_argument('--latest-available-version',
+                        '-l',
+                        required=False,
+                        default=False,
+                        action='store_true',
+                        help="Return the latest available toolchain version.")
+    parser.add_argument('--kind',
+                        required=False,
+                        default=TOOLCHAIN_KIND,
+                        choices=ASSET_PREFIXES.keys(),
+                        help="Toolchain kind (default: %(default)s)")
+    parser.add_argument(
+        '--update',
+        '-u',
+        required=False,
+        default=False,
+        action='store_true',
+        help="Update to target version if needed (default: %(default)s)")
+    args = parser.parse_args()
+
+    available_toolchain = get_available_toolchain_info(args.release_version,
+                                                       args.kind)
+
+    if args.latest_available_version:
+        print(available_toolchain['version'])
+        sys.exit(0)
+
+    log.info("Found available %s toolchain version %s, %s",
+             available_toolchain['kind'], available_toolchain['version'],
+             available_toolchain['name'])
+
+    install_dir = Path(args.install_dir)
+    if args.dest_dir is None:
+        unpack_dir = install_dir
+    else:
+        unpack_dir = Path(args.dest_dir)
+
+    if args.update and unpack_dir.is_dir():
+        installed_toolchain = get_installed_toolchain_info(unpack_dir)
+        if installed_toolchain is None:
+            sys.exit('Unable to extract current toolchain version. '
+                     'Delete target directory %s and try again.' %
+                     str(unpack_dir))
+
+        version_matches = available_toolchain[
+            'version'] == installed_toolchain['version']
+        kind_matches = available_toolchain['kind'] == installed_toolchain[
+            'kind']
+
+        if installed_toolchain[
+                'kind'] != 'unknown' and version_matches and kind_matches:
+            log.info(
+                'Downloaded %s toolchain is version %s, '
+                'same as the %s toolchain installed at %s (version %s).',
+                available_toolchain['kind'], available_toolchain['version'],
+                installed_toolchain['kind'], installed_toolchain['version'],
+                str(unpack_dir))
+            log.warning("Skipping install.")
+            sys.exit(0)
+
+        log.info(
+            "Found installed %s toolchain version %s, updating to %s toolchain "
+            "version %s.",
+            installed_toolchain['kind'], installed_toolchain['version'],
+            available_toolchain['kind'], available_toolchain['version'])
+    else:
+        if unpack_dir.exists():
+            sys.exit('Target directory %s already exists. '
+                     'Delete it first, or use --update.' % str(unpack_dir))
+
+    archive_file = None
+    try:
+        archive_file = download(available_toolchain['download_url'])
+
+        if args.update and unpack_dir.exists():
+            # We only reach this point if |unpack_dir| contained a toolchain
+            # before, so removing it is reasonably safe.
+            shutil.rmtree(str(unpack_dir))
+
+        install(archive_file, unpack_dir)
+        postinstall_rewrite_configs(unpack_dir.resolve(),
+                                    install_dir.resolve())
+    finally:
+        if archive_file:
+            archive_file.unlink()
+
+    log.info('Installed %s toolchain version %s to %s.',
+             available_toolchain['kind'], available_toolchain['version'],
+             str(unpack_dir))
+
+
+if __name__ == "__main__":
+    main()