CI: Bump Ubuntu 18 jobs to Ubuntu 20 (ldc-developers#4367)

(cherry picked from commit efe822e)

.circleci/config.yml

@@ -100,14 +100,14 @@ commonSteps: &commonSteps
 
 version: 2
 jobs:
-  Ubuntu-18.04-multilib-rtSanitizers:
+  Ubuntu-20.04-multilib-rtSanitizers:
     <<: *commonSteps
     docker:
-      - image: ubuntu:18.04
+      - image: ubuntu:20.04
     environment:
       - PARALLELISM: 2
       - CI_OS: linux
-      - EXTRA_APT_PACKAGES: llvm-9-dev libclang-common-9-dev
+      - EXTRA_APT_PACKAGES: llvm-dev libclang-common-10-dev
       - HOST_LDC_VERSION: 1.24.0
       - EXTRA_CMAKE_FLAGS: "-DMULTILIB=ON -DRT_SUPPORT_SANITIZERS=ON -DBUILD_LTO_LIBS=ON"
   Ubuntu-20.04-sharedLibsOnly-gdmd:
@@ -147,7 +147,7 @@ workflows:
   version: 2
   build:
     jobs:
-      - Ubuntu-18.04-multilib-rtSanitizers
+      - Ubuntu-20.04-multilib-rtSanitizers
       - Ubuntu-20.04-sharedLibsOnly-gdmd
       - macOS-x64
       - macOS-x64-sharedLibsOnly

.cirrus.yml

@@ -307,3 +307,267 @@ task:
     elif [[ "$CIRRUS_PR" = "" && "$CIRRUS_BRANCH" = "master" ]]; then
       tools/upload-to-github.sh CI ../artifacts/ldc2-*.tar.xz
     fi
+
+# Installs Ubuntu 20.04+ prerequisites.
+# Requires env variables CI_ARCH, HOST_LDC_VERSION, EXTRA_APT_PACKAGES and EXTRA_CMAKE_FLAGS.
+install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
+  install_prerequisites_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    nproc
+    export DEBIAN_FRONTEND=noninteractive
+    if [[ "$EXTRA_CMAKE_FLAGS" = *-DMULTILIB?ON* ]]; then
+      dpkg --add-architecture i386
+      gcc_pkg="g++-multilib"
+      libcurl_pkg="libcurl4 libcurl4:i386"
+    else
+      gcc_pkg="g++"
+      libcurl_pkg="libcurl4"
+    fi
+    apt-get -q update
+    apt-get -yq install \
+      git-core cmake ninja-build $gcc_pkg \
+      zlib1g-dev $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
+      $EXTRA_APT_PACKAGES
+    python3 --version
+    # Download & extract host LDC
+    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-$CI_ARCH.tar.xz
+    mkdir host-ldc
+    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
+    rm ldc2.tar.xz
+
+# Installs macOS prerequisites.
+# Requires env variables CI_ARCH, HOST_LDC_VERSION and LLVM_VERSION.
+install_macos_prerequisites_template: &INSTALL_MACOS_PREREQUISITES_TEMPLATE
+  install_prerequisites_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    sysctl -n hw.logicalcpu
+    # Download & extract CMake
+    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-macos-universal.tar.gz
+    mkdir cmake
+    tar -xf cmake.tar.gz --strip 3 -C cmake
+    rm cmake.tar.gz
+    # Download & extract Ninja
+    curl -fL --retry 3 --max-time 60 -O https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
+    mkdir ninja
+    tar -xf ninja-mac.zip -C ninja
+    rm ninja-mac.zip
+    # Download & extract LDC-flavoured LLVM
+    echo 'Using LLVM with enabled assertions'
+    assertsSuffix="-withAsserts"
+    llvm_tag="CI"
+    if [[ "$LLVM_VERSION" == *.* ]]; then llvm_tag="ldc-v$LLVM_VERSION"; fi
+    curl -fL --retry 3 --max-time 300 -o llvm.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/$llvm_tag/llvm-$LLVM_VERSION-osx-${CI_ARCH}${assertsSuffix}.tar.xz
+    mkdir llvm
+    tar -xf llvm.tar.xz --strip 1 -C llvm
+    rm llvm.tar.xz
+    # Download & extract host LDC
+    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-osx-$CI_ARCH.tar.xz
+    mkdir host-ldc
+    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
+    rm ldc2.tar.xz
+    # Install Python3
+    brew install python
+    python3 --version
+
+environment:
+  CIRRUS_CLONE_DEPTH: 50
+  HOST_LDC_VERSION: 1.31.0
+  LLVM_VERSION: 15.0.7
+  GITHUB_TOKEN: ENCRYPTED[0955bd48c8d4e5391446fc0149d0719ad0b63df27ec9e6c180a5730a5b10dc7f28f09d1383423db158d21380ee2b022a]
+
+task:
+  name: Ubuntu 20.04 x64 multilib rtSanitizers
+  container:
+    image: ubuntu:20.04
+    cpu: 8
+    memory: 16G
+  timeout_in: 20m
+  environment:
+    CI_ARCH: x86_64
+    CI_OS: linux
+    EXTRA_APT_PACKAGES: "llvm-dev libclang-common-10-dev"
+    EXTRA_CMAKE_FLAGS: "-DMULTILIB=ON -DRT_SUPPORT_SANITIZERS=ON -DBUILD_LTO_LIBS=ON"
+    PARALLELISM: 8
+  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
+  << : *COMMON_STEPS_TEMPLATE
+
+task:
+  name: Ubuntu rolling x64 shared-libs-only gdmd
+  # allow failures - gdb v10 came with regressions
+  allow_failures: true
+  container:
+    image: ubuntu:rolling
+    cpu: 8
+    memory: 16G
+  timeout_in: 15m
+  environment:
+    CI_ARCH: x86_64
+    CI_OS: linux
+    EXTRA_APT_PACKAGES: "gdmd llvm-dev libclang-common-15-dev"
+    EXTRA_CMAKE_FLAGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_LTO_LIBS=ON -DD_COMPILER=gdmd -DLDC_LINK_MANUALLY=ON"
+    PARALLELISM: 8
+    # for gdmd:
+    LANG: C.UTF-8
+  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
+  << : *COMMON_STEPS_TEMPLATE
+
+task:
+  name: Ubuntu 20.04 x64 bootstrap
+  container:
+    image: ubuntu:20.04
+    cpu: 8
+    memory: 16G
+  timeout_in: 15m
+  environment:
+    CI_ARCH: x86_64
+    CI_OS: linux
+    HOST_LDC_VERSION: 1.9.0
+    EXTRA_APT_PACKAGES: "llvm-dev libclang-common-10-dev"
+    EXTRA_CMAKE_FLAGS: "-DBUILD_LTO_LIBS=ON"
+    PARALLELISM: 8
+  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
+  << : *COMMON_STEPS_TEMPLATE
+
+task:
+  name: macOS 12 $TASK_NAME_SUFFIX
+  allow_failures: true # FIXME
+  macos_instance:
+    image: ghcr.io/cirruslabs/macos-monterey-xcode:latest
+  timeout_in: 25m
+  environment:
+    CI_OS: osx
+    # OS is preset to `darwin`
+    OS: osx
+    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${CIRRUS_WORKING_DIR}/../llvm/bin:${PATH}
+    PARALLELISM: 4
+    matrix:
+      #- TASK_NAME_SUFFIX: x64 (M1)
+      #  EXTRA_CMAKE_FLAGS: -DCMAKE_OSX_ARCHITECTURES=x86_64 -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON
+      #  CI_ARCH: x86_64
+      #- TASK_NAME_SUFFIX: x64 (M1) shared-libs-only
+      #  EXTRA_CMAKE_FLAGS: -DCMAKE_OSX_ARCHITECTURES=x86_64 -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=ON
+      #  CI_ARCH: x86_64
+      - TASK_NAME_SUFFIX: arm64
+        EXTRA_CMAKE_FLAGS: -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON
+        CI_ARCH: arm64
+      - TASK_NAME_SUFFIX: arm64 shared-libs-only
+        EXTRA_CMAKE_FLAGS: -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=ON
+        CI_ARCH: arm64
+  brew_cache:
+    folder: "$HOME/Library/Caches/Homebrew"
+  << : *INSTALL_MACOS_PREREQUISITES_TEMPLATE
+  << : *COMMON_STEPS_TEMPLATE
+
+task:
+  name: Ubuntu 20.04 aarch64
+  arm_container:
+    image: ubuntu:20.04
+    cpu: 4
+    memory: 8G
+  timeout_in: 60m
+  environment:
+    CI_ARCH: aarch64
+    CI_OS: linux
+    EXTRA_APT_PACKAGES: libxml2-dev p7zip-full
+    EXTRA_CMAKE_FLAGS: >-
+      -DBUILD_LTO_LIBS=ON
+      -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
+      -DEXTRA_CXXFLAGS=-flto=full
+      -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++
+      -DJITRT_EXTRA_LDFLAGS=-static-libstdc++
+      -DLDC_INSTALL_LLVM_RUNTIME_LIBS_OS=aarch64-unknown-linux-gnu
+      -DLLVM_ROOT_DIR=$CIRRUS_WORKING_DIR/../llvm
+      -DD_COMPILER=$CIRRUS_WORKING_DIR/../bootstrap-ldc/bin/ldmd2
+    PARALLELISM: 4
+    CLANG_VERSION: '15.0.3' # 15.0.6 requires a more recent libstdc++.so.6 than shipped with Ubuntu 20
+    CC: $CIRRUS_WORKING_DIR/../clang/bin/clang
+    CXX: $CIRRUS_WORKING_DIR/../clang/bin/clang++
+  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
+  # Download & extract LDC-flavoured LLVM
+  download_prebuilt_llvm_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    assertsSuffix=""
+    if [[ "${CIRRUS_TAG:-}" == "" ]]; then
+      echo 'Using LLVM with enabled assertions'
+      assertsSuffix="-withAsserts"
+    fi
+    llvm_tag="CI"
+    if [[ "$LLVM_VERSION" = *.* ]]; then llvm_tag="ldc-v$LLVM_VERSION"; fi
+    llvm_url="https://github.com/ldc-developers/llvm-project/releases/download/$llvm_tag/llvm-$LLVM_VERSION-linux-$CI_ARCH$assertsSuffix.tar.xz"
+    curl -fL --retry 3 --max-time 300 -o llvm.tar.xz $llvm_url
+    mkdir -p llvm
+    tar -xf llvm.tar.xz --strip 1 -C llvm
+    rm llvm.tar.xz
+    # Make lld the default linker (likely with enabled assertions unfortunately)
+    ln -sf "$PWD/llvm/bin/ld.lld" /usr/bin/ld
+    ld --version
+  # Download & extract clang
+  download_prebuilt_clang_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    curl -fL --retry 3 --max-time 300 -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-$CI_ARCH-linux-gnu.tar.xz
+    mkdir clang
+    tar -xf clang.tar.xz --strip 1 -C clang
+    rm clang.tar.xz
+  clone_submodules_early_script: |
+    cd $CIRRUS_WORKING_DIR
+    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
+  # Build a first LDC as host compiler for the actual build
+  build_bootstrap_ldc_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    mkdir bootstrap-ldc
+    cd bootstrap-ldc
+    cmake -G Ninja $CIRRUS_WORKING_DIR \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ROOT_DIR=$PWD/../llvm \
+      -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
+      -DBUILD_SHARED_LIBS=OFF \
+      -DBUILD_LTO_LIBS=ON
+    ninja -j$PARALLELISM
+    bin/ldc2 -version
+  << : *COMMON_STEPS_TEMPLATE
+  << : *PACKAGING_STEPS_TEMPLATE
+
+task:
+  name: FreeBSD 12.2 x64
+  freebsd_instance:
+    image_family: freebsd-12-2
+    cpu: 4
+    memory: 8G
+  timeout_in: 60m
+  environment:
+    CI_ARCH: x86_64
+    CI_OS: freebsd
+    EXTRA_CMAKE_FLAGS: >-
+      -DBUILD_LTO_LIBS=ON
+      -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
+      -DEXTRA_CXXFLAGS=-flto=full
+    PARALLELISM: 4
+    # use clang from the `llvm` ports package to be installed, incl. lld support for .deplibs (`pragma(lib, "execinfo")`)
+    CC: clang90
+  install_prerequisites_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    sysctl -n hw.ncpu
+    pkg install -y git cmake ninja gmake llvm bash gtar 7-zip
+    python3 --version
+    python3 -m ensurepip
+    # Download & extract host LDC
+    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-freebsd-$CI_ARCH.tar.xz
+    mkdir pre-ldc
+    gtar -xf ldc2.tar.xz --strip 1 -C pre-ldc
+  clone_submodules_early_script: |
+    cd $CIRRUS_WORKING_DIR
+    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
+  # Build a first LDC as host compiler for the actual build
+  build_bootstrap_ldc_script: |
+    cd $CIRRUS_WORKING_DIR/..
+    mkdir host-ldc
+    cd host-ldc
+    cmake -G Ninja $CIRRUS_WORKING_DIR \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DD_COMPILER=$PWD/../pre-ldc/bin/ldmd2 \
+      -DBUILD_SHARED_LIBS=OFF \
+      -DBUILD_LTO_LIBS=ON
+    ninja -j$PARALLELISM
+    bin/ldc2 -version
+  << : *COMMON_STEPS_TEMPLATE
+  << : *PACKAGING_STEPS_TEMPLATE

.github/actions/1-setup/action.yml

@@ -18,11 +18,11 @@ runs:
         export DEBIAN_FRONTEND=noninteractive
         sudo dpkg --add-architecture i386
         sudo apt-get -q update
-        # Don't use latest gdb v10 from Ubuntu toolchain PPA with regressions, use official v8
+        # Don't use latest gdb v10+ from Ubuntu toolchain PPA with regressions, use official v9
         sudo apt-get -yq install \
           git-core cmake g++-multilib \
           libcurl4 libcurl4:i386 \
-          curl gdb=8.1.1-0ubuntu1 p7zip-full tzdata unzip zip python3-pip
+          curl gdb=9.1-0ubuntu1 p7zip-full tzdata unzip zip python3-pip
 
     - name: 'Linux: Download & extract clang' # into ../clang
       if: runner.os == 'Linux'

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
         include:
 
           - job_name: Linux x86_64
-            os: ubuntu-18.04
+            os: ubuntu-20.04
             arch: x86_64
             bootstrap_cmake_flags: -DBUILD_LTO_LIBS=ON
             # To improve portability of the generated binaries, link the C++ standard library statically.

.github/workflows/supported_llvm_versions.yml

@@ -15,24 +15,24 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - job_name: Ubuntu 18.04, LLVM 14, latest LDC beta
-            os: ubuntu-18.04
+          - job_name: Ubuntu 20.04, LLVM 14, latest LDC beta
+            os: ubuntu-20.04
             host_dc: ldc-beta
             llvm_version: 14.0.0
             # the compiler-rt libs installation is somehow non-standard
             cmake_flags: -DLDC_INSTALL_LLVM_RUNTIME_LIBS_OS=x86_64-unknown-linux-gnu -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=""
-          - job_name: Ubuntu 18.04, LLVM 12, latest LDC beta
-            os: ubuntu-18.04
+          - job_name: Ubuntu 20.04, LLVM 12, latest LDC beta
+            os: ubuntu-20.04
             host_dc: ldc-beta
             llvm_version: 12.0.1
             cmake_flags: -DLIB_SUFFIX=64
-          - job_name: Ubuntu 18.04, LLVM 11, latest LDC beta
-            os: ubuntu-18.04
+          - job_name: Ubuntu 20.04, LLVM 11, latest LDC beta
+            os: ubuntu-20.04
             host_dc: ldc-beta
             llvm_version: 11.1.0
             cmake_flags: -DBUILD_SHARED_LIBS=ON -DRT_SUPPORT_SANITIZERS=ON
-          - job_name: Ubuntu 18.04, LLVM 9, latest DMD beta
-            os: ubuntu-18.04
+          - job_name: Ubuntu 20.04, LLVM 9, latest DMD beta
+            os: ubuntu-20.04
             host_dc: dmd-beta
             llvm_version: 9.0.1
             cmake_flags: -DBUILD_SHARED_LIBS=OFF -DRT_SUPPORT_SANITIZERS=ON -DLDC_LINK_MANUALLY=ON
@@ -67,13 +67,13 @@ jobs:
           python3 -m pip install --user setuptools wheel
           python3 -m pip install --user lit
           python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
-      - name: 'Linux: Install gdb'
+      - name: 'Linux: Install gdb and llvm-symbolizer'
         if: runner.os == 'Linux'
         run: |
           set -eux
           sudo apt-get update
-          # Don't use latest gdb v10 from Ubuntu toolchain PPA with regressions, use official v8
-          sudo apt-get install gdb=8.1.1-0ubuntu1
+          # Don't use latest gdb v10+ from Ubuntu toolchain PPA with regressions, use official v9
+          sudo apt-get install gdb=9.1-0ubuntu1 llvm
 
       - name: Try to restore cached LLVM
         uses: actions/cache@v2
@@ -136,7 +136,14 @@ jobs:
           if [[ '${{ matrix.llvm_version }}' = 9.* ]]; then
             echo "config.available_features.remove('Fuzzer')" >> tests/sanitizers/lit.local.cfg
           fi
+          # LLVM 14+ on Linux: don't use vanilla llvm-symbolizer (no support for zlib-compressed debug sections => failing ASan tests)
+          if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.llvm_version }}' =~ ^1[4-9]\. ]]; then
+            mv llvm/bin/llvm-symbolizer llvm/bin/llvm-symbolizer.bak
+          fi
           ctest -V -R "lit-tests"
+          if [[ -f llvm/bin/llvm-symbolizer.bak ]]; then
+            mv llvm/bin/llvm-symbolizer.bak llvm/bin/llvm-symbolizer
+          fi
       - name: Run DMD testsuite
         if: success() || failure()
         run: ctest -V -R "dmd-testsuite"