wdl-ci.yml

name: WDL CI

on:
  push

env:
  LC_ALL: C.UTF-8
  LANG: C.UTF-8
  DEBIAN_FRONTEND: noninteractive

jobs:
  linters:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - uses: actions/cache@v1
        with:
          path: ~/.cache
          key: ${{runner.os}}-cache
      - name: dependencies
        run: |
          source /etc/profile
          sudo apt-get -qq update
          sudo apt-get -qq install --yes jq moreutils make virtualenv zip unzip httpie git shellcheck
          virtualenv --python=python3.8 .venv
          source .venv/bin/activate
          pip install -r requirements-dev.txt
      - name: linters
        run: |
          source .venv/bin/activate
          make lint
      - uses: andymckay/cancel-action@0.2
        if: ${{ failure() }}

  cancel-previous:
    runs-on: ubuntu-20.04
    steps:
      - uses: styfle/cancel-workflow-action@0.9.0
        with:
          access_token: ${{ github.token }}

  list-workflow-dirs:
    runs-on: ubuntu-20.04
    outputs:
      matrix: ${{steps.list_dirs.outputs.matrix}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - id: list_dirs
        # only run on wdl dirs with diff
        run: echo "::set-output name=matrix::$(ls */*.wdl|xargs dirname|xargs git diff --name-status HEAD^|cut -c 3-|xargs dirname|sed 's/\/.*$//g'|sort|uniq|jq -cnR '[inputs|select(length>0)]')"

  wdl-ci:
    runs-on: ubuntu-20.04
    needs: list-workflow-dirs
    strategy:
      matrix:
        workflow_dir: ${{fromJson(needs['list-workflow-dirs'].outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
      - name: docker login ghcr.io
        uses: docker/login-action@v1
        with:
            registry: ghcr.io
            username: ${{ github.repository_owner }}
            password: ${{ secrets.GITHUB_TOKEN }}
      - name: docker build + push to ghcr.io
        run: |
          TAG=$(git describe --long --tags --always --dirty)
          IMAGE_NAME=idseq-${{ matrix.workflow_dir }}-public
          IMAGE_URI="ghcr.io/${GITHUB_REPOSITORY}/${IMAGE_NAME}"

          CACHE_FROM=""; docker pull "$IMAGE_URI" && CACHE_FROM="--cache-from $IMAGE_URI"
          docker build "${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}" $CACHE_FROM \
            || docker build "${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}"
          docker push "${IMAGE_URI}:${TAG}"
          if [[ ${GITHUB_REF##*/} == "main" ]]; then
            docker tag "${IMAGE_URI}:${TAG}" "${IMAGE_URI}:latest"
            docker push "${IMAGE_URI}:latest"
          fi

          echo "IMAGE_URI=${IMAGE_URI}" >> $GITHUB_ENV
          echo "TAG=${TAG}" >> $GITHUB_ENV
      - name: run tests
        run: |
          # explicitly block EC2 IMDS endpoint to work around awscli issue:
          # https://github.com/aws/aws-cli/issues/5234#issuecomment-635459464
          # https://github.com/aws/aws-cli/issues/5262
          sudo ip route add blackhole 169.254.169.254

          # configure miniwdl to auto-delete task working directories, to reduce chance of worker
          # running out of space
          export MINIWDL__FILE_IO__OUTPUT_HARDLINKS=true
          export MINIWDL__FILE_IO__DELETE_WORK=success
          export MINIWDL__DOWNLOAD_CACHE__GET=true
          export MINIWDL__DOWNLOAD_CACHE__PUT=true
          export MINIWDL__DOWNLOAD_CACHE__DIR="$(mktemp -d --tmpdir miniwdl_download_cache_XXXXXX)"

          source /etc/profile
          sudo apt-get -qq update
          sudo apt-get -qq install --yes jq moreutils make virtualenv zip unzip httpie git shellcheck
          virtualenv --python=python3.8 .venv
          source .venv/bin/activate
          pip install -r requirements-dev.txt

          export DOCKER_IMAGE_ID="${IMAGE_URI}:${TAG}"

          make test-${{ matrix.workflow_dir }}
      - name: docker push to docker.pkg.github.com (legacy)
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" \
            | docker login docker.pkg.github.com --username $(dirname $GITHUB_REPOSITORY) --password-stdin
          LEGACY_IMAGE_URI=$(sed s/ghcr.io/docker.pkg.github.com/ <<< "$IMAGE_URI")
          docker tag "${IMAGE_URI}:${TAG}" "${LEGACY_IMAGE_URI}:${TAG}"
          docker push "${LEGACY_IMAGE_URI}:${TAG}"
          if [[ ${GITHUB_REF##*/} == "main" ]]; then
            docker tag "${IMAGE_URI}:${TAG}" "${LEGACY_IMAGE_URI}:latest"
            docker push "${LEGACY_IMAGE_URI}:latest"
          fi