.gitlab-ci.yml

image: quay.io/ucsc_cgl/toil_ci_prebake:latest
# Note that we must run in a privileged container for our internal Docker daemon to come up.

variables:
  PYTHONIOENCODING: "utf-8"
  DEBIAN_FRONTEND: "noninteractive"
  TOIL_OWNER_TAG: "shared"
  MAIN_PYTHON_PKG: "python3.11"
  # Used to tell pytest which tests to be run by specifying markers,
  # Allows partitioning of tests to prevent duplicate running of tests in different jobs.
  # Currently specifies special tests that are not run by quick_test_offline.
  MARKER: "(tes or integrative or encryption or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3) and not kubernetes"
  TEST_THREADS: "3"
before_script:
  # Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
  - ip addr
  # Configure Docker and Buildkit to use a mirror for Docker Hub and restart the daemon
  # Set the registry as insecure because it is probably cluster-internal over plain HTTP.
  - |
    if [[ ! -z "${DOCKER_HUB_MIRROR}" ]] ; then
        echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
        export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
        echo "[registry.\"docker.io\"]" >buildkitd.toml
        echo "  mirrors = [\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
        echo "[registry.\"${DOCKER_HUB_MIRROR##*://}\"]" >>buildkitd.toml
        echo "  http = true" >>buildkitd.toml
    else
        echo "" >buildkitd.toml
    fi
  # Restart or start the Docker daemon
  - stopdocker || true
  - sudo rm -f /var/run/docker.sock
  - startdocker || true
  - docker info
  - cat /etc/hosts
  - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config
  - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials
  # We need to make sure docker buildx create can't see the ~/.kube/config that we deploy. It has
  # a service account bearer token for auth and triggers https://github.com/docker/buildx/issues/267
  # where buildx can't use a bearer token from a kube config and falls back to anonymous instead
  # of using the system's service account.
  - if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then KUBECONFIG=/dev/null docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container --config ./buildkitd.toml ; fi
  # Report on the builders, and make sure they exist.
  - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
  # This will hang if we can't talk to the builder
  - (echo "y" | docker buildx prune --keep-storage 80G) || true

after_script:
  # We need to clean up any files that Toil may have made via Docker that
  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
  # and clean them up before running the next job on the runner, fail, and fail
  # that next job.
  - pwd
  - sudo rm -rf tmp
  - stopdocker || true

stages:
  - linting_and_dependencies
  - basic_tests
  - main_tests
  - integration

lint:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
    - python3.11 -m pip freeze
    - python3.11 --version
    - make mypy
    - make docs
    # - make diff_pydocstyle_report

cwl_dependency_is_stand_alone:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample

wdl_dependency_is_stand_alone:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: linting_and_dependencies
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py::WDLTests::test_MD5sum

quick_test_offline:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: basic_tests
  script:
    - ${MAIN_PYTHON_PKG} -m virtualenv venv
    - . venv/bin/activate
    - pip install -U pip wheel
    - make prepare
    - make develop extras=[aws,google,wdl]
    - TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"

py38_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: basic_tests
  script:
    - pwd
    - python3.8 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py39_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: basic_tests
  script:
    - pwd
    - python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py310_appliance_build:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: basic_tests
  script:
    - pwd
    - python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.0.post1'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py311_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 &&  pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all] packages='htcondor==10.2.3'
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py311_main:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: basic_tests
  script:
    - pwd
    - python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'
    - make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
    - TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib

batch_systems:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.3'

slurm_test:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - cd contrib/slurm-test/
    - docker compose version
    - ./slurm_test.sh

cwl_v1.2:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/cwl/*'
          - 'src/toil/test/cwl/*'
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    # Run CWL integration tests excluded from cwl_misc
    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(CWLWorkflowTest or cwl_small) and integrative'"
    # Run CWL conformance tests, with file store bypassed
    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv12Test::test_run_conformance_with_in_place_update
  artifacts:
    reports:
      junit: "*.junit.xml"
    paths:
    - "*.junit.xml"
    when: always
    expire_in: 14 day

cwl_on_arm:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/cwl/*'
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    # Run CWL conformance tests, on an ARM cluster on AWS, using the file store
    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLOnARMTest
  artifacts:
    reports:
      junit: "*.junit.xml"
    paths:
    - "*.junit.xml"
    when: always
    expire_in: 14 day

cwl_misc:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: main_tests
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(CWLWorkflowTest or cwl_small) and not integrative'"

#cwl_v1.2_kubernetes:
#  stage: main_tests
#  script:
#    - pwd
#    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[cwl,aws,kubernetes]
#    - export TOIL_KUBERNETES_OWNER=toiltest
#    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
#    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
#    - export TOIL_WORKDIR=/var/lib/toil
#    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
#    - if [[ ! -z "${KUBERNETES_DOCKER_HUB_MIRROR}" ]] ; then export SINGULARITY_DOCKER_HUB_MIRROR="${KUBERNETES_DOCKER_HUB_MIRROR}" ; fi
#    - mkdir -p ${TOIL_WORKDIR}
#    - make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py::CWLv12Test::test_kubernetes_cwl_conformance src/toil/test/cwl/cwlTest.py::CWLv12Test::test_kubernetes_cwl_conformance_with_caching"
#  artifacts:
#    reports:
#      junit: "*.junit.xml"
#    paths:
#      - "*.junit.xml"
#    when: always
#    expire_in: 14 days

wdl:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  script:
    - pwd
    - apt update && apt install -y default-jre
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests=src/toil/test/wdl/wdltoil_test.py

jobstore:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py"

provisioner:
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  stage: main_tests
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - make test threads="${TEST_THREADS}" marker="${MARKER}" tests="src/toil/test/lib/aws/ src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"

# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.

jobstore_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test threads="${TEST_THREADS}" tests="src/toil/test/jobStores/jobStoreTest.py"

server_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    # Test server and its integration with AWS
    - make test threads="${TEST_THREADS}" tests="src/toil/test/server"

provisioner_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - python setup_gitlab_docker.py
    - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/wdl/wdltoil_test.py::WDLKubernetesClusterTest"
#    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run

google_jobstore:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
  stage: integration
  script:
    - pwd
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - export TOIL_TEST_INTEGRATIVE=True
    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
    - export TOIL_GOOGLE_KEYNAME=id_rsa
    - export TOIL_GOOGLE_PROJECTID=toil-dev
    - make test threads="${TEST_THREADS}" tests=src/toil/test/jobStores/jobStoreTest.py::GoogleJobStoreTest

mesos:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/test/mesos/*'
          - 'src/toil/batchSystems/mesos/*'
  stage: integration
  script:
    - pwd
    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos,aws]
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
    - export TOIL_GOOGLE_KEYNAME=id_rsa
    - export TOIL_GOOGLE_PROJECTID=toil-dev
    - make test threads="${TEST_THREADS}" tests=src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest
    - make test threads="${TEST_THREADS}" tests=src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest
    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testAwsMesos
    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testFileMesos
    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos
    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance
    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance_with_caching
    - make test threads="${TEST_THREADS}" tests=src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
    - make test threads="${TEST_THREADS}" tests="src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"

# Cactus-on-Kubernetes integration (as a script and not a pytest test)
cactus_integration:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
    - if: $CI_COMMIT_BRANCH
      changes:
        compare_to: 'refs/heads/master'
        paths:
          - 'src/toil/test/cactus/test_cactus_integration.py'
  stage: integration
  script:
    - export CACTUS_COMMIT_SHA=f5adf4013326322ae58ef1eccb8409b71d761583
    - set -e
    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws]
    - python setup_gitlab_docker.py  # login to increase the docker.io rate limit
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test tests=src/toil/test/cactus/test_cactus_integration.py