build.yaml

steps:
  - kind: buildImage2
    name: git_make_bash_image
    publishAs: git-make-bash
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ global.docker_root_image }}
        FROM $BASE_IMAGE
        RUN apt-get update && apt-get install -y git make bash
  - kind: runImage
    name: merge_code
    image:
      valueFrom: git_make_bash_image.image
    script: |
      set -ex
      cd /io
      mkdir -p repo
      cd repo
      {{ code.checkout_script }}
      make -C hail python-version-info
      git rev-parse HEAD > git_version
    outputs:
      - from: /io/repo/
        to: /repo
      - from: /io/repo/hail/python/hail/hail_version
        to: /hail_version
      - from: /io/repo/hail/python/hail/hail_pip_version
        to: /hail_pip_version
      - from: /io/repo/git_version
        to: /git_version
    dependsOn:
      - git_make_bash_image
  - kind: createNamespace
    name: default_ns
    namespaceName: default
    secrets:
      - name: auth-oauth2-client-secret
      - name: hail-ci-0-1-github-oauth-token
      - name: testns-test-gsa-key
      - name: testns-test-dev-gsa-key
      - name: testns-auth-gsa-key
      - name: testns-batch-gsa-key
      - name: testns-ci-gsa-key
      - name: testns-grafana-gsa-key
      - name: test-aws-key
        clouds:
          - gcp
      - name: test-azure-key
        clouds:
          - gcp
      - name: zulip-config
      - name: billing-monitor-gsa-key
        clouds:
          - gcp
      - name: hail-ci-0-1-service-account-key
      - name: batch-worker-ssh-public-key
        clouds:
          - azure
  - kind: runImage
    name: copy_third_party_images
    image: quay.io/skopeo/stable:v1.13.2
    script: |
      set -ex

      retry() {
          "$@" ||
              (sleep 2 && "$@") ||
              (sleep 5 && "$@");
      }

      REGISTRY={{ global.docker_prefix.split('/')[0] }}

      {% if global.cloud == "gcp" %}
      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
      {% elif global.cloud == "azure" %}
      dnf install -y jq
      USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.appId')
      cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}

      cd /io/docker/third-party
      DOCKER_PREFIX={{ global.docker_prefix }} retry bash copy_images.sh
    inputs:
      - from: /repo/docker
        to: /io/docker
    scopes:
      - deploy
    dependsOn:
      - default_ns
      - merge_code
  - kind: buildImage2
    name: hail_ubuntu_image
    dockerFile: /io/hail-ubuntu/Dockerfile
    contextPath: /io/hail-ubuntu
    publishAs: hail-ubuntu
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/hail-ubuntu
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: buildImage2
    name: hail_ubuntu_image_python_3_10
    dockerFile: /io/hail-ubuntu/Dockerfile
    contextPath: /io/hail-ubuntu
    publishAs: hail-ubuntu-python-3-10
    buildArgs:
      - name: PYTHON_VERSION
        value: "3.10"
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/hail-ubuntu
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: buildImage2
    name: hail_ubuntu_image_python_3_11
    dockerFile: /io/hail-ubuntu/Dockerfile
    contextPath: /io/hail-ubuntu
    publishAs: hail-ubuntu-python-3-11
    buildArgs:
      - name: PYTHON_VERSION
        value: "3.11"
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/hail-ubuntu
        to: /io/hail-ubuntu
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: deploy
    name: deploy_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/service-account.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_ci_agent
    namespace:
      valueFrom: default_ns.name
    config: ci/ci-agent.yaml
    dependsOn:
      - default_ns
  - kind: deploy
    name: deploy_test_batch_sa
    namespace:
      valueFrom: default_ns.name
    config: batch/test-sa.yaml
    dependsOn:
      - default_ns
  - kind: buildImage2
    name: base_image
    dockerFile: /io/repo/docker/Dockerfile.base
    contextPath: /io/repo
    publishAs: base
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/docker/Dockerfile.base
        to: /io/repo/docker/Dockerfile.base
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: create_certs_image
    dockerFile: /io/tls/Dockerfile
    contextPath: /io/tls
    publishAs: create_certs_image
    inputs:
      - from: /repo/tls
        to: /io/tls
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: runImage
    name: create_ssl_config_hail_root
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |

      {% if not deploy %}
      kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root \
        --template={% raw %}'{{index .data "hail-root-cert.pem"}}'{% endraw %} \
        | base64 --decode \
        | openssl x509 -checkend 0 -noout -in -

      if [ "$?" -ne 0 ]
      then
          kubectl delete secret -n {{ default_ns.name }} ssl-config-hail-root
      fi
      {% endif %}

      openssl req -new -x509 -subj /CN=hail-root -nodes -newkey rsa:4096 -keyout hail-root-key.pem -out hail-root-cert.pem
      until kubectl get secret -n {{ default_ns.name }} ssl-config-hail-root
      do
          kubectl create secret generic -n {{ default_ns.name }} ssl-config-hail-root \
                  --from-file=hail-root-key.pem \
                  --from-file=hail-root-cert.pem
      done
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - create_certs_image
  - kind: runImage
    name: create_certs
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |
      set -ex
      python3 create_certs.py \
              {{ default_ns.name }} \
              config.yaml \
              /ssl-config-hail-root/hail-root-key.pem \
              /ssl-config-hail-root/hail-root-cert.pem
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    secrets:
      - name: ssl-config-hail-root
        namespace:
          valueFrom: default_ns.name
        mountPath: /ssl-config-hail-root
    dependsOn:
      - default_ns
      - create_certs_image
      - create_ssl_config_hail_root
  - kind: buildImage2
    name: ci_utils_image
    dockerFile: /io/repo/ci/Dockerfile.ci-utils
    contextPath: /io/repo
    publishAs: ci-utils
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: create_test_database_server_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: create_certs_image.image
    script: |
      set -ex

      if ! kubectl get secret -n {{ default_ns.name }} database-server-config;
      then
          NAMESPACE={{ default_ns.name }} bash /create_test_db_config.sh
      fi
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - create_certs_image
  - kind: deploy
    name: deploy_test_db
    namespace:
      valueFrom: default_ns.name
    config: docker/mysql/db.yaml
    wait:
      - kind: Service
        name: db
        for: alive
        resource_type: statefulset
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - create_test_database_server_config
  - kind: buildImage2
    name: admin_pod_image
    dockerFile: /io/repo/admin-pod/Dockerfile
    contextPath: /io/repo
    publishAs: admin-pod
    inputs:
      - from: /repo/admin-pod
        to: /io/repo/admin-pod
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: auth_image
    dockerFile: /io/repo/auth/Dockerfile
    contextPath: /io/repo
    publishAs: auth
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/auth
        to: /io/repo/auth
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: runImage
    name: delete_auth_tables
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} auth
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - auth_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: auth_database
    databaseName: auth
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: copy-paste-tokens
        script: /io/sql/copy-paste-tokens.sql
      - name: drop-bucket
        script: /io/sql/drop-bucket.sql
      - name: add-trial-billing-project
        script: /io/sql/add-trial-billing-project.sql
      - name: add-roles
        script: /io/sql/add-roles.sql
      - name: rename-gsa-identity
        script: /io/sql/rename-gsa-identity.sql
      - name: support-azure-oauth
        script: /io/sql/support-azure-oauth.sql
      - name: change-username-collation
        script: /io/sql/change-username-collation.sql
      - name: add-hail-identity-uid
        script: /io/sql/add-hail-identity-uid.sql
    inputs:
      - from: /repo/auth/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: auth
    dependsOn:
      - default_ns
      - merge_code
      - delete_auth_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: runImage
    name: create_deploy_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex

      # k8s deploy config
      {% if default_ns.name == "default" %}
      cat > deploy-config.json <<EOF
      {"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"{{ global.domain }}"}
      EOF
      {% else %}
      cat > deploy-config.json <<EOF
      {"location":"k8s","default_namespace":"{{ default_ns.name }}","domain":"internal.{{ global.domain }}","base_path":"/{{ default_ns.name }}"}
      EOF
      {% endif %}
      kubectl -n {{ default_ns.name }} create secret generic deploy-config \
              --from-file=./deploy-config.json \
              --save-config --dry-run=client -o yaml \
          | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: deploy
    name: deploy_admin_pod
    namespace:
      valueFrom: default_ns.name
    config: admin-pod/admin-pod.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - admin_pod_image
      - create_deploy_config
      - create_certs
  - kind: runImage
    name: create_session_key
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      # create session keys
      N=$(kubectl -n {{ default_ns.name }} get secret --ignore-not-found=true --no-headers session-secret-key | wc -l | tr -d '[:space:]')
      if [[ $N != 0 ]]; then
        exit
      fi
      mkdir /session-secret-key
      cat > generate-session-key.py <<EOF
      import base64
      from cryptography import fernet
      with open('/session-secret-key/session-secret-key', 'wb') as f:
          f.write(base64.urlsafe_b64decode(fernet.Fernet.generate_key()))
      EOF
      python3 generate-session-key.py
      kubectl -n {{ default_ns.name }} create secret generic session-secret-key --from-file=/session-secret-key/session-secret-key
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: runImage
    name: create_test_gsa_keys
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      kubectl -n {{ default_ns.name }} get -o json secret testns-test-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-test-dev-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "test-dev-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-auth-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "auth-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-batch-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "batch-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-ci-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "ci-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
      kubectl -n {{ default_ns.name }} get -o json secret testns-grafana-gsa-key | jq '{apiVersion, kind, type, data, metadata: {name: "grafana-gsa-key"}}' | kubectl -n {{ default_ns.name }} apply -f -
    scopes:
      - test
      - dev
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - default_ns
      - ci_utils_image
  - kind: deploy
    name: deploy_auth_driver_service_account
    namespace:
      valueFrom: default_ns.name
    config: auth/auth-driver-service-account.yaml
    dependsOn:
      - default_ns
  - kind: runImage
    name: create_accounts
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: auth_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_SCOPE={{ scope }}
      export GOOGLE_APPLICATION_CREDENTIALS=/auth-gsa-key/key.json

      python3 /io/bootstrap_create_accounts.py
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    secrets:
      - name:
          valueFrom: auth_database.user_secret_name
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /database-server-config
      - name: auth-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /auth-gsa-key
      - name: global-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /global-config
    inputs:
      - from: /repo/ci/bootstrap_create_accounts.py
        to: /io/bootstrap_create_accounts.py
    dependsOn:
      - default_ns
      - deploy_test_batch_sa
      - auth_database
      - auth_image
      - create_deploy_config
      - deploy_auth_driver_service_account
      - create_test_gsa_keys
      - create_test_database_server_config
  - kind: buildImage2
    name: hailgenetics_vep_grch37_85_image
    dockerFile: /io/repo/docker/hailgenetics/vep/grch37/85/Dockerfile
    contextPath: /io/repo/docker/vep/
    publishAs: hailgenetics/vep-grch37-85
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: buildImage2
    name: hailgenetics_vep_grch38_95_image
    dockerFile: /io/repo/docker/hailgenetics/vep/grch38/95/Dockerfile
    contextPath: /io/repo/docker/vep/
    publishAs: hailgenetics/vep-grch38-95
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: buildImage2
    name: monitoring_image
    dockerFile: /io/repo/monitoring/Dockerfile
    contextPath: /io/repo
    publishAs: monitoring
    inputs:
      - from: /repo/monitoring
        to: /io/repo/monitoring
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: batch_image
    dockerFile: /io/repo/batch/Dockerfile
    contextPath: /io/repo
    publishAs: batch
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: hailgenetics_hailtop_image
    dockerFile: /io/repo/docker/hailgenetics/hailtop/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hailtop
    inputs:
      - from: /repo/docker/hailgenetics/hailtop
        to: /io/repo/docker/hailgenetics/hailtop
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail_version
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: buildImage2
    name: ci_image
    dockerFile: /io/repo/ci/Dockerfile
    contextPath: /io/repo
    publishAs: ci
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: buildImage2
    name: hail_buildkit_image
    dockerFile: /io/repo/ci/buildkit/Dockerfile
    contextPath: /io/repo/ci
    publishAs: hail-buildkit
    inputs:
      - from: /repo/ci
        to: /io/repo/ci
    dependsOn:
      - merge_code
      - copy_third_party_images
  - kind: runImage
    name: build_hail_jar_and_wheel
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail
      time retry sh millw --version
      time retry make shadowJar wheel HAIL_RELEASE_MODE=1

      # Check wheel size is small enough for pypi (< 200 MiB)
      HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version)
      WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl"
      du -h $WHEEL_PATH
      $(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)")

      mkdir /io/wheel
      mv build/deploy/dist/hail-*-py3-none-any.whl /io/wheel/
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/out/assembly.dest/out.jar
        to: /hail.jar
      - from: /io/wheel
        to: /wheel
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: build_hail_debug_jar_and_wheel
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail
      time retry sh millw --version
      time retry make jars wheel
      mkdir /io/debug-wheel
      mv build/deploy/dist/hail-*-py3-none-any.whl /io/debug-wheel/
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/out/test/assembly.dest/out.jar
        to: /hail-debug-test.jar
      - from: /io/debug-wheel
        to: /debug-wheel
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: build_hail_test_artifacts
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail
      time retry sh millw --version
      time retry make shadowTestJar

      time tar czf test.tar.gz -C python test
      time tar czf resources.tar.gz -C src/test resources
      time tar czf data.tar.gz -C python/hail/docs data
      time TESTNG_SPLITS=5 python3 generate_splits.py
      time tar czf splits.tar.gz testng-splits-*.xml
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/repo/hail/out/test/assembly.dest/out.jar
        to: /hail-test.jar
      - from: /io/repo/hail/test.tar.gz
        to: /test.tar.gz
      - from: /io/repo/hail/resources.tar.gz
        to: /resources.tar.gz
      - from: /io/repo/hail/splits.tar.gz
        to: /splits.tar.gz
      - from: /io/repo/hail/data.tar.gz
        to: /data.tar.gz
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: build_azure_wheel
    image:
      valueFrom: base_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io/repo/hail
      time retry sh millw --version
      export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_RELEASE_MODE=1
      time retry make wheel

      mkdir /io/azure-wheel
      mv build/deploy/dist/hail-*-py3-none-any.whl /io/azure-wheel/
    inputs:
      - from: /repo
        to: /io/repo
    outputs:
      - from: /io/azure-wheel
        to: /azure-wheel
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: hail_run_image
    dockerFile: /io/repo/hail/Dockerfile.hail-run
    contextPath: /io/repo
    publishAs: hail-run
    resources:
      storage: 10Gi
      cpu: "2"
      memory: standard
    inputs:
      - from: /repo/hail/Dockerfile.hail-run
        to: /io/repo/hail/Dockerfile.hail-run
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
      - from: /repo/docker/core-site.xml
        to: /io/repo/docker/core-site.xml
    dependsOn:
      - base_image
      - merge_code
  - kind: runImage
    name: jvm_entryway_jar
    image:
      valueFrom: base_image.image
    script: |
      set -ex

      cd /io/batch/jvm-entryway

      chmod 755 ./gradlew
      ./gradlew shadowJar
    inputs:
      - from: /repo/batch
        to: /io/batch
    outputs:
      - from: /io/batch/jvm-entryway/build/libs/jvm-entryway.jar
        to: /jvm-entryway.jar
    dependsOn:
      - base_image
      - merge_code
  - kind: buildImage2
    name: batch_worker_image
    dockerFile: /io/repo/batch/Dockerfile.worker
    contextPath: /io/repo
    publishAs: batch-worker
    inputs:
      - from: /repo/letsencrypt/subdomains.txt
        to: /io/repo/letsencrypt/subdomains.txt
      - from: /repo/docker
        to: /io/repo/docker
      - from: /repo/batch
        to: /io/repo/batch
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
      - from: /repo/letsencrypt/subdomains.txt
        to: /repo/letsencrypt/subdomains.txt
      - from: /jvm-entryway.jar
        to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar
      - from: /hail_version
        to: /io/repo/hail_version
    dependsOn:
      - merge_code
      - hail_ubuntu_image
      - jvm_entryway_jar
  - kind: runImage
    name: upload_test_resources_to_blob_storage
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      cd /io/repo/hail/
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "src/test/resources",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/test/resources"},
      {"from": "python/hail/docs/data",
       "to":   "{{ global.test_storage_uri }}/{{ token }}/doctest/data"}
      ]'
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    inputs:
      - from: /repo/hail/src/test/resources
        to: /io/repo/hail/src/test/resources
      - from: /repo/hail/python/hail/docs/data
        to: /io/repo/hail/python/hail/docs/data
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - create_test_gsa_keys
      - merge_code
  - kind: runImage
    name: test_hail_java
    numSplits: 5
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test
      tar xzf splits.tar.gz
      export HAIL_TEST_SKIP_R=1
      java -cp hail-test.jar:$SPARK_HOME/jars/* org.testng.TestNG -listener is.hail.LogTestListener testng-splits-$HAIL_RUN_IMAGE_SPLIT_INDEX.xml
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /hail-debug-test.jar
        to: /io/hail-test.jar
      - from: /splits.tar.gz
        to: /io/splits.tar.gz
    outputs:
      - from: /io/test-output
        to: /test-output
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_hail_debug_jar_and_wheel
      - build_hail_test_artifacts
  - kind: runImage
    name: test_hail_python
    numSplits: 28
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/debug-wheel/hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/batch/ \
              --ignore=test/hailtop/inter_cloud \
              --timeout=120 \
              test
    inputs:
      - from: /debug-wheel
        to: /io/debug-wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_hail_debug_jar_and_wheel
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_fs
    numSplits: 5
    image:
      valueFrom: hailgenetics_hailtop_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz

      python3 -m pip install -r test-requirements.txt

      export HAIL_TEST_GCS_BUCKET={{ global.hail_test_gcs_bucket }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_S3_BUCKET=hail-test-dy5rg
      export AWS_SHARED_CREDENTIALS_FILE=/test-aws-key/credentials

      export HAIL_TEST_AZURE_ACCOUNT=hailtest
      export HAIL_TEST_AZURE_CONTAINER=hail-test-4nxei
      # Required for SAS testing on Azure
      export HAIL_TEST_AZURE_RESGRP=hail-dev
      export HAIL_TEST_AZURE_SUBID=22cd45fe-f996-4c51-af67-ef329d977519
      export AZURE_APPLICATION_CREDENTIALS=/test-azure-key/credentials.json

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              test/hailtop/inter_cloud
    timeout: 1200
    inputs:
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/test-requirements.txt
      - from: /test.tar.gz
        to: /io/test.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-aws-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-aws-key
      - name: test-azure-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-azure-key
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - create_test_gsa_keys
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_unchecked_allocator
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      # pyspark/conf/core-site.xml already points at /gsa-key/key.json
      mv /test-gsa-key/key.json /gsa-key/key.json

      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              -m unchecked_allocator \
              --ignore=test/hailtop/batch/ \
              --ignore=test/hailtop/inter_cloud \
              --timeout=120 \
              test
    inputs:
      - from: /wheel
        to: /io/wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - build_hail_jar_and_wheel
      - build_hail_test_artifacts
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_local_backend
    numSplits: 48
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      tar xzf test.tar.gz
      tar xzf resources.tar.gz
      tar xzf data.tar.gz
      python3 -m pip install --no-dependencies /io/debug-wheel/hail-*-py3-none-any.whl
      mkdir -p /io/tmp

      # The test should use the test credentials, not CI's credentials
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_TEST_STORAGE_URI=/io/tmp/
      export HAIL_TEST_RESOURCES_DIR=$(realpath ./resources)
      export HAIL_DOCTEST_DATA_DIR=$(realpath ./data)
      export HAIL_LOCAL_BACKEND_HEAP_SIZE=3G

      hailctl config set query/backend local

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/ \
              --timeout=120 \
              test
    timeout: 1800
    inputs:
      - from: /debug-wheel
        to: /io/debug-wheel
      - from: /test.tar.gz
        to: /io/test.tar.gz
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /data.tar.gz
        to: /io/data.tar.gz
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - hail_run_image
      - create_test_gsa_keys
      - build_hail_debug_jar_and_wheel
      - build_hail_test_artifacts
  - kind: runImage
    name: test_python_docs
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      export HAIL_QUERY_N_CORES=2
      export OMP_NUM_THREADS=2
      export OPENBLAS_NUM_THREADS=2
      export MKL_NUM_THREADS=2
      export VECLIB_MAXIMUM_THREADS=2
      export NUMEXPR_NUM_THREADS=2

      export PYSPARK_SUBMIT_ARGS="--driver-memory 6g pyspark-shell"
      export HAIL_LOCATION=$(dirname $(python3 -c 'import hail; print(hail.__file__)'))

      mv /io/docs $HAIL_LOCATION/docs

      # NB: do not use --timeout with doctest because there is no way to override it.
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --doctest-modules \
              --doctest-glob='*.rst' \
              --doctest-continue-on-failure \
              --ignore=$HAIL_LOCATION/test \
              --ignore=$HAIL_LOCATION/docs/conf.py \
              --ignore=$HAIL_LOCATION/docs/doctest_write_data.py \
              --rootdir $HAIL_LOCATION \
              $HAIL_LOCATION
    inputs:
      - from: /repo/hail/python/hail/docs
        to: /io/docs
      - from: /wheel
        to: /io/wheel
    dependsOn:
      - build_hail_jar_and_wheel
      - hail_run_image
    clouds:
      - gcp
  - kind: runImage
    name: make_docs
    resources:
      memory: standard
      cpu: '4'
    image:
      valueFrom: hail_run_image.image
    script: |
      set -ex
      set -o pipefail
      export HAIL_SHORT_VERSION='0.2'
      export SPHINXOPTS='-tgenerate_notebook_outputs'

      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/hail
      export HAIL_WEBSITE_DIR=/io/website/website/

      sed -E "s/\(hail\#([0-9]+)\)/(\[#\1](https:\/\/github.com\/hail-is\/hail\/pull\/\1))/g" \
        < python/hail/docs/change_log.md \
        | pandoc -o python/hail/docs/change_log.rst --fail-if-warnings --verbose

      make -C python/hail/docs BUILDDIR=_build clean html
      make -C python/hailtop/batch/docs BUILDDIR=_build clean html

      mkdir -p www/docs
      mv python/hail/docs/_build/html www/docs/0.2
      mv python/hailtop/batch/docs/_build/html www/docs/batch

      HAIL_CACHE_VERSION=$(cat python/hail/hail_version)
      find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +;

      tar czf /io/www.tar.gz www
    inputs:
      - from: /repo/hail/python
        to: /io/hail/python
      - from: /hail_version
        to: /io/repo/hail/python/hail/hail_version
      - from: /repo/website/website
        to: /io/website/website
      - from: /wheel
        to: /io/wheel
    outputs:
      - from: /io/www.tar.gz
        to: /www.tar.gz
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_run_image
  - kind: buildImage2
    name: hailgenetics_hail_image
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /wheel
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image
  - kind: buildImage2
    name: hailgenetics_hail_image_python_3_10
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image_python_3_10.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /wheel
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image_python_3_10
  - kind: buildImage2
    name: hailgenetics_hail_image_python_3_11
    dockerFile: /io/repo/docker/hailgenetics/hail/Dockerfile
    contextPath: /io/repo
    publishAs: hailgenetics/hail
    buildArgs:
      - name: BASE_IMAGE
        value:
          valueFrom: hail_ubuntu_image_python_3_11.image
      - name: HAIL_WHEEL_DIR
        value: /wheel
    inputs:
      - from: /repo/docker/hailgenetics/hail
        to: /io/repo/docker/hailgenetics/hail
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /wheel
        to: /io/repo/wheel
    dependsOn:
      - merge_code
      - build_hail_jar_and_wheel
      - hail_ubuntu_image_python_3_11
  - kind: runImage
    name: test_hailgenetics_hail_image
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex
      python3 --version | grep '3.9'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image
  - kind: runImage
    name: test_hailgenetics_hail_image_python_3_10
    image:
      valueFrom: hailgenetics_hail_image_python_3_10.image
    script: |
      set -ex
      python3 --version | grep '3.10'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image_python_3_10
  - kind: runImage
    name: test_hailgenetics_hail_image_python_3_11
    image:
      valueFrom: hailgenetics_hail_image_python_3_11.image
    script: |
      set -ex
      python3 --version | grep '3.11'
      python3 -c 'import hail as hl; hl.balding_nichols_model(3, 100, 100)._force_count_rows()'
      python3 -c 'import numpy; import pandas; import sklearn; import scipy'
    dependsOn:
      - hailgenetics_hail_image_python_3_11
  - kind: buildImage2
    name: hail_dev_image
    dockerFile: /io/repo/docker/Dockerfile.hail-dev
    contextPath: /io/repo
    publishAs: hail-dev
    inputs:
      - from: /repo/docker/Dockerfile.hail-dev
        to: /io/repo/docker/Dockerfile.hail-dev
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
    dependsOn:
      - hailgenetics_hail_image
      - merge_code
  - kind: buildImage2
    name: hail_linting_image
    contextPath: /io/repo
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-apt-get-install git make openjdk-11-jre-headless
        COPY hail/python/pinned-requirements.txt hail-requirements.txt
        COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt
        COPY gear/pinned-requirements.txt gear-requirements.txt
        COPY web_common/pinned-requirements.txt web_common-requirements.txt
        COPY batch/pinned-requirements.txt batch-requirements.txt
        COPY ci/pinned-requirements.txt ci-requirements.txt
        RUN python3 -m pip install \
            -r hail-requirements.txt \
            -r dev-requirements.txt \
            -r gear-requirements.txt \
            -r web_common-requirements.txt \
            -r batch-requirements.txt \
            -r ci-requirements.txt \
            && \
            python3 -m pyright --version # Force the installation of node and npm pyright in the image
    publishAs: linting
    inputs:
      - from: /repo/hail/python/pinned-requirements.txt
        to: /io/repo/hail/python/pinned-requirements.txt
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/repo/hail/python/dev/pinned-requirements.txt
      - from: /repo/gear/pinned-requirements.txt
        to: /io/repo/gear/pinned-requirements.txt
      - from: /repo/web_common/pinned-requirements.txt
        to: /io/repo/web_common/pinned-requirements.txt
      - from: /repo/batch/pinned-requirements.txt
        to: /io/repo/batch/pinned-requirements.txt
      - from: /repo/ci/pinned-requirements.txt
        to: /io/repo/ci/pinned-requirements.txt
    dependsOn:
      - merge_code
      - hail_ubuntu_image
  - kind: runImage
    name: check_pip_requirements
    image:
      valueFrom: hail_linting_image.image
    script: |
      set -ex
      cd /io/repo
      chmod 755 ./check_pip_requirements.sh
      make check-pip-requirements
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - hail_linting_image
      - merge_code
  - kind: runImage
    name: check_services
    image:
      valueFrom: hail_linting_image.image
    script: |
      set -ex
      cd /io/repo
      {% if 'target_sha' in code %}
      export HAIL_TARGET_SHA={{ code.target_sha }}
      {% endif %}
      make -k check-services
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - hail_linting_image
      - merge_code
  - kind: runImage
    name: check_hail
    image:
      valueFrom: hail_linting_image.image
    script: |
      set -ex
      cd /io/repo
      make check-hail
    inputs:
      - from: /repo/Makefile
        to: /io/repo/Makefile
      - from: /repo/pylintrc
        to: /io/repo/pylintrc
      - from: /repo/pyproject.toml
        to: /io/repo/pyproject.toml
      - from: /repo/hail
        to: /io/repo/hail
      - from: /repo/config.mk
        to: /io/repo/config.mk
    dependsOn:
      - hail_linting_image
      - merge_code
  - kind: runImage
    name: get_pip_versioned_docs
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex

      cd /io/repo

      # dev deploy elides the hail-is remote, add it and retrieve the tags
      git remote add hail-is https://github.com/hail-is/hail.git

      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)

      if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION
      then
        # In this case, we want to get the docs from Google Storage.
        python3 -m hailtop.aiotools.copy 'null' '[
        {"from": "gs://hail-common/website/'$HAIL_PIP_VERSION'/www.tar.gz",
        "to": "www.tar.gz"}
        ]'
      else
        cp /io/www.tar.gz .
      fi

      mkdir -p hail/build/www/docs/0.2
      tar -xvf www.tar.gz -C hail/build/www/docs --strip-components 2 --no-same-owner

      python3 -m hailtop.aiotools.copy 'null' '[
      {"from": "gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz",
       "to": "hail-0.1-docs-5a6778710097.tar.gz"}
      ]'

      mkdir -p hail/build/www/docs/0.1
      tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2 --no-same-owner

      tar czf /io/docs.tar.gz -C hail/build/www .
    resources:
      memory: standard
      cpu: '2'
    inputs:
      - from: /repo
        to: /io/repo
      - from: /www.tar.gz
        to: /io/www.tar.gz
      - from: /hail_pip_version
        to: /io/hail_pip_version
    outputs:
      - from: /io/docs.tar.gz
        to: /docs.tar.gz
    dependsOn:
      - hailgenetics_hail_image
      - merge_code
      - make_docs
    clouds:
      - gcp
  - kind: deploy
    name: deploy_grafana
    namespace:
      valueFrom: default_ns.name
    config: grafana/deployment.yaml
    scopes:
      - deploy
      - test
      - dev
    dependsOn:
      - default_ns
      - hail_ubuntu_image
      - create_certs
      - copy_third_party_images
  - kind: deploy
    name: deploy_prometheus
    namespace:
      valueFrom: default_ns.name
    config: prometheus/prometheus.yaml
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - create_certs
      - copy_third_party_images
  - kind: deploy
    name: deploy_auth
    namespace:
      valueFrom: default_ns.name
    config: auth/deployment.yaml
    wait:
      - kind: Service
        name: auth
        for: alive
      - kind: Service
        name: auth-driver
        for: alive
    dependsOn:
      - default_ns
      - create_deploy_config
      - create_session_key
      - auth_database
      - auth_image
      - create_certs
      - create_accounts
  - kind: runImage
    name: create_initial_user
    runIfRequested: true
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      {% if default_ns.name == "default" %}
      hailctl auth create-user --developer {{ code.username }} {{ code.login_id }}
      {% else %}
      hailctl auth create-user \
        --developer \
        --hail-identity {{ code.hail_identity }} \
        --hail-credentials-secret-name {{ code.username }}-gsa-key \
        {{ code.username }} {{ code.login_id }}
      {% endif %}
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - merge_code
      - deploy_auth
  - kind: runImage
    name: delete_monitoring_tables
    image:
      valueFrom: monitoring_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} monitoring
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - monitoring_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: monitoring_database
    databaseName: monitoring
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/initial.sql
    inputs:
      - from: /repo/monitoring/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: monitoring
    dependsOn:
      - default_ns
      - merge_code
      - delete_monitoring_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: deploy
    name: deploy_monitoring
    namespace:
      valueFrom: default_ns.name
    config: monitoring/deployment.yaml
    wait:
      - kind: Service
        name: monitoring
        for: alive
    dependsOn:
      - default_ns
      - monitoring_image
      - monitoring_database
      - deploy_auth
      - create_certs
      - create_accounts
    clouds:
      - gcp
  - kind: runImage
    name: test_monitoring
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=120 \
              /io/monitoring/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    timeout: 300
    inputs:
      - from: /repo/monitoring/test
        to: /io/monitoring/test
    dependsOn:
      - hail_dev_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_monitoring
    clouds:
      - gcp
  - kind: runImage
    name: test_auth_copy_paste_login
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      # Or else hailctl will try to authenticate as CI
      unset HAIL_IDENTITY_PROVIDER_JSON

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 10 \
                         --retry-delay 5 \
                         -XPOST)
      hailctl auth copy-paste-login "$COPY_PASTE_TOKEN"

      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN"
      then
          echo "reusing a token should not work, but did"
          exit 1
      fi

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 3 \
                         --retry-delay 5 \
                         -XPOST)
      python3 -c '
      from hailtop.auth import copy_paste_login;
      copy_paste_login("'$COPY_PASTE_TOKEN'")
      '

      python3 -c '
      from hailtop.auth import copy_paste_login;
      import aiohttp
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'")
          print("reusing a token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - hailgenetics_hailtop_image
  - kind: runImage
    name: test_auth_copy_paste_login_timeout
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      # Or else hailctl will try to authenticate as CI
      unset HAIL_IDENTITY_PROVIDER_JSON

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json

      COPY_PASTE_TOKEN=$(hailctl curl {{ default_ns.name }} \
                         auth /api/v1alpha/copy-paste-token \
                         -fsSL \
                         --retry 10 \
                         --retry-delay 5 \
                         -XPOST)
      sleep $(( 5 * 60 + 1))
      if hailctl auth copy-paste-login "$COPY_PASTE_TOKEN" --namespace {{ default_ns.name }}
      then
          echo "using an expired token should not work, but did"
          exit 1
      fi

      python3 -c '
      import aiohttp
      from hailtop.auth import copy_paste_login;
      try:
          copy_paste_login("'$COPY_PASTE_TOKEN'")
          print("using an expired token should not work, but did")
          sys.exit(1)
      except aiohttp.client_exceptions.ClientResponseError as exc:
          assert(exc.status == 401)
      '
    secrets:
      - name: test-tokens
        namespace:
          valueFrom: default_ns.name
        mountPath: /user-tokens
    dependsOn:
      - default_ns
      - create_accounts
      - deploy_auth
      - create_deploy_config
      - hailgenetics_hailtop_image
  - kind: runImage
    name: delete_batch_tables
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} batch
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - batch_image
      - merge_code
      - create_test_database_server_config
  - kind: runImage
    name: delete_ci_tables
    image:
      valueFrom: ci_image.image
    script: |
      set -ex
      cd /io
      python3 delete_dev_database.py {{ default_ns.name }} ci
    inputs:
      - from: /repo/ci/delete_dev_database.py
        to: /io/delete_dev_database.py
    secrets:
      - name: database-server-config
        namespace:
          valueFrom: default_ns.name
        mountPath: /sql-config
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    network: private
    runIfRequested: true
    scopes:
      - dev
    dependsOn:
      - default_ns
      - ci_image
      - merge_code
      - create_test_database_server_config
  - kind: createDatabase2
    name: ci_database
    databaseName: ci
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: invalidated-batches
        script: /io/sql/invalidated-batches.sql
      - name: add-frozen-mode
        script: /io/sql/add-frozen-mode.sql
        online: true
      - name: active-namespaces
        script: /io/sql/active-namespaces.sql
        online: true
      - name: fix-default-namespace-record
        script: /io/sql/fix-default-namespace.py
        online: true
      - name: record-failure-notifications
        script: /io/sql/record-failure-notifications.sql
        online: true
    inputs:
      - from: /repo/ci/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: ci
    dependsOn:
      - default_ns
      - merge_code
      - delete_ci_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: createDatabase2
    name: batch_database
    databaseName: batch
    image:
      valueFrom: ci_utils_image.image
    migrations:
      - name: initial
        script: /io/sql/initial.sql
      - name: insert-globals
        script: /io/sql/insert_globals.py
      - name: create-billing-projects
        script: /io/sql/create-billing-projects.sql
      - name: increase-spec-size
        script: /io/sql/increase-spec-size.sql
      - name: remove-fk-batches-user-resources
        script: /io/sql/remove-fk-batches-user-resources.sql
      - name: change-ready-cores-bigint
        script: /io/sql/change-ready-cores-bigint.sql
      - name: add-user-resources-ready-cores-token
        script: /io/sql/add-user-resources-ready-cores-token.sql
      - name: add-lock-in-share-mode
        script: /io/sql/add-lock-in-share-mode.sql
      - name: add-batch-create-token
        script: /io/sql/add-batch-create-token.sql
      - name: improve-cancel
        script: /io/sql/improve-cancel.sql
      - name: store-specs-in-gcs
        script: /io/sql/store-specs-in-gcs.sql
      - name: add-batches-time-closed
        script: /io/sql/add-batches-time-closed.sql
      - name: add-instance-zone
        script: /io/sql/add-instance-zone.sql
      - name: add-batches-time-created-index
        script: /io/sql/add-batches-time-created-index.sql
      - name: change-test-worker-disk-size
        script: /io/sql/change_test_worker_disk_size.py
      - name: add-resources
        script: /io/sql/add-resources.sql
      - name: insert-resources
        script: /io/sql/insert_resources.py
      - name: insert-standing-worker-globals
        script: /io/sql/insert_standing_worker_globals.py
      - name: insert-local-ssd-resource
        script: /io/sql/insert_local_ssd_resource.py
      - name: fix-mark-job-complete-on-error
        script: /io/sql/fix-mark-job-complete-on-error.sql
      - name: add-worker-pd-ssd-data-disk
        script: /io/sql/add-worker-pd-ssd-data-disk.sql
      - name: add-aggregated-batch-resources-tokens
        script: /io/sql/add-aggregated-batch-resources-tokens.sql
      - name: change-test-worker-pool-size
        script: /io/sql/change_test_worker_pool_size.py
      - name: add-test-dev-to-billing-project-users
        script: /io/sql/add-test-dev-to-billing-project-users.sql
      - name: add-status-flag-billing-projects
        script: /io/sql/add-status-flag-billing-projects.sql
      - name: add-aggregated-billing-project-resources
        script: /io/sql/add-aggregated-billing-project-resources.sql
      - name: add-enable-standing-worker
        script: /io/sql/add-enable-standing-worker.py
      - name: add-inst-colls
        script: /io/sql/add-inst-colls.sql
      - name: add-creating-job-state
        script: /io/sql/add-creating-job-state.sql
      - name: add-job-private-inst-coll
        script: /io/sql/add-job-private-inst-coll.sql
      - name: insert-nonpreemptible-resources
        script: /io/sql/insert_nonpreemptible_resources.py
      - name: fix-schedule-job
        script: /io/sql/fix-schedule-job.sql
      - name: increase-test-and-dev-pool-sizes
        script: /io/sql/increase-test-and-dev-pool-sizes.py
      - name: fix-provisioning-bug
        script: /io/sql/fix-provisioning-bug.sql
      - name: do-not-lock-entire-batch
        script: /io/sql/do-not-lock-entire-batch.sql
      - name: add-fail-fast
        script: /io/sql/add-fail-fast.sql
      - name: add-frozen-mode
        script: /io/sql/add-frozen-mode.sql
      - name: add-instance-config
        script: /io/sql/add-instance-config.sql
      - name: cloud-agnostic
        script: /io/sql/cloud-agnostic.sql
      - name: support-azure
        script: /io/sql/support-azure.sql
      - name: add-azure-tables
        script: /io/sql/add_azure_tables.py
      - name: change-azure-test-highcpu-pool
        script: /io/sql/change_azure_test_highcpu_pool.py
      - name: add-latest-product-versions
        script: /io/sql/add-latest-product-versions.sql
      - name: add-products-and-versions
        script: /io/sql/add_products_and_versions.py
      - name: kill-more-deadlocks
        script: /io/sql/kill-more-deadlocks.sql
      - name: kill-more-deadlocks2
        script: /io/sql/kill-more-deadlocks2.sql
      - name: no-locks-add-attempt
        script: /io/sql/no-locks-add-attempt.sql
      - name: fix-n-cancelled-creating-jobs
        script: /io/sql/fix-n-cancelled-creating-jobs.sql
      - name: big-test-instances
        script: /io/sql/big-test-instances.py
      - name: kill-mjc-deadlocks
        script: /io/sql/kill-mjc-deadlocks.sql
      - name: add-nonpreemptible-pools
        script: /io/sql/add-nonpreemptible-pools.sql
      - name: add-pool-label
        script: /io/sql/add-pool-label.sql
      - name: set-test-and-dev-pools-to-8-core-max-2
        script: /io/sql/set-test-and-dev-pools-to-8-core-max-2.py
      - name: set-test-and-dev-jpim-to-max-5
        script: /io/sql/set-test-and-dev-jpim-to-max-5.py
        online: true
      - name: minimize-deadlock-errors
        script: /io/sql/minimize-deadlock-errors.sql
        online: true
      - name: add-resource-ids-sql
        script: /io/sql/add-resource-ids.sql
        online: true
      - name: add-resource-ids-python
        script: /io/sql/add_resource_ids.py
        online: true
      - name: modify-resource-id-trigger
        script: /io/sql/modify-resource-id-trigger.sql
        online: true
      - name: rm-resource-names-agg-resources
        script: /io/sql/rm-resource-names-agg-resources.sql
        online: true
      - name: rm-resource-foreign-keys
        script: /io/sql/rm-resource-foreign-keys.py
        online: true
      - name: rm-resource-names-agg-resources-pt-2
        script: /io/sql/rm-resource-names-agg-resources-pt-2.sql
        online: true
      - name: add-att-resources-format-version-lt-3
        script: /io/sql/add-att-resources-format-version-lt-3.sql
        online: true
      - name: insert-attempt-resources-format-version-lt-3
        script: /io/sql/insert_attempt_resources_format_version_lt_3.py
        online: true
      - name: revert-attempt-resources-trigger-back-compat
        script: /io/sql/revert-attempt-resources-trigger-back-compat.sql
        online: true
      - name: add-agg-billing-by-date
        script: /io/sql/add-agg-billing-by-date.sql
        online: true
      - name: populate-agg-billing-by-date
        script: /io/sql/populate_agg_billing_by_date.py
        online: true
      - name: revert-temp-agg-by-date-infra
        script: /io/sql/revert-temp-agg-by-date-infra.sql
        online: true
      - name: rename-timestamp-to-date
        script: /io/sql/rename-timestamp-to-date.sql
        online: true
      - name: add-real-time-billing
        script: /io/sql/add-real-time-billing.sql
        online: true
      - name: add-batch-updates
        script: /io/sql/add-batch-updates.sql
        online: true
      - name: populate-batch-updates
        script: /io/sql/populate_batch_updates.py
        online: true
      - name: cleanup-add-batch-updates
        script: /io/sql/cleanup-add-batch-updates.sql
        online: true
      - name: add-jobs-update-id-index
        script: /io/sql/add-jobs-update-id-index.sql
        online: true
      - name: no-dev-standing-workers-by-default
        script: /io/sql/no_dev_standing_workers_by_default.py
        online: true
      - name: add-commit-updates
        script: /io/sql/add-commit-updates.sql
        online: true
      - name: add-job-regions
        script: /io/sql/add-job-regions.sql
        online: true
      - name: faster-regions-pool-scheduler
        script: /io/sql/faster-regions-pool-scheduler.sql
        online: true
      - name: add-batch-state-index
        script: /io/sql/add-batch-state-index.sql
        online: true
      - name: case-sensitive-billing-project
        script: /io/sql/case-sensitive-billing-project.sql
        online: true
      - name: driver-behavior
        script: /io/sql/driver_behavior.py
        online: true
      - name: cleanup-old-billing-tables
        script: /io/sql/cleanup-old-billing-tables.sql
        online: true
      - name: add-deduped-resource-ids-to-resources
        script: /io/sql/add-deduped-resource-ids-to-resources.sql
        online: true
      - name: create-resource-dedup-mapping
        script: /io/sql/create_resource_dedup_mapping.py
        online: true
      - name: config-pool-min-instances
        script: /io/sql/config-pool-min-instances.sql
        online: true
      - name: add-deduped-resource-ids-to-attempt-resources
        script: /io/sql/add-deduped-resource-ids-to-attempt-resources.sql
        online: true
      - name: setup-deduped-resources-migration
        script: /io/sql/setup-deduped-resources-migration.sql
        online: true
      - name: dedup-attempt-resources
        script: /io/sql/dedup_attempt_resources.py
        online: true
      - name: mitigate-bad-attempt-resources-trigger
        script: /io/sql/mitigate-bad-attempt-resources-trigger.sql
        online: true
      - name: set-test-and-dev-pools-to-16-core-max-3
        script: /io/sql/set-test-and-dev-pools-to-16-core-max-3.py
        online: true
      - name: fix-billing-triggers-deduped-resource-id
        script: /io/sql/fix-billing-triggers-deduped-resource-id.sql
        online: true
      - name: set-test-and-dev-pools-to-max-16-standing-16
        script: /io/sql/set-test-and-dev-pools-to-max-16-standing-16.py
        online: true
      - name: set-test-min-pool-size-to-1
        script: /io/sql/set-test-min-pool-size-to-1.py
        online: true
      - name: dedup-billing-project-users-v2
        script: /io/sql/dedup_billing_project_users_v2.py
        online: true
      - name: list-jobs-extra-indices
        script: /io/sql/list-jobs-extra-indices.sql
        online: true
      - name: dedup-billing-project-users-by-date
        script: /io/sql/dedup_billing_project_users_by_date.py
        online: true
      - name: add-jobs-ready-time
        script: /io/sql/add-jobs-ready-time.sql
        online: true
      - name: add-list-batches-index
        script: /io/sql/add-list-batches-index.sql
        online: true
      - name: add-feature-flags
        script: /io/sql/add-feature-flags.sql
        online: true
      - name: add-oms-agent-flag
        script: /io/sql/add-oms-agent-flag.sql
        online: true
      - name: increase-test-max-idle-time
        script: /io/sql/increase_test_max_idle_time.py
        online: true
      - name: turn_off_oms_agent_test_dev
        script: /io/sql/turn_off_oms_agent_test_dev.py
        online: true
      - name: add-billing-index-token
        script: /io/sql/add-billing-index-token.sql
        online: true
      - name: jobs-after-update-simple.sql
        script: /io/sql/jobs-after-update-simple.sql
        online: true
      - name: turn-on-billing-compaction
        script: /io/sql/turn_on_billing_compaction.py
        online: true
      - name: cleanup-deprecated-functions
        script: /io/sql/cleanup-deprecated-functions.sql
        online: true
      - name: set-test-pools-to-known-parameters
        script: /io/sql/set-test-pools-to-known-parameters.py
        online: true
      - name: add-gcp-support-logs-specs-and-firewall-fees
        script: /io/sql/add-gcp-support-logs-specs-and-firewall-fees.py
        online: true
      - name: add-product-sku
        script: /io/sql/add-product-sku.sql
        online: true
      - name: dedup-agg-batch-resources
        script: /io/sql/dedup_agg_batch_resources.py
        online: true
      - name: setup-job-groups
        script: /io/sql/setup-job-groups.sql
        online: true
      - name: dedup-job-resources
        script: /io/sql/dedup_job_resources.py
        online: true
      - name: fix-job-groups-state-enum
        script: /io/sql/fix-job-groups-state-enum.sql
        online: true
      - name: populate-job-groups
        script: /io/sql/populate_job_groups.py
        online: true
      - name: rename-job-groups-tables
        script: /io/sql/rename-job-groups-tables.sql
        online: false  # this must be offline
      - name: remove-v2-billing-writes
        script: /io/sql/remove-v2-billing-writes.sql
        online: true
      - name: update-ip-fee-resource
        script: /io/sql/update-ip-fee-resource.py
        online: true
      - name: remove-v2-billing-tables
        script: /io/sql/remove-v2-billing-tables.sql
        online: true
      - name: finalize-job-groups
        script: /io/sql/finalize-job-groups.sql
        online: true
      - name: fix-mark-job-complete-deadlocks
        script: /io/sql/fix-mark-job-complete-deadlocks.sql
        online: true
    inputs:
      - from: /repo/batch/sql
        to: /io/sql
    namespace:
      valueFrom: default_ns.name
    shutdowns:
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch
      - kind: Deployment
        namespace:
          valueFrom: default_ns.name
        name: batch-driver
    dependsOn:
      - default_ns
      - merge_code
      - delete_batch_tables
      - ci_utils_image
      - create_test_database_server_config
      - deploy_test_db
  - kind: deploy
    name: deploy_batch
    namespace:
      valueFrom: default_ns.name
    config: batch/deployment.yaml
    wait:
      - kind: Service
        name: batch
        for: alive
      - kind: Service
        name: batch-driver
        for: alive
    dependsOn:
      - default_ns
      - deploy_batch_sa
      - create_accounts
      - batch_image
      - batch_worker_image
      - batch_database
      - deploy_auth
      - create_certs
  - kind: runImage
    name: add_developers
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex

      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      {% for user in code.get("developers", []) %}
      {% if user['username'] != 'test-dev' %}
      hailctl auth create-user \
        --developer \
        --hail-identity {{ user["hail_identity"] }} \
        --hail-credentials-secret-name {{ user["username"] }}-gsa-key \
        {{ user["username"] }} {{ user["login_id"] }}
      {% endif %}
      {% endfor %}
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    scopes:
      - dev
      - test
    dependsOn:
      - default_ns
      - hailgenetics_hailtop_image
      - deploy_batch
  - kind: runImage
    name: upload_query_jar
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/batch-gsa-key/key.json

      {% if deploy %}
      HAIL_QUERY_JAR_URL={{ global.query_storage_uri }}
      {% else %}
      HAIL_QUERY_JAR_URL={{ global.test_storage_uri }}/{{ default_ns.name }}
      {% endif %}
      HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/git_version).jar

      python3 -m hailtop.aiotools.copy 'null' '[{"from": "/io/hail.jar", "to": "'${HAIL_QUERY_JAR_URL}'"}]'
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    inputs:
      - from: /hail.jar
        to: /io/hail.jar
      - from: /git_version
        to: /io/git_version
    dependsOn:
      - default_ns
      - deploy_batch
      - hailgenetics_hailtop_image
      - build_hail_jar_and_wheel
      - merge_code
      - create_test_gsa_keys
  - kind: runImage
    name: test_hail_python_service_backend_gcp
    numSplits: 16
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.25'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }}
      export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export GCS_REQUESTER_PAYS_PROJECT=broad-ctsa

      export HAIL_SHUFFLE_MAX_BRANCH=4
      export HAIL_SHUFFLE_CUTOFF=1000000
      export HAIL_QUERY_BACKEND=batch
      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=test
      export HAIL_BATCH_REMOTE_TMPDIR={{ global.test_storage_uri }}

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/ \
              --ignore=test/hail/matrixtable/test_file_formats.py \
              -m backend \
              --timeout=600 \
              test
    timeout: 5400
    inputs:
      - from: /wheel
        to: /io/wheel
      - from: /repo/hail/python/pytest.ini
        to: /io/repo/hail/python/pytest.ini
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - create_deploy_config
      - create_accounts
      - hail_run_image
      - upload_query_jar
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_python_service_backend_azure
    numSplits: 16
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.25'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      cd /io/repo/hail/python

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}/{{ token }}
      export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/"
      export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/"
      export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }}
      export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_AZURE_SUBSCRIPTION_ID={{ global.azure_subscription_id }}
      export HAIL_AZURE_RESOURCE_GROUP={{ global.azure_resource_group }}

      export HAIL_SHUFFLE_MAX_BRANCH=4
      export HAIL_SHUFFLE_CUTOFF=1000000
      export HAIL_QUERY_BACKEND=batch
      export HAIL_BATCH_REGIONS={{ global.azure_location }}
      export HAIL_BATCH_BILLING_PROJECT=test
      export HAIL_BATCH_REMOTE_TMPDIR={{ global.test_storage_uri }}

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --ignore=test/hailtop/ \
              --ignore=test/hail/matrixtable/test_file_formats.py \
              -m backend \
              --timeout=600 \
              test
    timeout: 5400
    inputs:
      - from: /wheel
        to: /io/wheel
      - from: /repo/hail/python/pytest.ini
        to: /io/repo/hail/python/pytest.ini
      - from: /repo/hail/python/test
        to: /io/repo/hail/python/test
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - merge_code
      - deploy_batch
      - create_deploy_config
      - create_accounts
      - hail_run_image
      - upload_query_jar
      - upload_test_resources_to_blob_storage
      - build_hail_jar_and_wheel
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
    scopes:
      - deploy
      - dev
    clouds:
      - azure
  - kind: runImage
    name: test_hail_spark_conf_requester_pays_parsing
    image:
      valueFrom: hail_run_image.image
    resources:
      cpu: '0.5'
      preemptible: False
    script: |
      set -ex
      python3 -m pip install --no-dependencies /io/wheel/hail-*-py3-none-any.whl

      export YOU_MAY_OVERWRITE_MY_SPARK_DEFAULTS_CONF_AND_HAILCTL_SETTINGS=1

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              /io/test_requester_pays_parsing.py
    inputs:
      - from: /wheel
        to: /io/wheel
      - from: /repo/hail/scripts/test_requester_pays_parsing.py
        to: /io/test_requester_pays_parsing.py
    dependsOn:
      - default_ns
      - merge_code
      - hail_run_image
      - build_hail_jar_and_wheel
  - kind: buildImage2
    name: netcat_ubuntu_image
    publishAs: netcat
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-apt-get-install netcat
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: volume_image
    publishAs: volume
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        VOLUME ["/data"]
        WORKDIR "/tmp"
        VOLUME ["relative_volume"]
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: gpu_image
    publishAs: gpu
    resources:
      storage: 10Gi
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        RUN hail-pip-install torch torchvision -f https://download.pytorch.org/whl/cu113/torch_stable.html
    dependsOn:
      - hail_ubuntu_image
  - kind: buildImage2
    name: workdir_image
    publishAs: workdir
    dockerFile:
      inline: |
        ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
        FROM $BASE_IMAGE
        WORKDIR ["/work"]
    dependsOn:
      - hail_ubuntu_image
  - kind: runImage
    name: test_batch
    numSplits: 5
    image:
      valueFrom: batch_image.image
    script: |
      set -ex

      # Use the test identity's credentials instead of CI's
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json

      export HAIL_TEST_GSA_KEY_FILE=/test-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json

      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export CI_UTILS_IMAGE={{ ci_utils_image.image }}
      export HAIL_CURL_IMAGE={{ hail_ubuntu_image.image }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_VOLUME_IMAGE={{ volume_image.image }}
      export HAIL_WORKDIR_IMAGE={{ workdir_image.image }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}"
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      export HAIL_TOKEN="{{ token }}"
      export HAIL_CLOUD="{{ global.cloud }}"
      export HAIL_PRODUCTION_DOMAIN="{{ global.domain }}"
      export HAIL_GPU_IMAGE="{{ gpu_image.image }}"
      export HAIL_BATCH_REMOTE_TMPDIR="{{ global.test_storage_uri }}/test_batch/{{ token }}/"

      hail-pip-install -r /io/dev-requirements.txt

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              -k "not test_scale and not test_invariants" \
              --timeout=360 \
              /io/test/
    inputs:
      - from: /repo/batch/test
        to: /io/test
      - from: /repo/hail/python/dev/pinned-requirements.txt
        to: /io/dev-requirements.txt
    port: 5000
    timeout: 1500
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
      - batch_image
      - ci_utils_image
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
      - netcat_ubuntu_image
      - volume_image
      - workdir_image
      - hail_ubuntu_image
      - gpu_image
  - kind: runImage
    name: delete_test_billing_projects
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: batch_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json
      export HAIL_TOKEN="{{ test_batch.token }}"
      cd /io/test
      python3 -c '
      import billing_projects
      import asyncio
      asyncio.get_event_loop().run_until_complete(billing_projects.delete_all_test_billing_projects())
      '
    inputs:
      - from: /repo/batch/test
        to: /io/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    alwaysRun: true
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - batch_image
      - deploy_batch
      - test_batch
  - kind: runImage
    name: create_ci_test_repo
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      REPO_NAME="ci-test-{{token}}"

      echo creating $REPO_NAME...
      curl -XPOST \
        -i \
        -fsSL \
        https://api.github.com/orgs/hail-ci-test/repos \
        -H "Authorization: token ${TOKEN}" \
        -d "{ \"name\" : \"$REPO_NAME\" }"

      # checkout new ci repo
      # Repeated clones should happen in fresh directories so we mv to the final
      # destination on successful clone
      set +x
      clone() {
        tmp_dir=$(mktemp -d) && git clone https://$1@github.com/hail-ci-test/$2.git $tmp_dir && mv $tmp_dir $2
      }
      typeset -fx clone
      retry clone $TOKEN $REPO_NAME
      set -x
      cd $REPO_NAME

      mkdir -p ./ci/test ./hail/
      cp /io/repo/ci/test/resources/build.yaml ./
      cp -R /io/repo/ci/* ./ci/
      cp /io/repo/tls/Dockerfile ./ci/test/resources/Dockerfile.certs
      cp /io/repo/tls/create_certs.py ./ci/test/resources/
      cp /io/repo/tls/create_test_db_config.sh ./ci/test/resources/
      cp /io/repo/pylintrc ./
      cp /io/repo/check-sql.sh ./
      cp /io/repo/pyproject.toml ./
      cp -R /io/repo/docker ./
      cp -R /io/repo/gear ./
      cp -R /io/repo/hail/python ./hail/
      cp /io/repo/hail/Makefile ./hail/
      cp /io/repo/hail/env_var.mk ./hail/
      cp /io/repo/hail/version.mk ./hail/
      cp -R /io/repo/web_common ./
      cp /io/repo/hail/python/setup-hailtop.py ./hail/python/

      git config user.name ci
      git config user.email ci@hail.is
      git add * && git commit -m "setup repo"
      retry git push
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    scopes:
      - test
      - dev
    inputs:
      - from: /repo
        to: /io/repo
    dependsOn:
      - default_ns
      - ci_utils_image
      - merge_code
  - kind: runImage
    name: create_ci_config
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      kubectl -n {{ default_ns.name }} create secret generic ci-config \
          --from-literal=github_context="ci-test" \
          --from-literal=storage_uri="{{ global.test_storage_uri }}" \
          --from-literal=deploy_steps="[]" \
          --from-literal=watched_branches="[[\"hail-ci-test/ci-test-{{create_ci_test_repo.token}}:main\", true, true]]" \
          --save-config --dry-run=client -o yaml \
        | kubectl -n {{ default_ns.name }} apply -f -
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - create_ci_test_repo
  - kind: deploy
    name: deploy_ci
    namespace:
      valueFrom: default_ns.name
    config: ci/deployment.yaml
    wait:
      - kind: Service
        name: ci
        for: alive
    dependsOn:
      - default_ns
      - ci_image
      - ci_utils_image
      - create_accounts
      - ci_database
      - deploy_auth
      - deploy_batch
      - create_ci_config
      - deploy_ci_agent
      - create_certs
      - hail_buildkit_image
  - kind: runImage
    name: test_ci
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export ORGANIZATION=hail-ci-test
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export REPO_NAME=ci-test-"{{ create_ci_test_repo.token }}"
      export NAMESPACE="{{ default_ns.name }}"
      pip install /io/ci
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              /io/ci/test
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    timeout: 5400
    inputs:
      - from: /repo/ci
        to: /io/ci
    scopes:
      - test
      - dev
    dependsOn:
      - merge_code
      - create_deploy_config
      - create_accounts
      - default_ns
      - create_certs
      - deploy_ci
      - hail_dev_image
      - create_ci_test_repo
  - kind: runImage
    name: test_hailtop_batch
    resources:
      memory: standard
      cpu: '0.25'
    numSplits: 5
    image:
      valueFrom: hail_dev_image.image
    script: |
      cd /io/hailtop
      set -ex
      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"
      export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}"

      {% if global.cloud == 'gcp' %}
      export GCS_REQUESTER_PAYS_PROJECT={{ global.gcp_project }}
      {% endif %}

      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/test_hailtop_batch/{{ token }}/

      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --timeout=360 \
              /io/test/hailtop/batch/
    inputs:
      - from: /repo/hail/python/pytest.ini
        to: /io/pytest.ini
      - from: /repo/hail/python/test
        to: /io/test
      - from: /repo/hail/python/hailtop
        to: /io/hailtop
    timeout: 1200
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - hail_dev_image
      - deploy_batch
  - kind: runImage
    name: test_hailctl_batch
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hailgenetics_hail_image.image
    script: |
      set -ex

      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}"

      {% if global.cloud == "gcp" %}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      {% elif global.cloud == "azure" %}
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}


      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/hailctl-test/{{ token }}

      mkdir -p foo
      echo "bar" > foo/baz.txt

      cat >simple_hail.py <<EOF
      import hail as hl

      with open('foo/baz.txt') as f:
          print(f.read())

      hl.init(app_name='test-hailctl-batch-submit-query')
      hl.utils.range_table(10).collect()
      EOF

      BATCH_ID=$(hailctl batch submit simple_hail.py --name=test-hailctl-batch-submit --files=foo -o json | jq '.id')
      STATUS=$(hailctl batch wait -o json $BATCH_ID)
      STATE=$(echo $STATUS | jq -jr '.state')
      if [ "$STATE" == "success" ]; then
          exit 0;
      else
          echo $STATUS;
          exit 1;
      fi

      cat >hail_with_args.py <<EOF
      import hail as hl
      import sys

      with open('foo/baz.txt') as f:
          print(f.read())

      hl.init(app_name='test-hailctl-batch-submit-query')
      assert hl.utils.range_table(int(sys.argv[1]))._force_count() == 100
      EOF

      BATCH_ID=$(hailctl batch submit --name=test-hailctl-batch-submit --files=foo -o json hail_with_args.py 100 | jq '.id')
      STATUS=$(hailctl batch wait -o json $BATCH_ID)
      STATE=$(echo $STATUS | jq -jr '.state')
      if [ "$STATE" == "success" ]; then
          exit 0;
      else
          echo $STATUS;
          exit 1;
      fi

      cat >file.sh <<EOF
      set -ex

      cat foo
      echo "Hello World!"
      EOF

      BATCH_ID=$(hailctl batch submit --name=test-hailctl-batch-submit --files=foo -o json file.sh | jq '.id')
      STATUS=$(hailctl batch wait -o json $BATCH_ID)
      STATE=$(echo $STATUS | jq -jr '.state')
      if [ "$STATE" == "success" ]; then
          exit 0;
      else
          echo $STATUS;
          exit 1;
      fi

      cat >file-with-args.sh <<EOF
      set -ex

      [[ $# -eq 2 ]]

      cat foo
      echo "Hello World! $1 $2"
      EOF

      BATCH_ID=$(hailctl batch submit --name=test-hailctl-batch-submit --files=foo -o json file-with-args.sh abc 123 | jq '.id')
      STATUS=$(hailctl batch wait -o json $BATCH_ID)
      STATE=$(echo $STATUS | jq -jr '.state')
      if [ "$STATE" == "success" ]; then
          exit 0;
      else
          echo $STATUS;
          exit 1;
      fi
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - hailgenetics_hail_image
      - upload_query_jar
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - deploy_batch
  - kind: runImage
    name: test_batch_docs
    image:
      valueFrom: hail_dev_image.image
    script: |
      set -ex
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      hailctl config set batch/billing_project test
      hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/test_batch_docs/{{ token }}/
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --doctest-modules \
              --doctest-glob='*.rst' \
              --doctest-continue-on-failure \
              --ignore=docs/cookbook/files/ \
              --timeout=120 \
              --ignore=docs/conf.py \
              /io/hailtop/batch
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - hail_dev_image
      - merge_code
      - deploy_batch
    timeout: 1200
    inputs:
      - from: /repo/hail/python/hailtop
        to: /io/hailtop
  - kind: runImage
    name: cleanup_ci_test_repo
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: hail_ubuntu_image.image
    script: |
      set -e
      TOKEN=$(cat /secret/ci-secrets/user1)
      echo deleting ci-test-{{ create_ci_test_repo.token }}...
      curl -XDELETE \
        -ifsSL \
        https://api.github.com/repos/hail-ci-test/ci-test-{{ create_ci_test_repo.token }} \
        -H "Authorization: token ${TOKEN}"
    secrets:
      - name: hail-ci-0-1-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /secret/ci-secrets
    alwaysRun: true
    scopes:
      - test
      - dev
    dependsOn:
      - default_ns
      - hail_ubuntu_image
      - create_ci_test_repo
      - deploy_ci
      - test_ci
  - kind: buildImage2
    name: blog_nginx_image
    dockerFile: /io/blog/Dockerfile.nginx
    contextPath: /io/blog
    publishAs: blog_nginx
    inputs:
      - from: /repo/blog
        to: /io/blog
    dependsOn:
      - hail_ubuntu_image
      - merge_code
  - kind: deploy
    name: deploy_blog
    namespace:
      valueFrom: default_ns.name
    config: blog/deployment.yaml
    wait:
      - kind: Service
        name: blog
        for: alive
        resource_type: statefulset
    dependsOn:
      - default_ns
      - blog_nginx_image
      - create_certs
  - kind: runImage
    name: test_dataproc-37
    image:
      valueFrom: ci_utils_image.image
    resources:
      preemptible: False
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          exit 0
      fi

      cd hail
      time retry sh millw --version
      make test-dataproc-37 DEV_CLARIFIER=ci_test_dataproc-37 HAIL_RELEASE_MODE=1
    dependsOn:
      - ci_utils_image
      - default_ns
      - merge_code
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /repo
        to: /io/repo
    secrets:
      - name: test-dataproc-service-account-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dataproc-service-account-key
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
  - kind: runImage
    name: test_dataproc-38
    image:
      valueFrom: ci_utils_image.image
    resources:
      preemptible: False
    script: |
      set -ex

      cd /io/repo

      gcloud auth activate-service-account --key-file=/test-dataproc-service-account-key/test-dataproc-service-account-key.json
      gcloud config set project hail-vdc
      gcloud config set dataproc/region us-central1

      if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version)
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          exit 0
      fi

      cd hail
      time retry sh millw --version
      make test-dataproc-38 DEV_CLARIFIER=ci_test_dataproc-38 HAIL_RELEASE_MODE=1
    dependsOn:
      - ci_utils_image
      - default_ns
      - merge_code
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /repo
        to: /io/repo
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
  - kind: runImage
    name: release
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      cd /io

      gcloud auth activate-service-account --key-file=/ci-deploy-0-1--hail-is-hail/ci-deploy-0-1--hail-is-hail.json

      gcloud auth -q configure-docker {{ global.docker_prefix.split('/')[0] }}
      cat /docker-hub-hailgenetics/password | skopeo login --username hailgenetics --password-stdin docker.io

      cp /pypi-credentials/pypirc $HOME/.pypirc
      printf 'Authorization: token ' > github-oauth
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth
      printf '#!/bin/bash\necho ' > git-askpass
      cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass
      chmod 755 git-askpass
      export GIT_ASKPASS=/io/git-askpass

      cd /io/repo/hail
      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)

      if git ls-remote --exit-code --tags origin $HAIL_PIP_VERSION
      then
          echo "tag $HAIL_PIP_VERSION already exists"
          echo '0' > /io/release-hail-flag
          exit 0
      fi

      # #14452 - must build wheel with release `hailtop/hailctl/deploy.yaml`
      # use `assume-old` to prevent `mill` from rebuilding the "SHADOW_JAR"
      make upload-artifacts DEPLOY_REMOTE=origin \
        --assume-old=out/assembly.dest/out.jar

      echo Setting arguments and invoking release.sh.

      HAIL_VERSION=$(cat /io/hail_version) \
      WHEEL="build/deploy/dist/hail-${HAIL_PIP_VERSION}-py3-none-any.whl" \
      GIT_VERSION=$(cat /io/git_version) \
      REMOTE=origin \
      GITHUB_OAUTH_HEADER_FILE=/io/github-oauth \
      HAIL_GENETICS_HAIL_IMAGE=docker://{{ hailgenetics_hail_image.image }} \
      HAIL_GENETICS_HAIL_IMAGE_PY_3_10=docker://{{ hailgenetics_hail_image_python_3_10.image }} \
      HAIL_GENETICS_HAIL_IMAGE_PY_3_11=docker://{{ hailgenetics_hail_image_python_3_11.image }} \
      HAIL_GENETICS_HAILTOP_IMAGE=docker://{{ hailgenetics_hailtop_image.image }} \
      HAIL_GENETICS_VEP_GRCH37_85_IMAGE=docker://{{ hailgenetics_vep_grch37_85_image.image }} \
      HAIL_GENETICS_VEP_GRCH38_95_IMAGE=docker://{{ hailgenetics_vep_grch38_95_image.image }} \
      AZURE_WHEEL=/io/azure-wheel/hail-*-py3-none-any.whl \
      WEBSITE_TAR=/io/www.tar.gz \
      bash scripts/release.sh

      echo '1' > /io/release-hail-flag
    inputs:
      - from: /hail_version
        to: /io/hail_version
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /git_version
        to: /io/git_version
      - from: /repo
        to: /io/repo
      - from: /hail.jar
        to: /io/repo/hail/out/assembly.dest/out.jar
      - from: /azure-wheel
        to: /io/azure-wheel
      - from: /www.tar.gz
        to: /io/www.tar.gz
    outputs:
      - from: /io/release-hail-flag
        to: /release-hail-flag
    secrets:
      - name: pypi-credentials
        namespace:
          valueFrom: default_ns.name
        mountPath: /pypi-credentials
      - name: ci-deploy-0-1--hail-is-hail
        namespace:
          valueFrom: default_ns.name
        mountPath: /ci-deploy-0-1--hail-is-hail
      - name: docker-hub-hailgenetics
        namespace:
          valueFrom: default_ns.name
        mountPath: /docker-hub-hailgenetics
      - name: hail-ci-0-1-github-oauth-token
        namespace:
          valueFrom: default_ns.name
        mountPath: /hail-ci-0-1-github-oauth-token
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - merge_code
      - hailgenetics_hail_image
      - hailgenetics_hail_image_python_3_10
      - hailgenetics_hail_image_python_3_11
      - hailgenetics_hailtop_image
      - hailgenetics_vep_grch37_85_image
      - hailgenetics_vep_grch38_95_image
      - build_hail_jar_and_wheel
      - build_azure_wheel
      - test_hail_java
      - test_hail_python_local_backend
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - test_hail_scala_fs
      - test_hail_services_java
      - test_hail_spark_conf_requester_pays_parsing
      - test_dataproc-37
      - test_dataproc-38
      - make_docs
    clouds:
      - gcp
  - kind: runImage
    name: deploy_wheel
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      gcloud auth activate-service-account --key-file=/gsa-key/key.json
      HAIL_WHEEL_FILE=hail-$(cat /io/hail_pip_version)-py3-none-any.whl
      gcloud storage cp "/io/wheel/$HAIL_WHEEL_FILE" "gs://cpg-hail-ci/wheels/$HAIL_WHEEL_FILE"
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /wheel
        to: /io/wheel
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - build_hail_jar_and_wheel
    clouds:
      - gcp
  - kind: runImage
    name: deploy_hailgenetics_image
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      gcloud auth -q configure-docker australia-southeast1-docker.pkg.dev
      skopeo copy docker://{{ hailgenetics_hail_image.image }} docker://australia-southeast1-docker.pkg.dev/hail-295901/hail/hailgenetics/hail:$(cat /io/hail_pip_version)
      skopeo copy docker://{{ hailgenetics_hailtop_image.image }} docker://australia-southeast1-docker.pkg.dev/hail-295901/hail/hailgenetics/hailtop:$(cat /io/hail_pip_version)
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - ci_utils_image
      - hailgenetics_hail_image
      - hailgenetics_hailtop_image
    clouds:
      - gcp
  - kind: runImage
    name: mirror_hailgenetics_images
    image: quay.io/skopeo/stable:v1.13.2
    script: |
      set -ex

      REGISTRY={{ global.docker_prefix.split('/')[0] }}

      set +x
      {% if global.cloud == "gcp" %}
      cat $GOOGLE_APPLICATION_CREDENTIALS | base64 -w 0 | skopeo login -u _json_key_base64 --password-stdin $REGISTRY
      {% elif global.cloud == "azure" %}
      dnf install -y jq
      USERNAME=$(cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.appId')
      cat $AZURE_APPLICATION_CREDENTIALS | jq -jr '.password' | skopeo login -u $USERNAME --password-stdin $REGISTRY
      {% else %}
      echo "unknown cloud {{ global.cloud }}"
      exit 1
      {% endif %}
      set -x

      cd /io/docker/hailgenetics
      export HAIL_PIP_VERSION=$(cat /io/hail_pip_version)
      export DOCKER_PREFIX={{ global.docker_prefix }}
      bash mirror_images.sh
    inputs:
      - from: /hail_pip_version
        to: /io/hail_pip_version
      - from: /repo/docker/hailgenetics/mirror_images.sh
        to: /io/docker/hailgenetics/mirror_images.sh
      - from: /repo/docker/copy_image.sh
        to: /io/docker/copy_image.sh
    scopes:
      - deploy
      - dev
    dependsOn:
      - default_ns
      - merge_code
  - kind: buildImage2
    name: website_image
    dockerFile: /io/repo/website/Dockerfile
    contextPath: /io/repo
    publishAs: website
    inputs:
      - from: /repo/website
        to: /io/repo/website
      - from: /docs.tar.gz
        to: /io/repo/docs.tar.gz
      - from: /repo/hail/python/setup-hailtop.py
        to: /io/repo/hail/python/setup-hailtop.py
      - from: /repo/hail/python/MANIFEST.in
        to: /io/repo/hail/python/MANIFEST.in
      - from: /repo/hail/python/hailtop
        to: /io/repo/hail/python/hailtop
      - from: /hail_version
        to: /io/repo/hail/python/hailtop/hail_version
      - from: /repo/gear
        to: /io/repo/gear
      - from: /repo/web_common
        to: /io/repo/web_common
    dependsOn:
      - hail_ubuntu_image
      - get_pip_versioned_docs
    clouds:
      - gcp
    resources:
      storage: 20Gi
      cpu: "2"
      memory: standard
  - kind: deploy
    name: deploy_website
    namespace:
      valueFrom: default_ns.name
    config: website/deployment.yaml
    wait:
      - kind: Service
        name: www
        for: alive
    dependsOn:
      - default_ns
      - website_image
      - create_certs
      - create_session_key
      - create_deploy_config
    clouds:
      - gcp
  - kind: runImage
    name: test_website
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      set -ex
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      hailctl curl {{ default_ns.name }} www / \
              -vvv \
              -fsSL \
              --retry 3 \
              --retry-delay 5
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    dependsOn:
      - default_ns
      - create_accounts
      - hailgenetics_hailtop_image
      - deploy_website
    clouds:
      - gcp
  - kind: runImage
    name: test_hail_scala_fs
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex
      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json
      export HAIL_FS_TEST_CLOUD_RESOURCES_URI={{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/fs
      export HAIL_TEST_STORAGE_URI={{ global.test_storage_uri }}

      set +e
      java -Xms7500M -Xmx7500M \
           -cp hail-test.jar:$SPARK_HOME/jars/* \
           org.testng.TestNG \
           -listener is.hail.LogTestListener \
           testng-fs.xml
      exit_code=$?
      set -e
      if [[ $exit_code -eq 2 ]]
      then
          echo "some tests were skipped, but exiting success anyway"
          exit 0
      else
          exit $exit_code
      fi
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /hail-test.jar
        to: /io/hail-test.jar
      - from: /repo/hail/testng-fs.xml
        to: /io/testng-fs.xml
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - hail_run_image
      - build_hail_test_artifacts
      - build_hail_jar_and_wheel
      - upload_test_resources_to_blob_storage
  - kind: runImage
    name: test_hail_services_java
    image:
      valueFrom: hail_run_image.image
    resources:
      memory: standard
      cpu: '2'
    script: |
      set -ex

      export HAIL_CLOUD={{ global.cloud }}
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}

      cd /io
      mkdir -p src/test
      tar xzf resources.tar.gz -C src/test
      java -Xms7500M -Xmx7500M \
           -cp hail-test.jar:$SPARK_HOME/jars/* \
           org.testng.TestNG \
           -listener is.hail.LogTestListener \
           testng-services.xml
    inputs:
      - from: /resources.tar.gz
        to: /io/resources.tar.gz
      - from: /hail-test.jar
        to: /io/hail-test.jar
      - from: /repo/hail/testng-services.xml
        to: /io/testng-services.xml
    secrets:
      - name: test-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-gsa-key
    timeout: 1200
    dependsOn:
      - default_ns
      - create_certs
      - create_accounts
      - hail_run_image
      - build_hail_test_artifacts
      - deploy_batch
  - kind: runImage
    name: start_hail_benchmark
    scopes:
      - deploy
      - dev
    clouds:
      - gcp
    dependsOn:
      - default_ns
      - hail_dev_image
      - test_hail_java
      - test_hail_python_local_backend
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - test_hail_scala_fs
      - test_hail_services_java
      - test_hail_spark_conf_requester_pays_parsing
      - release
    image:
      valueFrom: hail_dev_image.image
    inputs:
      - from: /repo/hail/python
        to: /io/repo/hail/python
      - from: /repo/hail/scripts/benchmark_in_batch.py
        to: /io/repo/hail/scripts/benchmark_in_batch.py
      - from: /hail_version
        to: /io/hail_version
      - from: /release-hail-flag
        to: /io/release-hail-flag
    script: |
      set -ex

      # only run benchmarks iff we've published a new release
      if [[ $(< /io/release-hail-flag) != '1' ]]
      then
          echo 'Skipping benchmarks (no new release published).'
          exit 0
      fi

      pushd /io/repo/hail

      BUCKET=hail-benchmarks-2
      HAIL_VERSION=$(cat /io/hail_version)
      RESULTS_PREFIX="gs://${BUCKET}/{{ scope }}/${HAIL_VERSION}"

      export HAIL_BATCH_REGIONS={{ global.gcp_region }}
      export HAIL_BATCH_BILLING_PROJECT=benchmark
      export HAIL_BATCH_REMOTE_TMPDIR={{ ci_storage_uri }}/{{ token }}

      PYTHONPATH="$PWD/python" python3 ./scripts/benchmark_in_batch.py submit \
          {{ hail_dev_image.image }} \
          $RESULTS_PREFIX \
          $HAIL_VERSION

  - kind: runImage
    name: cancel_all_running_test_batches
    image:
      valueFrom: hailgenetics_hailtop_image.image
    script: |
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json

      cat >cancel_all_running_test_batches.py <<'EOF'
      from hailtop.batch_client.aioclient import BatchClient
      import asyncio
      import traceback

      async def cancel_all(query):
          bc = await BatchClient.create('test')
          async for b in bc.list_batches(query):
              status = await b.last_known_status()
              print(status)
              try:
                  await b.cancel()
              except:
                  traceback.print_exc()

      asyncio.get_event_loop().run_until_complete(cancel_all('user:test running'))
      EOF

      python3 cancel_all_running_test_batches.py
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    alwaysRun: true
    timeout: 300
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - hailgenetics_hailtop_image
      - deploy_batch
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - test_hail_services_java
      - test_batch_docs
      - test_hailctl_batch
  - kind: runImage
    name: test_batch_invariants
    image:
      valueFrom: hail_dev_image.image
    script: |
      export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export DOCKER_PREFIX="{{ global.docker_prefix }}"
      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      python3 -m pytest \
              -Werror:::hail -Werror:::hailtop -Werror::ResourceWarning \
              --log-cli-level=INFO \
              -s \
              -r A \
              -vv \
              --instafail \
              --durations=50 \
              --log-date-format="%Y-%m-%dT%H:%M:%S" \
              --log-format="%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)d:%(funcName)s %(message)s" \
              --timeout=120 \
              /io/test/test_invariants.py
    inputs:
      - from: /repo/batch/test
        to: /io/test
    timeout: 300
    secrets:
      - name: test-dev-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /test-dev-gsa-key
    scopes:
      - test
      - dev
    dependsOn:
      - create_deploy_config
      - create_accounts
      - default_ns
      - merge_code
      - hail_dev_image
      - deploy_batch
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - cancel_all_running_test_batches
  - kind: runImage
    name: delete_gcp_batch_instances
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    alwaysRun: true
    script: |
      set -ex
      gcloud -q auth activate-service-account --key-file=/batch-gsa-key/key.json
      set +e
      kubectl -n {{ default_ns.name }} scale deployment batch-driver --replicas=0
      gcloud -q compute instances list \
          --filter 'tags.items=batch2-agent AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -P8 -n2 -r sh -c 'gcloud -q compute instances delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
      gcloud -q compute disks list \
          --filter 'labels.batch=1 AND labels.namespace={{ default_ns.name }}' \
          --format="table[no-heading](zone.basename(), name)" \
          --project {{ global.gcp_project }} \
        | xargs -P8 -n2 -r sh -c 'gcloud -q compute disks delete --zone "$1" --project {{ global.gcp_project }} "$2" || true' argv0
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    scopes:
      - dev
      - test
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    clouds:
      - gcp
    dependsOn:
      - default_ns
      - ci_utils_image
      - test_batch_invariants
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - cancel_all_running_test_batches
  - kind: runImage
    name: delete_azure_batch_instances
    image: mcr.microsoft.com/azure-cli
    alwaysRun: true
    script: |
      set -e -o pipefail
      AZURE_USERNAME=$(jq -r '.appId' /batch-gsa-key/key.json)
      AZURE_PASSWORD=$(jq -r '.password' /batch-gsa-key/key.json)
      AZURE_TENANT_ID=$(jq -r '.tenant' /batch-gsa-key/key.json)
      az login --service-principal -u $AZURE_USERNAME -p $AZURE_PASSWORD --tenant $AZURE_TENANT_ID
      set +e
      set -x
      export AZURE_RESOURCE_GROUP="{{ global.azure_resource_group }}"
      az aks install-cli
      kubectl -n {{ default_ns.name }} scale deployment batch-driver --replicas=0
      az vm list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az vm delete --force-deletion true --resource-group $AZURE_RESOURCE_GROUP --name "$1" --yes || true' argv0
      az network nic list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az network nic delete --resource-group $AZURE_RESOURCE_GROUP --name "$1" || true' argv0
      az network public-ip list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az network public-ip delete --resource-group $AZURE_RESOURCE_GROUP --name "$1" || true' argv0
      az deployment group list --resource-group $AZURE_RESOURCE_GROUP -o tsv \
          --query "[?tags.namespace == '{{ default_ns.name }}'].name" \
        | xargs -n1 -r sh -c 'az deployment delete --name "$1" || true' argv0
    secrets:
      - name: batch-gsa-key
        namespace:
          valueFrom: default_ns.name
        mountPath: /batch-gsa-key
    scopes:
      - dev
      - test
    serviceAccount:
      name: admin
      namespace:
        valueFrom: default_ns.name
    clouds:
      - azure
    dependsOn:
      - default_ns
      - test_batch_invariants
      - test_batch
      - test_ci
      - test_hailtop_batch
      - test_hail_python_service_backend_gcp
      - test_hail_python_service_backend_azure
      - cancel_all_running_test_batches
  - kind: runImage
    name: setup_dev_namespace_autoscaledown
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      {% for user in code.get("developers", []) %}
      {% if user['username'] != 'test-dev' %}

      if kubectl get namespace {{ user["username"] }} 2>/dev/null
      then
      cat > the.yaml <<'EOF'
      apiVersion: batch/v1
      kind: CronJob
      metadata:
        name: dev-namespace-scaledown-{{ user["username"] }}
        namespace: {{ user["username"] }}
      spec:
        schedule: "0 0 * * *"  # Every day at midnight UTC
        concurrencyPolicy: Forbid
        successfulJobsHistoryLimit: 1
        failedJobsHistoryLimit: 1
        jobTemplate:
          spec:
            template:
              spec:
                serviceAccountName: admin
                containers:
                - name: dev-namespace-scaledown-{{ user["username"] }}
                  image: bitnami/kubectl:latest
                  command:
                  - /bin/sh
                  - -c
                  - set -ex ; kubectl scale deployments --all -n {{ user["username"] }} --replicas=0 && kubectl scale statefulsets --all -n {{ user["username"] }} --replicas=0
                restartPolicy: OnFailure
      EOF

      kubectl apply -f the.yaml
      echo "For {{ user["username"] }}, applied:"
      echo "---"
      cat the.yaml
      else
      echo "Skipping {{ user["username"] }} because they have no namespace."
      fi

      {% endif %}
      {% endfor %}
    scopes:
      - deploy
    serviceAccount:
      name: ci-agent
      namespace:
        valueFrom: default_ns.name
    dependsOn:
      - ci_utils_image
      - default_ns
    scopes:
      - deploy
  - kind: runImage
    name: test_gcp_ar_cleanup_policies
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    script: |
      set -ex
      cd /io/repo/
      new_policy=$(mktemp)
      python3 devbin/generate_gcp_ar_cleanup_policy.py > $new_policy
      diff $new_policy infra/gcp-broad/gcp-ar-cleanup-policy.txt || {
        echo '>>> AR cleanup policy has changed, new policy is <<<'
        cat $new_policy
        echo '--------------------------------------'
        echo "Please regenerate the policy file by running:"
        echo "> python3 devbin/generate_gcp_ar_cleanup_policy.py > infra/gcp-broad/gcp-ar-cleanup-policy.txt"
        exit 1
      }
    inputs:
      - from: /repo/build.yaml
        to: /io/repo/build.yaml
      - from: /repo/devbin
        to: /io/repo/devbin
      - from: /repo/infra
        to: /io/repo/infra
      - from: /repo/docker
        to: /io/repo/docker
      - from: /repo/ci/test/resources
        to: /io/repo/ci/test/resources
    clouds:
      - gcp
    dependsOn:
      - ci_utils_image
      - default_ns
  - kind: runImage
    name: set_gcp_ar_cleanup_policies
    resources:
      memory: standard
      cpu: '0.25'
    image:
      valueFrom: ci_utils_image.image
    scopes:
      - deploy
    script: |
      gcloud artifacts repositories set-cleanup-policies hail \
        --project={{ global.gcp_project }} \
        --location=us \
        --policy=/io/repo/infra/gcp-broad/gcp-ar-cleanup-policy.txt \
        --no-dry-run
    inputs:
      - from: /repo/infra
        to: /io/repo/infra
    clouds:
      - gcp
    dependsOn:
      - test_gcp_ar_cleanup_policies
      - ci_utils_image