Migration 01 - CI/CD, Docs and Project Configs

.github/workflows/docker.yaml

@@ -0,0 +1,70 @@
+name: Docker
+on:
+  #  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to use (defaults to "test")'
+        default: 'test'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  VERSION: 1.29.6
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    env:
+      DOCKER_BUILDKIT: 1
+      BUILDKIT_PROGRESS: plain
+      CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+      IMAGE_NAME: cpg_workflows
+      DOCKER_DEV: australia-southeast1-docker.pkg.dev/cpg-common/images-dev
+      DOCKER_MAIN: australia-southeast1-docker.pkg.dev/cpg-common/images
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - id: 'google-cloud-auth'
+        name: 'Authenticate to Google Cloud'
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: 'projects/1051897107465/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
+          service_account: 'gh-images-deployer@cpg-common.iam.gserviceaccount.com'
+
+      - name: set up gcloud sdk
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: cpg-common
+
+      - name: gcloud docker auth
+        run: |
+          gcloud auth configure-docker australia-southeast1-docker.pkg.dev
+
+      - name: build
+        run: |
+          docker build . -f Dockerfile --tag $IMAGE_NAME:${{ github.sha }}
+
+      - name: push latest
+        if: ${{ github.ref_name == 'main' }}
+        run: |
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_MAIN/$IMAGE_NAME:$VERSION
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_MAIN/$IMAGE_NAME:latest
+          docker push $DOCKER_MAIN/$IMAGE_NAME:$VERSION
+          docker push $DOCKER_MAIN/$IMAGE_NAME:latest
+
+      - name: manually triggered build
+        if: ${{ github.event_name == 'workflow_dispatch' && github.ref_name != 'main' }}
+        run: |
+          docker tag $IMAGE_NAME:${{ github.sha }} $DOCKER_DEV/$IMAGE_NAME:${{github.event.inputs.tag}}
+          docker push $DOCKER_DEV/$IMAGE_NAME:${{github.event.inputs.tag}}

.github/workflows/lint.yaml

@@ -1,5 +1,9 @@
 name: Lint
-on: [push]
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   lint:

.github/workflows/package.yaml

@@ -0,0 +1,40 @@
+name: Package
+on:
+  # Building on pull-requests, manual dispatch, and pushes to main; but restricting
+  # publishing only to main pushes and manual dispatch with `if`s in specific steps.
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Build
+        run: python setup.py sdist
+
+      - name: Test install
+        run: pip install dist/*
+
+      - name: Test import
+        run: python -c "import cpg_flow"
+
+      - name: Publish the wheel to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: ${{ github.event_name != 'pull_request' }}
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages_dir: dist/
+          skip_existing: true

DEVELOPERS.md

@@ -0,0 +1,137 @@
+# Requirements
+
+You will need the following software installed to complete this setup:
+
+- Homebrew (if you are on macOS), or an equivalent package manager
+- Python 3.10.x (there have been some issues with 3.11)
+- Java 8 or Java 11 (required for running Hail locally)
+
+Since most of our org use Apple laptops for development, this rest of this guide assumes
+that you are operating in a macOS environment.
+
+## Homebrew
+
+Set up homebrew according to the instructions [here](https://brew.sh/). Before closing
+your terminal, please read any output from the installation script because it may print
+additional instructions on how to complete the setup. Before proceeding, ensure that it
+has installed correctly by running `brew doctor` in your terminal and checking that the
+output does not contain any issues.
+
+## Python
+
+It is recommended that you install Python with pyenv, or some flavour of conda. This
+document will guide you through installing alternative Python versions using a Python
+version manager called [pyenv](https://github.com/pyenv/pyenv). You can skip this section
+if you are using [conda](https://docs.conda.io/en/latest/) to manage your Python versions
+and virtual environments. However, please make sure to create a new virtual environment
+with **Python 3.10**, for example:
+
+```sh
+conda create --name pipelines python=3.10.15
+```
+
+### Pyenv Users
+
+Take some time to understand how pyenv works by reading the short
+[How It Works](https://github.com/pyenv/pyenv#how-it-works) section; this understanding
+will help you to debug operating system and path related issues (i.e _'command not
+found'_, or seemingly missing packages) when they occur. When you're ready, please
+follow the setup instructions [here](https://github.com/pyenv/pyenv#installation).
+
+Once you're done, open a new terminal and verify your pyenv installation by entering
+`pyenv --version`. Now install Python 3.10 using
+
+```sh
+pyenv install 3.10.15
+```
+
+You may use a different 3.10 version which you can find by executing the command
+`pyenv install --list` and scrolling to the top of the output. The rest of this guide
+assumes that you are using `3.10.15`, so change any relevant commands to suit your
+version of Python.
+
+## Java
+
+If using homebrew (recommended), install Java 11 using the command:
+
+```sh
+brew install openjdk@11
+```
+
+Follow the instructions that homebrew gives at the end of the installation which will
+put Java into the system path so that other programs can find it. At the time of writing
+this document, the command to do this is:
+
+```sh
+sudo ln -sfn \
+    $HOMEBREW_PREFIX/opt/openjdk@11/libexec/openjdk.jdk \
+    /Library/Java/JavaVirtualMachines/openjdk-11.jdk
+```
+
+**IMPORTANT:** this command might have changed, so it's important to follow the
+instructions that homebrew gives you when running the install command above. Open a new
+terminal and check that everything has installed correctly by running `java --version`.
+You should see 11 as the major version in the output.
+
+## Repository Setup
+
+Open a new terminal and change directory to where your git repositories are stored. Now,
+clone the repository recursively and `cd` into the folder:
+
+```sh
+git clone --recurse-submodules git@github.com:populationgenomics/cpg-flow.git
+cd cpg-flow
+```
+
+### Conda Users
+
+If using `conda`, activate the virtual environment that you created earlier.
+
+### Pyenv Environment
+
+If using `pyenv`, execute `pyenv local 3.10.15` in your terminal. This will create an
+application-specific file called `.python-version` that tells `pyenv` which version of
+Python to use when inside this directory. Verify that you are using the correct version
+of Python by running `python --version` in your terminal. Now create and activate a virtual
+environment called `venv`:
+
+```sh
+python -m venv venv && source venv/bin/activate
+```
+
+Verify that you are using your virtual environment by confirming that the command
+`which python` is pointing to the Python binary in the local `venv` folder that
+was just created.
+
+## Installation
+
+Run the following pip commands:
+
+```sh
+pip install .
+pip install -r requirements/dev.txt
+```
+
+## Visual Studio Code
+
+### Selecting a Python Interpreter
+
+Press `cmd + shift + p` to open the command palette and type in `Select Interpreter`.
+Click the menu item that says `Python: Select Interpreter`. Select the interpreter
+option which has the path `./venv/bin/python`. VSCode might highlight this option by
+placing a star to the left of it, and the text `(Recommended)` to the right. If using
+conda, select the menu item relating to your virtual environment created earlier. See
+Microsoft's documentation for more [details](https://code.visualstudio.com/docs/python/environments#_creating-environments).
+
+### Setting up Testing
+
+Set up the test runner in Visual Studio Code by following Microsoft's
+[instructions](https://code.visualstudio.com/docs/python/testing#_configure-tests). When
+asked, select `pytest` as your testing framework and `test` as your testing directory.
+VSCode should now auto-detect tests and populate the sidebar with all the tests. Press
+the `play` button to run the tests and make sure they're all passing. To debug tests,
+place a breakpoint somewhere in the code, right-click on any test and select the
+`Debug Test` menu item. Please use the interactive debugger to debug the application as
+opposed to print statements as it is much more time efficient; being able to inspect
+variables and step through code in real time is extremely helpful in understanding
+complex applications.

Dockerfile

@@ -0,0 +1,6 @@
+FROM australia-southeast1-docker.pkg.dev/analysis-runner/images/driver:latest
+
+RUN pip install metamist
+COPY README.md .
+COPY cpg_workflows cpg_workflows
+RUN pip install .