initial commit

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.10-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+RUN apt-get update && \
+    export DEBIAN_FRONTEND=noninteractive && \
+    sudo apt install -y gcc g++ pigz libhdf5-dev vim
+
+USER vscode
+ENV PATH="${PATH}:/home/vscode/.local/bin/:/home/vscode/micromamba/bin"
+ENV ENV_NAME="base"
+ENV MAMBA_ROOT_PREFIX="/home/vscode/conda"
+ENV MAMBA_EXE="/home/vscode/micromamba"
+RUN curl micro.mamba.pm/install.sh > /tmp/micromamba_install.sh && \
+    yes | bash /tmp/micromamba_install.sh
+RUN micromamba shell init -p "$MAMBA_ROOT_PREFIX" -s bash > /dev/null && \
+    echo "micromamba activate \$ENV_NAME" >> ~/.bashrc
+
+COPY .devcontainer/mamba_environment.yaml /tmp/mamba_environment.yaml
+RUN micromamba install -y -f /tmp/mamba_environment.yaml

.devcontainer/devcontainer.json

@@ -0,0 +1,45 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": {
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.8",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": {
+				"python.defaultInterpreterPath": "/home/vscode/conda/bin/python",
+				"python.linting.enabled": true,
+				"python.linting.pylintEnabled": true,
+				"python.formatting.provider": "black",
+				"editor.formatOnSave": true
+			},
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance"
+			]
+		}
+	},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "conda init && poetry config virtualenvs.create false && poetry install",
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"git": "os-provided",
+	}
+}

.devcontainer/mamba_environment.yaml

@@ -0,0 +1,12 @@
+name: base
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  # TODO lock versions?
+  - python=3.10
+  - mamba
+  - fastp
+  - python-annoy
+  - tox
+  - poetry

.dockerignore

@@ -0,0 +1,85 @@
+# Ignore everything
+.github
+cue.mod
+docker-bake.hcl
+docs
+CHANGELOG.md
+pytest-results
+conda-recipe
+
+# Dockerfiles
+*.Dockerfile
+
+# Editors
+.idea
+.vscode
+
+
+# Tools caches
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.tox
+.coverage
+.coverage.*
+.cache
+
+# Misc
+**/*.pyc
+playground/
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+*.mo
+*.pot
+*.log
+local_settings.py
+instance/
+.webassets-cache
+.scrapy
+docs/_build/
+target/
+.ipynb_checkpoints
+.python-version
+celerybeat-schedule
+*.sage.py
+.env
+.venv
+venv/
+ENV/
+.spyderproject
+.spyproject
+.ropeproject
+/site

.editorconfig

@@ -0,0 +1,32 @@
+# http://editorconfig.org
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true
+charset = utf-8
+end_of_line = lf
+
+[*.{yaml,yml}]
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+insert_final_newline = true
+charset = utf-8
+end_of_line = lf
+
+[*.json]
+indent_size = 2
+
+[*.bat]
+indent_style = tab
+end_of_line = crlf
+
+[LICENSE]
+insert_final_newline = false
+
+[Makefile]
+indent_style = tab

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+extend-ignore = E501,E402,W503,E203,D213,D203,DOC301
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,cue.mod
+docstring-convention = all
+style = sphinx
+require-return-section-when-returning-none = False
+arg-type-hints-in-signature = True
+arg-type-hints-in-docstring = False

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,60 @@
+<!--
+# Pixelgen Technologies pull request template
+
+Many thanks for contributing to it if you think it could be improved. This is a self-improving process!
+
+Please fill in the appropriate checklist below (delete whatever is not relevant).
+These are the most common things requested on pull requests (PRs).
+
+Remember that PRs should be made against the dev branch, unless you're preparing a pipeline release.
+
+Learn more about contributing: [CONTRIBUTING.md](https://github.com/PixelgenTechnologies/pixelator/blob/main/CONTRIBUTING.md)
+-->
+
+## Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+## How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+**Test Configuration**:
+* Firmware version:
+* Hardware:
+* Toolchain:
+* SDK:
+
+## PR checklist:
+
+- [ ] This comment contains a description of changes (with reason).
+- [ ] My code follows the style guidelines of this project
+- [ ] Make sure your code lints, is well-formatted and type checks
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] If you've added a new stage - have you followed the pipeline CLI conventions in the [USAGE guide](../USAGE.md)
+- [ ] [README.md](./README.md) is updated
+- [ ] If a new tool or package is included, update poetry.lock, [the conda recipe](../conda-recipe/pixelator/meta.yaml) and [cited it properly](../CITATIONS.md)
+- [ ] Include any new [authors/contributors](../AUTHORS.md)
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules
+- [ ] I have checked my code and documentation and corrected any misspellings
+- [ ] Usage Documentation in [USAGE.md](../USAGE.md) is updated
+- [ ] If you are doing a [release](../RELEASING.md#Releasing), or a significant change to the code, update [CHANGELOG.md](../CHANGELOG.md)

.github/actions/build-dev-image/action.yml

@@ -0,0 +1,82 @@
+name: "build-dev-image"
+description: "Build a dev image and push it to the Pixelgen container registries"
+inputs:
+  tag:
+    description: "The tag to use for the image"
+    required: true
+  cache-from:
+    description: "The cache to use for the build"
+    default: ""
+  cache-to:
+    description: "The cache to push to for the build"
+    default: ""
+  platforms:
+    description: "The platforms to build for (seperated by comma) - defaults to linux/amd64/v3"
+    default: linux/amd64/v3
+
+outputs:
+  dev-tag:
+    description: "The tags that were pushed"
+    value: ${{ steps.meta.outputs.tags }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v4
+      with:
+        images: |
+          ghcr.io/pixelgentechnologies/pixelator
+          890888997283.dkr.ecr.eu-north-1.amazonaws.com/pixelator
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=semver,pattern={{version}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=sha
+
+    - name: Build input options
+      id: build_input_options
+      uses: actions/github-script@v4
+      with:
+        script: |
+          const shaShort = context.sha.substring(0, 7)
+          const inputParams = ${{ toJSON(inputs) }}
+          const meta = ${{ steps.meta.outputs.json }}
+          const tags = meta["tags"]
+
+          const cacheFromArray = inputParams["cache-from"].split('\n').map((line) => line.trim())
+            .filter((line) => line !== "")
+            .map((line) => "*.cache-from=" + line)
+
+          const cacheToArray = inputParams["cache-to"].split('\n').map((line) => line.trim())
+            .filter((line) => line !== "")
+            .map((line) => "*.cache-to=" + line)
+
+          const tagsArray = tags.map((tag) => "*.tags=" + tag)
+
+          const platformArray = inputParams["platforms"].split(',').map((line) => line.trim())
+            .filter((line) => line !== "")
+            .map((line) => "*.platform=" + line)
+
+          let bakeSetCommands = []
+          bakeSetCommands.push(...tagsArray)
+          bakeSetCommands.push(...cacheFromArray)
+          bakeSetCommands.push(...cacheToArray)
+          bakeSetCommands.push(...platformArray)
+
+          const gitSHAtag = tagsArray.find((tag) => tag.includes('ghcr') && tag.includes('sha'))
+
+          bakeSetCommandsText = bakeSetCommands.join('\n')
+          core.setOutput('set', bakeSetCommandsText)
+          core.setOutput('dev-tag', gitSHAtag)
+
+
+    - name: Build and push to ghcr and ecr
+      uses: docker/bake-action@v3
+      with:
+        push: true
+        targets: pixelator-prod
+        provenance: false
+        set: ${{ steps.build_input_options.outputs.set }}

.github/actions/setup-image-build-env/action.yml

@@ -0,0 +1,40 @@
+name: "setup-image-build-env"
+description: "Install and setup Docker and authenticate with the Pixelgen container registries"
+inputs:
+  aws-role-to-assume:
+    description: "The AWS IAM role to assume to get access to the AWS ECR registry"
+    default: "arn:aws:iam::890888997283:role/GithubActionsRole"
+  aws-role-session-name:
+    description: "The name of the AWS IAM role session"
+    default: "GitHubActions"
+  aws-registry:
+    description: "The AWS ECR registry to authenticate with"
+    default: "890888997283.dkr.ecr.eu-north-1.amazonaws.com"
+outputs: {}
+runs:
+  using: "composite"
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-region: eu-north-1
+        role-to-assume: ${{ inputs.aws-role-to-assume }}
+        role-session-name: ${{ inputs.aws-session-name }}
+
+    - name: Login to AWS ECR
+      uses: docker/login-action@v2
+      with:
+        registry: ${{ inputs.aws-registry }}
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}