Docker Workflow and Python 3.12 (#6)

Co-authored-by: Byron Himes <byro93@live.com> Co-authored-by: TheByronHimes <TheByronHimes@gmail.com>
ghga-de · May 29, 2024 · 67dcc97 · 67dcc97
1 parent f3757fa
commit 67dcc97
Show file tree

Hide file tree

Showing 36 changed files with 2,113 additions and 1,490 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.9-bullseye
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bookworm
 
 ENV PYTHONUNBUFFERED 1
 

diff --git a/.github/workflows/check_config_docs.yaml b/.github/workflows/check_config_docs.yaml
@@ -9,12 +9,12 @@ jobs:
   check-config:
     name: Check config schema and example
     needs: get-changed-services
-    if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }}
+    if: ${{ needs.get-changed-services.outputs.services != '' }}
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }}
+        service: ${{ fromJson(needs.get-changed-services.outputs.services) }}
       fail-fast: false
 
     env:
@@ -27,6 +27,12 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and pyopenssl
         id: pip-update
         run: python -m pip install --upgrade pip && pip install --upgrade pyopenssl

diff --git a/.github/workflows/check_openapi_specs.yaml b/.github/workflows/check_openapi_specs.yaml
@@ -10,12 +10,12 @@ jobs:
   check-openapi-specs:
     name: Check config schema and example
     needs: get-changed-services
-    if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }}
+    if: ${{ needs.get-changed-services.outputs.services != '' }}
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }}
+        service: ${{ fromJson(needs.get-changed-services.outputs.services) }}
       fail-fast: false
 
     env:
@@ -28,6 +28,12 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and install dependencies
         id: install-dependencies
         run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}}

diff --git a/.github/workflows/check_pyproject.yaml b/.github/workflows/check_pyproject.yaml
@@ -13,6 +13,12 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and install dependencies
         id: install-dependencies
         run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt

diff --git a/.github/workflows/check_readmes.yaml b/.github/workflows/check_readmes.yaml
@@ -15,6 +15,12 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and install dependencies
         id: install-dependencies
         run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt
@@ -27,11 +33,11 @@ jobs:
   check-service-readme:
     name: Check README file for ${{matrix.service}}
     needs: get-changed-services
-    if: ${{ needs.get-changed-services.outputs.all-changes != '' }}
+    if: ${{ needs.get-changed-services.outputs.services != '' }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }}
+        service: ${{ fromJson(needs.get-changed-services.outputs.services) }}
       fail-fast: false
     env:
       IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml
@@ -43,6 +49,12 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and install dependencies
         id: install-dependencies
         run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt

diff --git a/.github/workflows/ci_docker.yaml b/.github/workflows/ci_docker.yaml
@@ -0,0 +1,117 @@
+# Copyright 2022-2023 Universität Tübingen, DKFZ and EMBL
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "Push to docker on new commit to main"
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  DOCKERHUB_NAMESPACE: ghga
+  DOCKERHUB_PLATFORMS: "linux/amd64,linux/arm64"
+  TRIVY_SEVERITY: "CRITICAL,HIGH"
+  IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml
+  IRS_CONFIG_YAML: ./services/irs/dev_config.yaml
+  PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml
+
+jobs:
+  get-changed-services:
+    uses: ./.github/workflows/get_affected_services.yaml
+
+  push-to-docker:
+    runs-on: ubuntu-latest
+    if: ${{ needs.get-changed-services.outputs.services != '' }}
+    needs: get-changed-services
+    strategy:
+      matrix:
+        service: ${{ fromJson(needs.get-changed-services.outputs.services) }}
+      fail-fast: false
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install stringcase and typer
+        id: setup-stringcase
+        run: pip install stringcase typer
+
+      - name: Extract service name for ${{ matrix.service }}
+        id: extract-service-name
+        shell: bash
+        run: |
+          echo name=$(python3 scripts/service_name.py ${{ matrix.service }}) >> $GITHUB_OUTPUT
+
+      - name: Extract service version for ${{ matrix.service }}
+        id: extract-service-version
+        shell: bash
+        run: |
+          echo version=$(python3.12 -c "import tomllib; print(tomllib.load(open('services/${{ matrix.service }}/pyproject.toml', 'rb'))['project']['version'])") >> $GITHUB_OUTPUT
+
+      - name: Ensure that tag complies with semantic versioning.
+        uses: matt-usurp/validate-semver@v2
+        with:
+          version: ${{ steps.extract-service-version.outputs.version }}
+
+      - name: Symlink requirement files
+        id: symlink-requirements
+        run: ln -sf lock services/${{ matrix.service }}/lock
+
+      - name: Prepare Dockerfile
+        id: prepare-dockerfile
+        run: cp Dockerfile services/${{ matrix.service }}/ && sed -i "s/\(ENTRYPOINT \)\[\]/\1[\"${{ matrix.service }}\"]/" services/${{ matrix.service }}/Dockerfile
+
+      - uses: docker/setup-qemu-action@v3
+        name: Set up QEMU
+
+      - uses: docker/setup-buildx-action@v3
+        name: Set up Docker Buildx
+
+      - uses: docker/login-action@v3
+        name: Login to DockerHub
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: docker/build-push-action@v5
+        name: Build and push
+        id: docker_build
+        with:
+          platforms: "${{ env.DOCKERHUB_PLATFORMS }}"
+          tags: "${{ env.DOCKERHUB_NAMESPACE }}/${{ steps.extract-service-name.outputs.name }}:${{ steps.extract-service-version.outputs.version }}"
+          context: "services/${{ matrix.service }}"
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "docker.io/${{ env.DOCKERHUB_NAMESPACE }}/${{ steps.extract-service-name.outputs.name }}:${{ steps.extract-service-version.outputs.version }}"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: ${{ env.TRIVY_SEVERITY }}
+
+      - name: Image digest
+        shell: bash
+        run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/workflows/get_affected_services.yaml b/.github/workflows/get_affected_services.yaml
@@ -3,15 +3,12 @@ name: Get services affected by changes
 on:
   workflow_call:
     outputs:
-      all-changes:
-        description: "Services affected by changes for all commits on branch"
-        value: ${{ jobs.get_changed_services_pr.outputs.services }}
-      since-last-commit:
+      services:
         description: "Services affected by changes since last commit"
-        value: ${{ jobs.get_changed_services_commit.outputs.services }}
+        value: ${{ jobs.get-changed-services.outputs.services }}
 
 jobs:
-  get_changed_services_commit:
+  get-changed-services:
     runs-on: ubuntu-latest
     outputs:
       services: ${{ steps.services-changed.outputs.affected }}
@@ -22,29 +19,11 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: Changed Files
-        id: changed-files
-        uses: tj-actions/changed-files@v44
-
-      - name: Install Typer to check changed services
-        id: install-typer
-        run: pip install typer>=0.9.0
-
-      - name: Generate list of changed services
-        id: services-changed
-        run: |
-          echo "affected=$(python3 ./scripts/get_affected_services.py ${{ steps.changed-files.outputs.all_changed_files }} )" >> $GITHUB_OUTPUT
-
-  get_changed_services_pr:
-    runs-on: ubuntu-latest
-    outputs:
-      services: ${{ steps.services-changed.outputs.affected }}
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
         with:
-          fetch-depth: 1
+          python-version: 3.12
 
       - name: Changed Files
         id: changed-files

diff --git a/.github/workflows/tests_on_push.yaml → .github/workflows/run_tests.yaml b/.github/workflows/tests_on_push.yaml → .github/workflows/run_tests.yaml
@@ -1,34 +1,40 @@
-name: Monorepo Push Tests
+name: Run tests on non main branch
 
 on:
   push:
     branches-ignore:
       - main
 
+env:
+  IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml
+  IRS_CONFIG_YAML: ./services/irs/dev_config.yaml
+  PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml
+
 jobs:
   get-changed-services:
     uses: ./.github/workflows/get_affected_services.yaml
 
   test:
     needs: get-changed-services
-    if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }}
+    if: ${{ needs.get-changed-services.outputs.services != '' }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }}
+        service: ${{ fromJson(needs.get-changed-services.outputs.services) }}
       fail-fast: false
 
-    env:
-      IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml
-      IRS_CONFIG_YAML: ./services/irs/dev_config.yaml
-      PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml
-
     steps:
       - name: Checkout ${{matrix.service}}
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
+      - name: Set up Python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
       - name: Update pip and pyopenssl
         id: pip-update
         run: python -m pip install --upgrade pip && pip install --upgrade pyopenssl

diff --git a/.github/workflows/static_code_analysis.yaml b/.github/workflows/static_code_analysis.yaml
@@ -13,11 +13,11 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
+      - name: Set up Python 3.12
         id: setup-python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.1

diff --git a/.github/workflows/tests_on_pr.yaml b/.github/workflows/tests_on_pr.yaml