From 07a5b3dae7eb3fd75b8c361fa56a41d074acebf4 Mon Sep 17 00:00:00 2001 From: "Thomas J. Zajac" Date: Thu, 16 May 2024 12:29:39 +0000 Subject: [PATCH] Reusable build-cache workflow --- .github/workflows/check_config_docs.yaml | 50 ++++++--- .github/workflows/check_openapi_specs.yaml | 51 ++++++--- .github/workflows/check_pyproject.yaml | 35 ++++-- .github/workflows/check_readmes.yaml | 62 +++++++---- .github/workflows/get_affected_services.yaml | 60 ---------- .github/workflows/prepare_build_cache.yaml | 103 ++++++++++++++++++ .github/workflows/static_code_analysis.yaml | 35 ++++-- .github/workflows/tests_on_pr.yaml | 53 +++++---- .github/workflows/tests_on_push.yaml | 53 +++++---- .pyproject_generation/pyproject_template.toml | 2 +- 10 files changed, 326 insertions(+), 178 deletions(-) delete mode 100644 .github/workflows/get_affected_services.yaml create mode 100644 .github/workflows/prepare_build_cache.yaml diff --git a/.github/workflows/check_config_docs.yaml b/.github/workflows/check_config_docs.yaml index 77a6b582..c1f130dc 100644 --- a/.github/workflows/check_config_docs.yaml +++ b/.github/workflows/check_config_docs.yaml @@ -1,35 +1,51 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Check if the config schema and the example are up to date on: push +env: + IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml + IRS_CONFIG_YAML: ./services/irs/dev_config.yaml + PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml + jobs: - get-changed-services: - uses: ./.github/workflows/get_affected_services.yaml + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} check-config: name: Check config schema and example - needs: get-changed-services - if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }} + needs: build-cache + if: ${{ needs.build-cache.outputs.services != '' }} runs-on: ubuntu-latest strategy: matrix: - service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }} + service: ${{ fromJson(needs.build-cache.outputs.services) }} fail-fast: false - env: - IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml - IRS_CONFIG_YAML: ./services/irs/dev_config.yaml - PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml - steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Update pip and install dependencies - id: install-dependencies - run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}} + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 + with: + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Check config docs id: check-config-docs diff --git a/.github/workflows/check_openapi_specs.yaml b/.github/workflows/check_openapi_specs.yaml index 7f310091..b8734ea9 100644 --- a/.github/workflows/check_openapi_specs.yaml +++ b/.github/workflows/check_openapi_specs.yaml @@ -1,36 +1,51 @@ -# This file is only needed if your repository uses FastAPI +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Check if OpenAPI spec is up to date on: push +env: + IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml + IRS_CONFIG_YAML: ./services/irs/dev_config.yaml + PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml + jobs: - get-changed-services: - uses: ./.github/workflows/get_affected_services.yaml + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} check-openapi-specs: name: Check config schema and example - needs: get-changed-services - if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }} + needs: build-cache + if: ${{ needs.build-cache.outputs.services != '' }} runs-on: ubuntu-latest strategy: matrix: - service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }} + service: ${{ fromJson(needs.build-cache.outputs.services) }} fail-fast: false - env: - IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml - IRS_CONFIG_YAML: ./services/irs/dev_config.yaml - PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml - steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Update pip and install dependencies - id: install-dependencies - run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}} + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 + with: + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Check openapi.yaml id: check-openapi-docs diff --git a/.github/workflows/check_pyproject.yaml b/.github/workflows/check_pyproject.yaml index e3cb121e..46497eed 100644 --- a/.github/workflows/check_pyproject.yaml +++ b/.github/workflows/check_pyproject.yaml @@ -1,21 +1,40 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Check if pyproject.toml file is up to date on: push jobs: + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} + static-code-analysis: name: Check pyproject file - + needs: build-cache runs-on: ubuntu-latest steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Update pip and install dependencies - id: install-dependencies - run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 + with: + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Check pyproject.toml id: check-pyproject diff --git a/.github/workflows/check_readmes.yaml b/.github/workflows/check_readmes.yaml index 32936fdb..966d33df 100644 --- a/.github/workflows/check_readmes.yaml +++ b/.github/workflows/check_readmes.yaml @@ -1,23 +1,44 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Check if the README files are up to date on: push +env: + IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml + IRS_CONFIG_YAML: ./services/irs/dev_config.yaml + PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml + jobs: - get-changed-services: - uses: ./.github/workflows/get_affected_services.yaml + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} check-repo-readme: name: Check repository README file runs-on: ubuntu-latest steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Update pip and install dependencies - id: install-dependencies - run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 + with: + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Check top level README id: check-main-readme @@ -26,26 +47,21 @@ jobs: check-service-readme: name: Check README file for ${{matrix.service}} - needs: get-changed-services - if: ${{ needs.get-changed-services.outputs.all-changes != '' }} + needs: build-cache + if: ${{ needs.build-cache.outputs.services != '' }} runs-on: ubuntu-latest strategy: matrix: - service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }} + service: ${{ fromJson(needs.build-cache.outputs.services) }} fail-fast: false - env: - IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml - IRS_CONFIG_YAML: ./services/irs/dev_config.yaml - PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Update pip and install dependencies - id: install-dependencies - run: python -m pip install --upgrade pip && pip install --no-deps -r ./lock/requirements-dev.txt + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 + with: + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Check service level READMEs id: check-service-readmes diff --git a/.github/workflows/get_affected_services.yaml b/.github/workflows/get_affected_services.yaml deleted file mode 100644 index aeca418f..00000000 --- a/.github/workflows/get_affected_services.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: Get services affected by changes - -on: - workflow_call: - outputs: - all-changes: - description: "Services affected by changes for all commits on branch" - value: ${{ jobs.get_changed_services_pr.outputs.services }} - since-last-commit: - description: "Services affected by changes since last commit" - value: ${{ jobs.get_changed_services_commit.outputs.services }} - -jobs: - get_changed_services_commit: - runs-on: ubuntu-latest - outputs: - services: ${{ steps.services-changed.outputs.affected }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Changed Files - id: changed-files - uses: tj-actions/changed-files@v44 - - - name: Install Typer to check changed services - id: install-typer - run: pip install typer>=0.9.0 - - - name: Generate list of changed services - id: services-changed - run: | - echo "affected=$(python3 ./scripts/get_affected_services.py ${{ steps.changed-files.outputs.all_changed_files }} )" >> $GITHUB_OUTPUT - - get_changed_services_pr: - runs-on: ubuntu-latest - outputs: - services: ${{ steps.services-changed.outputs.affected }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Changed Files - id: changed-files - uses: tj-actions/changed-files@v44 - - - name: Install Typer to check changed services - id: install-typer - run: pip install typer>=0.9.0 - - - name: Generate list of changed services - id: services-changed - run: | - echo "affected=$(python3 ./scripts/get_affected_services.py ${{ steps.changed-files.outputs.all_changed_files }} )" >> $GITHUB_OUTPUT diff --git a/.github/workflows/prepare_build_cache.yaml b/.github/workflows/prepare_build_cache.yaml new file mode 100644 index 00000000..aa7de8f0 --- /dev/null +++ b/.github/workflows/prepare_build_cache.yaml @@ -0,0 +1,103 @@ +name: Get services affected by changes + +concurrency: + group: ${{ github.sha }}-build-cache + +on: + workflow_call: + inputs: + cache-key: + description: 'Cache key to use for retrieving the names of changed services' + required: true + default: ${{ github.sha }} + type: string + python-version: + description: 'The Python version to use' + required: false + default: '3.9' + type: string + outputs: + services: + description: "Services affected by changes" + value: ${{ jobs.build-cache.outputs.services }} + python-location: + description: "Location of the prebuilt python environment" + value: ${{ env.pythonLocation }} + +jobs: + build-cache: + runs-on: ubuntu-latest + + outputs: + services: ${{ steps.set-output.outputs.services }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Restore cache + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: | + ${{ env.pythonLocation }} + services.txt + key: ${{ inputs.cache-key }} + + - name: Set up Python + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Update pip, pyopenssl and install dependencies + id: pip-update + if: steps.restore-cache.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade pip && pip install --upgrade pyopenssl + pip install --no-deps -r ./lock/requirements-dev.txt + for service_dir in $(find ./services/ -mindepth 1 -maxdepth 1 -type d) + do pip install --no-deps $service_dir + done + + - name: Set output variables on cache hit + id: on-cache-hit + if: steps.restore-cache.outputs.cache-hit == 'true' + run: | + echo "Cache hit, reading services.txt" + CHANGED_SERVICES=$(cat services.txt) + echo "services=$CHANGED_SERVICES" >> $GITHUB_ENV + + - name: Changed Files + id: changed-files + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: tj-actions/changed-files@v44 + + - name: Install Typer to check changed services + id: install-typer + if: steps.restore-cache.outputs.cache-hit != 'true' + run: pip install typer>=0.9.0 + + - name: Generate list of changed services + id: services-changed + if: steps.restore-cache.outputs.cache-hit != 'true' + run: | + CHANGED_SERVICES=$(python3 ./scripts/get_affected_services.py ${{ steps.changed-files.outputs.all_changed_files }} ) + echo "$CHANGED_SERVICES" > services.txt + echo "services=$CHANGED_SERVICES" >> $GITHUB_ENV + + - name: Cache the result + id: cache-results + uses: actions/cache/save@v4 + with: + path: | + ${{ env.pythonLocation }} + services.txt + key: ${{ inputs.cache-key }} + + - name: Set output + id: set-output + run: | + echo "services=${{toJson(env.services)}}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/static_code_analysis.yaml b/.github/workflows/static_code_analysis.yaml index ba6d9fd7..eff0d693 100644 --- a/.github/workflows/static_code_analysis.yaml +++ b/.github/workflows/static_code_analysis.yaml @@ -1,23 +1,40 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Static Code Analysis on: push jobs: + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} + static-code-analysis: name: Static Code Analysis - + needs: build-cache runs-on: ubuntu-latest steps: - - name: Checkout repository - id: checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.9 - id: setup-python - uses: actions/setup-python@v5 + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 with: - python-version: 3.9 + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/tests_on_pr.yaml b/.github/workflows/tests_on_pr.yaml index 4af3a4ed..059d279c 100644 --- a/.github/workflows/tests_on_pr.yaml +++ b/.github/workflows/tests_on_pr.yaml @@ -1,40 +1,51 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Monorepo PR Tests on: pull_request: types: [opened, reopened] +env: + IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml + IRS_CONFIG_YAML: ./services/irs/dev_config.yaml + PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml + jobs: - get-changed-services: - uses: ./.github/workflows/get_affected_services.yaml + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} test: - needs: get-changed-services - if: ${{ needs.get-changed-services.outputs.all-changes != '' }} + needs: build-cache + if: ${{ needs.build-cache.outputs.services != '' }} runs-on: ubuntu-latest strategy: matrix: - service: ${{ fromJson(needs.get-changed-services.outputs.all-changes) }} + service: ${{ fromJson(needs.build-cache.outputs.services) }} fail-fast: false - env: - IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml - IRS_CONFIG_YAML: ./services/irs/dev_config.yaml - PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml - steps: - - name: Checkout ${{matrix.service}} - uses: actions/checkout@v4 + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 with: - fetch-depth: 1 - - - name: Update pip and pyopenssl - id: pip-update - run: python -m pip install --upgrade pip && pip install --upgrade pyopenssl - - - name: Install global and local dependencies for ${{matrix.service}} - id: install-dependencies - run: pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}} + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Run tests for ${{matrix.service}} id: run-tests diff --git a/.github/workflows/tests_on_push.yaml b/.github/workflows/tests_on_push.yaml index 3ca78c15..d4ab9f57 100644 --- a/.github/workflows/tests_on_push.yaml +++ b/.github/workflows/tests_on_push.yaml @@ -1,3 +1,18 @@ +# Copyright 2022-2024 Universität Tübingen, DKFZ and EMBL +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: Monorepo Push Tests on: @@ -5,37 +20,33 @@ on: branches-ignore: - main +env: + IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml + IRS_CONFIG_YAML: ./services/irs/dev_config.yaml + PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml + jobs: - get-changed-services: - uses: ./.github/workflows/get_affected_services.yaml + build-cache: + uses: ./.github/workflows/prepare_build_cache.yaml + with: + cache-key: ${{ github.sha }} test: - needs: get-changed-services - if: ${{ needs.get-changed-services.outputs.since-last-commit != '' }} + needs: build-cache + if: ${{ needs.build-cache.outputs.services != '' }} runs-on: ubuntu-latest strategy: matrix: - service: ${{ fromJson(needs.get-changed-services.outputs.since-last-commit) }} + service: ${{ fromJson(needs.build-cache.outputs.services) }} fail-fast: false - env: - IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml - IRS_CONFIG_YAML: ./services/irs/dev_config.yaml - PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml - steps: - - name: Checkout ${{matrix.service}} - uses: actions/checkout@v4 + - name: Restore cache + id: restore-cache + uses: actions/cache@v4 with: - fetch-depth: 1 - - - name: Update pip and pyopenssl - id: pip-update - run: python -m pip install --upgrade pip && pip install --upgrade pyopenssl - - - name: Install global and local dependencies for ${{matrix.service}} - id: install-dependencies - run: pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}} + path: ${{ needs.build-cache.outputs.python-location }} + key: ${{ github.sha }} - name: Run tests for ${{matrix.service}} id: run-tests diff --git a/.pyproject_generation/pyproject_template.toml b/.pyproject_generation/pyproject_template.toml index 9c9b6828..6eda4783 100644 --- a/.pyproject_generation/pyproject_template.toml +++ b/.pyproject_generation/pyproject_template.toml @@ -72,7 +72,7 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] "scripts/*" = ["PL", "S", "SIM", "D"] -"services/*/tests/*" = ["S", "SIM", "PLR", "B011"] +"services/*/tests_*" = ["S", "SIM", "PLR", "B011"] ".devcontainer/*" = ["S", "SIM", "D"] "examples/*" = ["S", "D"] "__init__.py" = ["D"]