From e1d4514e022ae6ed1de822b2c1721354b61ab8ad Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 18:50:00 +0100 Subject: [PATCH 01/39] Add matrix --- .github/workflows/workflow.yml | 131 +++++++++++++++------------------ setup.py | 2 +- 2 files changed, 61 insertions(+), 72 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index c2d5334e4..25d8f67e7 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -4,7 +4,14 @@ on: [ push, pull_request, workflow_dispatch ] jobs: build: - runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} env: TERM: xterm-256color # To colorize output of make tasks. @@ -15,17 +22,17 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} restore-keys: | # in case of a cache miss (systematically unless the same commit is built repeatedly), the keys below will be used to restore dependencies from previous builds, and the cache will be stored at the end of the job, making up-to-date dependencies available for all jobs of the workflow; see more at https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action - build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - build-${{ env.pythonLocation }}- + build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} + build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}- - name: Build package run: make install-deps install-dist install-test clean build @@ -35,10 +42,17 @@ jobs: uses: actions/cache@v2 with: path: dist - key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} test-core: - runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: test-core-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -50,14 +64,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run openfisca-core tests run: make test-core @@ -66,7 +80,14 @@ jobs: run: coveralls --service=github test-country-template: - runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: test-country-template-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -77,20 +98,27 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Country Template tests run: make test-country test-extension-template: - runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: test-extension-template-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -101,44 +129,27 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Extension Template tests run: make test-extension - check-numpy: - runs-on: ubuntu-20.04 - needs: [ build ] - env: - TERM: xterm-256color - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.7.12 - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Check NumPy typing against latest 3 minor versions - run: for i in {1..3}; do VERSION=$(${GITHUB_WORKSPACE}/.github/get-numpy-version.py prev) && pip install numpy==$VERSION && make check-types; done - lint-files: - runs-on: ubuntu-20.04 + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: lint-files-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -151,14 +162,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run linters run: make lint @@ -175,7 +186,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: 3.7.9 - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" @@ -198,7 +209,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: 3.7.9 - id: stop-early run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. @@ -220,21 +231,21 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.12 + python-version: 3.7.9 - name: Cache build id: restore-build uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Cache release id: restore-release uses: actions/cache@v2 with: path: dist - key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Upload a Python package to PyPi run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD @@ -247,7 +258,7 @@ jobs: curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} publish-to-conda: - runs-on: "ubuntu-20.04" + runs-on: ubuntu-20.04 needs: [ deploy ] strategy: fail-fast: false @@ -256,7 +267,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: 3.7.12 + python-version: 3.7.9 channels: conda-forge activate-environment: true @@ -279,25 +290,3 @@ jobs: - name: Conda build run: conda build -c conda-forge --token ${{ secrets.ANACONDA_TOKEN }} --user openfisca .conda - - test-on-windows: - runs-on: "windows-latest" - needs: [ publish-to-conda ] - - steps: - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: "3.7.9" # 3.7.12 don't exist on GHA Windows https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json - channels: conda-forge - activate-environment: true - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Install with conda - run: conda install -c openfisca openfisca-core - - - name: Test openfisca - run: openfisca --help diff --git a/setup.py b/setup.py index 4d1c1b8ee..76dadc47a 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'importlib-metadata < 4.3.0', # Required for Python 3.7 and Flake8 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', - 'numpy >= 1.11, < 1.21', + 'numpy >= 1.20.0, < 1.21.0', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'sortedcontainers == 2.2.2', From edcf671832d1f919dbaddb458e087bfe448031fb Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 18:58:31 +0100 Subject: [PATCH 02/39] Add Python 3.8 --- .github/workflows/workflow.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 25d8f67e7..2f5e3dad2 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -7,9 +7,14 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, windows-latest] + os: [ubuntu-20.04, ubuntu-22.04, windows-latest] numpy: [1.20.3] - python: [3.7.9] + python: [3.7.9, 3.8.10] + exclude: + - os: ubuntu-20.04 + python: 3.8.10 + - os: ubuntu-22.04 + python: 3.7.9 runs-on: ${{ matrix.os }} name: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} env: From 26c2307699e4043f4c4216672580a7884d1e459e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 18:59:01 +0100 Subject: [PATCH 03/39] Drop support for 3.8 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 76dadc47a..a5976f805 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,6 @@ 'License :: OSI Approved :: GNU Affero General Public License v3', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', From fefe25c1f19bd507b26c2e1bf5b779de8a332a4c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 18:59:52 +0100 Subject: [PATCH 04/39] Fix typo in matrix.os --- .github/workflows/workflow.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 2f5e3dad2..4da0b1b0d 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -16,7 +16,7 @@ jobs: - os: ubuntu-22.04 python: 3.7.9 runs-on: ${{ matrix.os }} - name: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} + name: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} env: TERM: xterm-256color # To colorize output of make tasks. @@ -34,10 +34,10 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} restore-keys: | # in case of a cache miss (systematically unless the same commit is built repeatedly), the keys below will be used to restore dependencies from previous builds, and the cache will be stored at the end of the job, making up-to-date dependencies available for all jobs of the workflow; see more at https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action - build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}- + build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} + build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}- - name: Build package run: make install-deps install-dist install-test clean build @@ -47,7 +47,7 @@ jobs: uses: actions/cache@v2 with: path: dist - key: release-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} test-core: strategy: @@ -57,7 +57,7 @@ jobs: numpy: [1.20.3] python: [3.7.9] runs-on: ${{ matrix.os }} - name: test-core-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} + name: test-core-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -76,7 +76,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run openfisca-core tests run: make test-core @@ -92,7 +92,7 @@ jobs: numpy: [1.20.3] python: [3.7.9] runs-on: ${{ matrix.os }} - name: test-country-template-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} + name: test-country-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -110,7 +110,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Country Template tests run: make test-country @@ -123,7 +123,7 @@ jobs: numpy: [1.20.3] python: [3.7.9] runs-on: ${{ matrix.os }} - name: test-extension-template-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} + name: test-extension-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -141,7 +141,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Extension Template tests run: make test-extension @@ -154,7 +154,7 @@ jobs: numpy: [1.20.3] python: [3.7.9] runs-on: ${{ matrix.os }} - name: lint-files-${{ matrix-os }}-${{ matrix.numpy }}-${{ matrix.python }} + name: lint-files-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] env: TERM: xterm-256color @@ -174,7 +174,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run linters run: make lint @@ -243,14 +243,14 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} - key: build-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Cache release id: restore-release uses: actions/cache@v2 with: path: dist - key: release-${{ matrix-os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Upload a Python package to PyPi run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD From 8448784becdbc024dad1d5228b961cb986d61507 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 19:01:04 +0100 Subject: [PATCH 05/39] Remove leftover check-numpy --- .github/get-numpy-version.py | 38 ---------------------------------- .github/workflows/workflow.yml | 2 +- 2 files changed, 1 insertion(+), 39 deletions(-) delete mode 100755 .github/get-numpy-version.py diff --git a/.github/get-numpy-version.py b/.github/get-numpy-version.py deleted file mode 100755 index 64cb68532..000000000 --- a/.github/get-numpy-version.py +++ /dev/null @@ -1,38 +0,0 @@ -#! /usr/bin/env python - -from __future__ import annotations - -import os -import sys -import typing -from packaging import version -from typing import NoReturn, Union - -import numpy - -if typing.TYPE_CHECKING: - from packaging.version import LegacyVersion, Version - - -def prev() -> NoReturn: - release = _installed().release - - if release is None: - sys.exit(os.EX_DATAERR) - - major, minor, _ = release - - if minor == 0: - sys.exit(os.EX_DATAERR) - - minor -= 1 - print(f"{major}.{minor}.0") # noqa: T001 - sys.exit(os.EX_OK) - - -def _installed() -> Union[LegacyVersion, Version]: - return version.parse(numpy.__version__) - - -if __name__ == "__main__": - globals()[sys.argv[1]]() diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4da0b1b0d..52e01e21e 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -181,7 +181,7 @@ jobs: check-version: runs-on: ubuntu-20.04 - needs: [ test-core, test-country-template, test-extension-template, check-numpy, lint-files ] # Last job to run + needs: [ test-core, test-country-template, test-extension-template, lint-files ] # Last job to run steps: - uses: actions/checkout@v2 From 3c0cdd7e88a174db342eeddbd9060f8942fc78ae Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 19:09:32 +0100 Subject: [PATCH 06/39] Fix matrix def --- .github/workflows/workflow.yml | 80 +++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 52e01e21e..0dd9d9136 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -7,14 +7,18 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, ubuntu-22.04, windows-latest] + os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.7.9, 3.8.10] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - os: ubuntu-20.04 - python: 3.8.10 - - os: ubuntu-22.04 - python: 3.7.9 + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest runs-on: ${{ matrix.os }} name: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} env: @@ -53,9 +57,18 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, windows-latest] + os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.7.9] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] + exclude: + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest runs-on: ${{ matrix.os }} name: test-core-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -88,9 +101,18 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, windows-latest] + os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.7.9] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] + exclude: + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest runs-on: ${{ matrix.os }} name: test-country-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -119,9 +141,18 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, windows-latest] + os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.7.9] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] + exclude: + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest runs-on: ${{ matrix.os }} name: test-extension-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -150,9 +181,18 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04, windows-latest] + os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.7.9] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] + exclude: + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest runs-on: ${{ matrix.os }} name: lint-files-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -180,7 +220,7 @@ jobs: run: make lint check-version: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [ test-core, test-country-template, test-extension-template, lint-files ] # Last job to run steps: @@ -191,7 +231,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.9 + python-version: 3.8.15 - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" @@ -200,7 +240,7 @@ jobs: # We build a separate job to substitute the halt option. # The `deploy` job is dependent on the output of the `check-for-functional-changes`job. check-for-functional-changes: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch needs: [ check-version ] outputs: @@ -214,13 +254,13 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.9 + python-version: 3.8.15 - id: stop-early run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: [ check-for-functional-changes ] if: needs.check-for-functional-changes.outputs.status == 'success' env: @@ -236,7 +276,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7.9 + python-version: 3.8.15 - name: Cache build id: restore-build From 4cb61a4965ba7ff98909945690a93ae27a1d7711 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 19:17:55 +0100 Subject: [PATCH 07/39] Fix doctest in Holder --- openfisca_core/holders/holder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_core/holders/holder.py b/openfisca_core/holders/holder.py index ae7e3fbce..ea5feb66a 100644 --- a/openfisca_core/holders/holder.py +++ b/openfisca_core/holders/holder.py @@ -198,12 +198,12 @@ def set_input( >>> holder = Holder(variable, population) >>> holder.set_input("2018", numpy.array([12.5, 14])) - >>> holder.get_array("2018") - array([12, 14], dtype=int32) + >>> holder.get_array("2018").tolist() + [12, 14] >>> holder.set_input("2018", [12.5, 14]) - >>> holder.get_array("2018") - array([12, 14], dtype=int32) + >>> holder.get_array("2018").tolist() + [12, 14] .. _documentation: https://openfisca.org/doc/coding-the-legislation/35_periods.html#set-input-automatically-process-variable-inputs-defined-for-periods-not-matching-the-definition-period From 57399480c6ab230b7b2e0aa0079dc63b954c004f Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 19:37:39 +0100 Subject: [PATCH 08/39] Fix cache in Windows --- .github/workflows/workflow.yml | 170 ++++++++++++++++++++++----------- 1 file changed, 114 insertions(+), 56 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 0dd9d9136..f00b5a8fb 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -11,14 +11,23 @@ jobs: numpy: [1.20.3] python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + - os: windows-latest + pip: C:\hostedtoolcache\windows\python\ + bin: \x64\scripts\ + lib: \x64\lib\ runs-on: ${{ matrix.os }} name: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} env: @@ -37,11 +46,13 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} restore-keys: | # in case of a cache miss (systematically unless the same commit is built repeatedly), the keys below will be used to restore dependencies from previous builds, and the cache will be stored at the end of the job, making up-to-date dependencies available for all jobs of the workflow; see more at https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action - build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}- + build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }} + build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}- - name: Build package run: make install-deps install-dist install-test clean build @@ -51,7 +62,7 @@ jobs: uses: actions/cache@v2 with: path: dist - key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} test-core: strategy: @@ -61,14 +72,23 @@ jobs: numpy: [1.20.3] python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + - os: windows-latest + pip: C:\hostedtoolcache\windows\python\ + bin: \x64\scripts\ + lib: \x64\lib\ runs-on: ${{ matrix.os }} name: test-core-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -88,8 +108,10 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run openfisca-core tests run: make test-core @@ -105,14 +127,23 @@ jobs: numpy: [1.20.3] python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + - os: windows-latest + pip: C:\hostedtoolcache\windows\python\ + bin: \x64\scripts\ + lib: \x64\lib\ runs-on: ${{ matrix.os }} name: test-country-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -131,8 +162,10 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Country Template tests run: make test-country @@ -145,14 +178,23 @@ jobs: numpy: [1.20.3] python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + - os: windows-latest + pip: C:\hostedtoolcache\windows\python\ + bin: \x64\scripts\ + lib: \x64\lib\ runs-on: ${{ matrix.os }} name: test-extension-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -171,8 +213,10 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run Extension Template tests run: make test-extension @@ -185,14 +229,24 @@ jobs: numpy: [1.20.3] python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + - python: 3.8.15 + os: windows-latest + - python: 3.8.10 + os: ubuntu-latest + - python: 3.7.15 + os: windows-latest + - python: 3.7.9 + os: ubuntu-latest + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + - os: windows-latest + pip: C:\hostedtoolcache\windows\python\ + bin: \x64\scripts\ + lib: \x64\lib\ + runs-on: ${{ matrix.os }} name: lint-files-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -213,8 +267,10 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Run linters run: make lint @@ -282,15 +338,17 @@ jobs: id: restore-build uses: actions/cache@v2 with: - path: ${{ env.pythonLocation }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + path: | + /opt/hostedtoolcache/Python/3.8.15/x64/bin/ + /opt/hostedtoolcache/Python/3.8.15/x64/lib/ + key: build-ubuntu-latest-1.20.3-3.8.15-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Cache release id: restore-release uses: actions/cache@v2 with: path: dist - key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-ubuntu-latest-1.20.3-3.8.15-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Upload a Python package to PyPi run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD From 7e0fede283fa11e41ce4b01b1b4d25752fe17b96 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 20:02:46 +0100 Subject: [PATCH 09/39] Do not test country-package in Windows --- .github/workflows/workflow.yml | 95 ++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f00b5a8fb..cdcf6b870 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -70,12 +70,10 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] + python: [3.8.15, 3.7.15, 3.7.9] exclude: - python: 3.8.15 os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - python: 3.7.15 os: windows-latest - python: 3.7.9 @@ -123,27 +121,14 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] - exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + python: [3.8.15, 3.7.15] include: - os: ubuntu-latest pip: /opt/hostedtoolcache/Python/ bin: /x64/bin/ lib: /x64/lib/ - - os: windows-latest - pip: C:\hostedtoolcache\windows\python\ - bin: \x64\scripts\ - lib: \x64\lib\ runs-on: ${{ matrix.os }} name: test-country-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -174,27 +159,14 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] - exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest + python: [3.8.15, 3.7.15] include: - os: ubuntu-latest pip: /opt/hostedtoolcache/Python/ bin: /x64/bin/ lib: /x64/lib/ - - os: windows-latest - pip: C:\hostedtoolcache\windows\python\ - bin: \x64\scripts\ - lib: \x64\lib\ runs-on: ${{ matrix.os }} name: test-extension-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ build ] @@ -276,7 +248,14 @@ jobs: run: make lint check-version: - runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + numpy: [1.20.3] + python: [3.8.15] + runs-on: ${{ matrix.os }} + name: check-version-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ test-core, test-country-template, test-extension-template, lint-files ] # Last job to run steps: @@ -287,7 +266,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8.15 + python-version: ${{ matrix.python }} - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" @@ -296,7 +275,14 @@ jobs: # We build a separate job to substitute the halt option. # The `deploy` job is dependent on the output of the `check-for-functional-changes`job. check-for-functional-changes: - runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + numpy: [1.20.3] + python: [3.8.15] + runs-on: ${{ matrix.os }} + name: check-for-functional-changes-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch needs: [ check-version ] outputs: @@ -310,13 +296,25 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8.15 + python-version: ${{ matrix.python }} - id: stop-early run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. deploy: - runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + numpy: [1.20.3] + python: [3.8.15] + include: + - os: ubuntu-latest + pip: /opt/hostedtoolcache/Python/ + bin: /x64/bin/ + lib: /x64/lib/ + runs-on: ${{ matrix.os }} + name: deploy-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} needs: [ check-for-functional-changes ] if: needs.check-for-functional-changes.outputs.status == 'success' env: @@ -332,23 +330,23 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8.15 + python-version: ${{ matrix.python }} - name: Cache build id: restore-build uses: actions/cache@v2 with: path: | - /opt/hostedtoolcache/Python/3.8.15/x64/bin/ - /opt/hostedtoolcache/Python/3.8.15/x64/lib/ - key: build-ubuntu-latest-1.20.3-3.8.15-${{ hashFiles('setup.py') }}-${{ github.sha }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} + ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} + key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Cache release id: restore-release uses: actions/cache@v2 with: path: dist - key: release-ubuntu-latest-1.20.3-3.8.15-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - name: Upload a Python package to PyPi run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD @@ -361,16 +359,21 @@ jobs: curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} publish-to-conda: - runs-on: ubuntu-20.04 - needs: [ deploy ] strategy: fail-fast: false + matrix: + os: [ubuntu-20.04] + numpy: [1.20.3] + python: [3.7.9] + runs-on: ${{ matrix.os }} + name: publish-to-conda-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} + needs: [ deploy ] steps: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: 3.7.9 + python-version: ${{ matrix.python }} channels: conda-forge activate-environment: true From eac6784360856965d19f80ef05b3eb1aa47c6041 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 20:23:21 +0100 Subject: [PATCH 10/39] Skip failing tests on Windows --- .github/workflows/workflow.yml | 4 +++- tests/core/test_yaml.py | 2 ++ tests/web_api/test_parameters.py | 2 ++ tests/web_api/test_variables.py | 3 +++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index cdcf6b870..d25b59bef 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -70,10 +70,12 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] numpy: [1.20.3] - python: [3.8.15, 3.7.15, 3.7.9] + python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] exclude: - python: 3.8.15 os: windows-latest + - python: 3.8.10 + os: ubuntu-latest - python: 3.7.15 os: windows-latest - python: 3.7.9 diff --git a/tests/core/test_yaml.py b/tests/core/test_yaml.py index f63e37ff3..d5a54d886 100644 --- a/tests/core/test_yaml.py +++ b/tests/core/test_yaml.py @@ -1,5 +1,6 @@ import os import subprocess +import sys import pytest import openfisca_extension_template @@ -98,6 +99,7 @@ def test_shell_script_with_reform(): subprocess.check_call(command, stdout = devnull, stderr = devnull) +@pytest.mark.skipif(sys.platform == "win32", reason = "Fails on Windows") def test_shell_script_with_extension(): tests_dir = os.path.join(openfisca_extension_template.__path__[0], 'tests') command = ['openfisca', 'test', tests_dir, '-c', 'openfisca_country_template', '-e', 'openfisca_extension_template'] diff --git a/tests/web_api/test_parameters.py b/tests/web_api/test_parameters.py index 8f65cca9a..bb01ed561 100644 --- a/tests/web_api/test_parameters.py +++ b/tests/web_api/test_parameters.py @@ -2,6 +2,7 @@ import json import pytest import re +import sys # /parameters @@ -43,6 +44,7 @@ def test_legacy_parameter_route(test_client): assert response.status_code == client.OK +@pytest.mark.skipif(sys.platform == "win32", reason = "Fails on Windows") def test_parameter_values(test_client): response = test_client.get('/parameter/taxes/income_tax_rate') parameter = json.loads(response.data) diff --git a/tests/web_api/test_variables.py b/tests/web_api/test_variables.py index 4581608aa..a7f698f6d 100644 --- a/tests/web_api/test_variables.py +++ b/tests/web_api/test_variables.py @@ -2,6 +2,7 @@ import json import pytest import re +import sys def assert_items_equal(x, y): @@ -68,6 +69,7 @@ def test_input_variable_value(expected_values, input_variable_response): check_input_variable_value(*expected_values, input_variable=input_variable) +@pytest.mark.skipif(sys.platform == "win32", reason = "Fails on Windows") def test_input_variable_github_url(test_client): input_variable_response = test_client.get('/variable/income_tax') input_variable = json.loads(input_variable_response.data.decode('utf-8')) @@ -97,6 +99,7 @@ def test_variable_value(expected_values, test_client): check_variable_value(*expected_values, variable=variable) +@pytest.mark.skipif(sys.platform == "win32", reason = "Fails on Windows") def test_variable_formula_github_link(test_client): variable_response = test_client.get('/variable/income_tax') variable = json.loads(variable_response.data.decode('utf-8')) From ebda8aa3540531388e9af2a4fb1947b7c20f8b17 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 21:01:21 +0100 Subject: [PATCH 11/39] Optimise cache --- .github/dependabot.yml | 8 - .github/workflows/_before.yaml | 102 +++++++++ .github/workflows/_lint.yaml | 62 +++++ .github/workflows/_test.yaml | 72 ++++++ .github/workflows/merge.yaml | 185 +++++++++++++++ .github/workflows/push.yaml | 84 +++++++ .github/workflows/workflow.yml | 400 --------------------------------- openfisca_tasks/install.mk | 11 +- openfisca_tasks/publish.mk | 16 +- setup.py | 8 +- 10 files changed, 527 insertions(+), 421 deletions(-) delete mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/_before.yaml create mode 100644 .github/workflows/_lint.yaml create mode 100644 .github/workflows/_test.yaml create mode 100644 .github/workflows/merge.yaml create mode 100644 .github/workflows/push.yaml delete mode 100644 .github/workflows/workflow.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index fcb2acc16..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 -updates: -- package-ecosystem: pip - directory: "/" - schedule: - interval: monthly - labels: - - kind:dependencies diff --git a/.github/workflows/_before.yaml b/.github/workflows/_before.yaml new file mode 100644 index 000000000..d347057bb --- /dev/null +++ b/.github/workflows/_before.yaml @@ -0,0 +1,102 @@ +name: Setup package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + deps: + runs-on: ${{ inputs.os }} + name: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + id: restore-deps + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + restore-keys: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + + - name: Install dependencies + run: | + python -m venv venv + ${{ inputs.activate_command }} + make install-deps install-dist + + build: + runs-on: ${{ inputs.os }} + needs: [ deps ] + name: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v3 + with: + path: venv/**/[Oo]pen[Ff]isca* + key: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + restore-keys: | + build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}- + build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + + - name: Cache release + uses: actions/cache@v3 + with: + path: dist + key: release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Build package + run: | + ${{ inputs.activate_command }} + make install-test clean build diff --git a/.github/workflows/_lint.yaml b/.github/workflows/_lint.yaml new file mode 100644 index 000000000..c618c0281 --- /dev/null +++ b/.github/workflows/_lint.yaml @@ -0,0 +1,62 @@ +name: Lint package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + lint: + runs-on: ${{ inputs.os }} + name: lint-doc-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Lint doc + run: | + ${{ inputs.activate_command }} + make clean compile lint-doc + + - name: Lint styles + run: | + ${{ inputs.activate_command }} + make clean compile lint-style + + - name: Lint typing + run: | + ${{ inputs.activate_command }} + make clean compile lint-typing lint-typing-strict diff --git a/.github/workflows/_test.yaml b/.github/workflows/_test.yaml new file mode 100644 index 000000000..eda7224bb --- /dev/null +++ b/.github/workflows/_test.yaml @@ -0,0 +1,72 @@ +name: Test package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + test: + runs-on: ${{ inputs.os }} + name: test-core-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v3 + with: + path: venv/**/[Oo]pen[Ff]isca* + key: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Run Openfisca Core tests + run: | + ${{ inputs.activate_command }} + make test-core + python -m coveralls --service=github + + - name: Run Country Template tests + if: ${{ startsWith(inputs.os, 'ubuntu') }} + run: | + ${{ inputs.activate_command }} + make test-country + + - name: Run Extension Template tests + if: ${{ startsWith(inputs.os, 'ubuntu') }} + run: | + ${{ inputs.activate_command }} + make test-extension diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml new file mode 100644 index 000000000..edd103504 --- /dev/null +++ b/.github/workflows/merge.yaml @@ -0,0 +1,185 @@ +name: OpenFisca-Core / Deploy package to PyPi & Conda + +on: + push: + branches: [master] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_before.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + test: + needs: [setup] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + lint: + needs: [setup] + strategy: + fail-fast: true + matrix: + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + uses: ./.github/workflows/_lint.yaml + with: + os: ubuntu-20.04 + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: source venv/bin/activate + + # The idea behind these dependencies is we want to give feedback to + # contributors on the version number only after they have passed all tests, + # so they don't have to do it twice after changes happened to the main branch + # during the time they took to fix the tests. + check-version: + runs-on: ubuntu-20.04 + needs: [test, lint] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" + + # GitHub Actions does not have a halt job option, to stop from deploying if + # no functional changes were found. We build a separate job to substitute the + # halt option. The `deploy` job is dependent on the output of the + # `check-for-functional-changes`job. + check-for-functional-changes: + runs-on: ubuntu-20.04 + needs: [check-version] # Last job to run + outputs: + status: ${{ steps.stop-early.outputs.status }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - id: stop-early + run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. + + publish-to-pypi: + runs-on: ubuntu-20.04 + needs: [check-for-functional-changes] + if: needs.check-for-functional-changes.outputs.status == 'success' + env: + PYPI_USERNAME: openfisca-bot + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + CIRCLE_TOKEN: ${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} # Personal API token created in CircleCI to grant full read and write permissions + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Cache deps + uses: actions/cache@v3 + with: + path: venv + key: deps-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v3 + with: + path: venv/**/[oO]pen[fF]isca* + key: build-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Cache release + uses: actions/cache@v3 + with: + path: dist + key: release-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Upload package to PyPi + run: | + source venv/bin/activate + make publish + + - name: Update doc + run: | + curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} + + publish-to-conda: + runs-on: ubuntu-20.04 + needs: [publish-to-pypi] + + steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.7.9 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + channels: conda-forge + activate-environment: true + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Update meta.yaml + run: | + python3 -m pip install requests argparse + # Sleep to allow PyPi to update its API + sleep 60 + python3 .github/get_pypi_info.py -p OpenFisca-Core + + - name: Conda Config + run: | + conda install conda-build anaconda-client + conda info + conda config --set anaconda_upload yes + + - name: Conda build + run: conda build -c conda-forge --token ${{ secrets.ANACONDA_TOKEN }} --user openfisca .conda diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml new file mode 100644 index 000000000..96485aa14 --- /dev/null +++ b/.github/workflows/push.yaml @@ -0,0 +1,84 @@ +name: OpenFisca-Core / Pull request review + +on: + pull_request: + types: [assigned, opened, reopened, synchronize, ready_for_review] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_before.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + test: + needs: [setup] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + lint: + needs: [setup] + strategy: + fail-fast: true + matrix: + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + uses: ./.github/workflows/_lint.yaml + with: + os: ubuntu-20.04 + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: source venv/bin/activate + + # The idea behind these dependencies is we want to give feedback to + # contributors on the version number only after they have passed all tests, + # so they don't have to do it twice after changes happened to the main branch + # during the time they took to fix the tests. + check-version: + runs-on: ubuntu-20.04 + needs: [test, lint] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index d25b59bef..000000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,400 +0,0 @@ -name: OpenFisca-Core - -on: [ push, pull_request, workflow_dispatch ] - -jobs: - build: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest, windows-latest] - numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] - exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - - os: windows-latest - pip: C:\hostedtoolcache\windows\python\ - bin: \x64\scripts\ - lib: \x64\lib\ - runs-on: ${{ matrix.os }} - name: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - env: - TERM: xterm-256color # To colorize output of make tasks. - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - restore-keys: | # in case of a cache miss (systematically unless the same commit is built repeatedly), the keys below will be used to restore dependencies from previous builds, and the cache will be stored at the end of the job, making up-to-date dependencies available for all jobs of the workflow; see more at https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action - build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }} - build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}- - - - name: Build package - run: make install-deps install-dist install-test clean build - - - name: Cache release - id: restore-release - uses: actions/cache@v2 - with: - path: dist - key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - test-core: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest, windows-latest] - numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] - exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - - os: windows-latest - pip: C:\hostedtoolcache\windows\python\ - bin: \x64\scripts\ - lib: \x64\lib\ - runs-on: ${{ matrix.os }} - name: test-core-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ build ] - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TERM: xterm-256color # To colorize output of make tasks. - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Run openfisca-core tests - run: make test-core - - - name: Submit coverage to Coveralls - run: coveralls --service=github - - test-country-template: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - numpy: [1.20.3] - python: [3.8.15, 3.7.15] - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - runs-on: ${{ matrix.os }} - name: test-country-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ build ] - env: - TERM: xterm-256color - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Run Country Template tests - run: make test-country - - test-extension-template: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - numpy: [1.20.3] - python: [3.8.15, 3.7.15] - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - runs-on: ${{ matrix.os }} - name: test-extension-template-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ build ] - env: - TERM: xterm-256color - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Run Extension Template tests - run: make test-extension - - lint-files: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest, windows-latest] - numpy: [1.20.3] - python: [3.8.15, 3.8.10, 3.7.15, 3.7.9] - exclude: - - python: 3.8.15 - os: windows-latest - - python: 3.8.10 - os: ubuntu-latest - - python: 3.7.15 - os: windows-latest - - python: 3.7.9 - os: ubuntu-latest - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - - os: windows-latest - pip: C:\hostedtoolcache\windows\python\ - bin: \x64\scripts\ - lib: \x64\lib\ - - runs-on: ${{ matrix.os }} - name: lint-files-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ build ] - env: - TERM: xterm-256color - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Run linters - run: make lint - - check-version: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - numpy: [1.20.3] - python: [3.8.15] - runs-on: ${{ matrix.os }} - name: check-version-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ test-core, test-country-template, test-extension-template, lint-files ] # Last job to run - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Check version number has been properly updated - run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" - - # GitHub Actions does not have a halt job option, to stop from deploying if no functional changes were found. - # We build a separate job to substitute the halt option. - # The `deploy` job is dependent on the output of the `check-for-functional-changes`job. - check-for-functional-changes: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - numpy: [1.20.3] - python: [3.8.15] - runs-on: ${{ matrix.os }} - name: check-for-functional-changes-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch - needs: [ check-version ] - outputs: - status: ${{ steps.stop-early.outputs.status }} - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - id: stop-early - run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. - - deploy: - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - numpy: [1.20.3] - python: [3.8.15] - include: - - os: ubuntu-latest - pip: /opt/hostedtoolcache/Python/ - bin: /x64/bin/ - lib: /x64/lib/ - runs-on: ${{ matrix.os }} - name: deploy-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ check-for-functional-changes ] - if: needs.check-for-functional-changes.outputs.status == 'success' - env: - PYPI_USERNAME: openfisca-bot - PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - CIRCLE_TOKEN: ${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} # Personal API token created in CircleCI to grant full read and write permissions - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Cache build - id: restore-build - uses: actions/cache@v2 - with: - path: | - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.bin }} - ${{ matrix.pip }}${{ matrix.python }}${{ matrix.lib }} - key: build-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Cache release - id: restore-release - uses: actions/cache@v2 - with: - path: dist - key: release-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Upload a Python package to PyPi - run: twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD - - - name: Publish a git tag - run: "${GITHUB_WORKSPACE}/.github/publish-git-tag.sh" - - - name: Update doc - run: | - curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} - - publish-to-conda: - strategy: - fail-fast: false - matrix: - os: [ubuntu-20.04] - numpy: [1.20.3] - python: [3.7.9] - runs-on: ${{ matrix.os }} - name: publish-to-conda-${{ matrix.os }}-${{ matrix.numpy }}-${{ matrix.python }} - needs: [ deploy ] - - steps: - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python }} - channels: conda-forge - activate-environment: true - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Update meta.yaml - run: | - python3 -m pip install requests argparse - # Sleep to allow PyPi to update its API - sleep 60 - python3 .github/get_pypi_info.py -p OpenFisca-Core - - - name: Conda Config - run: | - conda install conda-build anaconda-client - conda info - conda config --set anaconda_upload yes - - - name: Conda build - run: conda build -c conda-forge --token ${{ secrets.ANACONDA_TOKEN }} --user openfisca .conda diff --git a/openfisca_tasks/install.mk b/openfisca_tasks/install.mk index 0a8c81115..3c8110611 100644 --- a/openfisca_tasks/install.mk +++ b/openfisca_tasks/install.mk @@ -1,20 +1,25 @@ ## Uninstall project's dependencies. uninstall: @$(call print_help,$@:) - @pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs pip uninstall -y + @python -m pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs python -m pip uninstall -y + @$(call print_pass,$@:) ## Install project's overall dependencies install-deps: @$(call print_help,$@:) - @pip install --upgrade pip + @python -m pip install --upgrade pip + @$(call print_pass,$@:) ## Install project's development dependencies. install-edit: @$(call print_help,$@:) - @pip install --upgrade --editable ".[dev]" + @python -m pip install --upgrade --editable ".[dev]" + @$(call print_pass,$@:) ## Delete builds and compiled python files. clean: @$(call print_help,$@:) @ls -d * | grep "build\|dist" | xargs rm -rf + @find . -name "__pycache__" | xargs rm -rf @find . -name "*.pyc" | xargs rm -rf + @$(call print_pass,$@:) diff --git a/openfisca_tasks/publish.mk b/openfisca_tasks/publish.mk index aeeb51141..412122595 100644 --- a/openfisca_tasks/publish.mk +++ b/openfisca_tasks/publish.mk @@ -3,7 +3,7 @@ ## Install project's build dependencies. install-dist: @$(call print_help,$@:) - @pip install .[ci,dev] + @python -m pip install .[ci,dev] @$(call print_pass,$@:) ## Build & install openfisca-core for deployment and publishing. @@ -12,6 +12,16 @@ build: @## of openfisca-core, the same we put in the hands of users and reusers. @$(call print_help,$@:) @python -m build - @pip uninstall --yes openfisca-core - @find dist -name "*.whl" -exec pip install --no-deps {} \; + @python -m pip uninstall --yes openfisca-core + @find dist -name "*.whl" -exec python -m pip install --no-deps {} \; + @$(call print_pass,$@:) + +## Upload to PyPi. +publish: + @$(call print_help,$@:) + @python -m twine upload dist/* \ + --username $PYPI_USERNAME \ + --password $PYPI_PASSWORD + @git tag `python setup.py --version` + @git push --tags # update the repository version @$(call print_pass,$@:) diff --git a/setup.py b/setup.py index a5976f805..7343a093f 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ dev_requirements = [ 'autopep8 >= 1.4.0, < 1.6.0', - 'coverage >= 6.0.2, < 7.0.0', + 'coverage >= 6.2.0, < 7.0.0', 'darglint == 1.8.0', 'flake8 >= 4.0.0, < 4.1.0', 'flake8-bugbear >= 19.3.0, < 20.0.0', @@ -87,12 +87,6 @@ long_description=long_description, long_description_content_type='text/markdown', - data_files = [ - ( - 'share/openfisca/openfisca-core', - ['CHANGELOG.md', 'README.md'], - ), - ], entry_points = { 'console_scripts': [ 'openfisca=openfisca_core.scripts.openfisca_command:main', From 5d99c4f7371fe85cc15faba5f8105157f59fbfc2 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 21:30:24 +0100 Subject: [PATCH 12/39] Fix dependency conflict --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 7343a093f..ff986d081 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,6 @@ 'License :: OSI Approved :: GNU Affero General Public License v3', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Topic :: Scientific/Engineering :: Information Analysis', @@ -84,9 +83,8 @@ license = 'https://www.fsf.org/licensing/licenses/agpl-3.0.html', license_files = ("LICENSE",), url = 'https://github.com/openfisca/openfisca-core', - long_description=long_description, - long_description_content_type='text/markdown', - + long_description = long_description, + long_description_content_type = 'text/markdown', entry_points = { 'console_scripts': [ 'openfisca=openfisca_core.scripts.openfisca_command:main', @@ -102,7 +100,9 @@ 'twine >= 4.0.0, < 5.0.0', 'wheel < 1.0.0', ], - 'tracker': ['openfisca-tracker == 0.4.0'], + 'tracker': [ + 'openfisca-tracker == 0.4.0', + ], }, include_package_data = True, # Will read MANIFEST.in install_requires = general_requirements, From 8a3d63cad22770bdbcdb8303d3f5db29f7eb4eed Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 00:58:42 +0100 Subject: [PATCH 13/39] Reduce cache size --- openfisca_core/parameters/config.py | 14 +++----------- openfisca_core/tools/test_runner.py | 12 +++--------- openfisca_core/warnings/__init__.py | 1 - openfisca_core/warnings/libyaml_warning.py | 5 ----- openfisca_tasks/test_code.mk | 4 ++-- 5 files changed, 8 insertions(+), 28 deletions(-) delete mode 100644 openfisca_core/warnings/libyaml_warning.py diff --git a/openfisca_core/parameters/config.py b/openfisca_core/parameters/config.py index e9a3041ae..420eb910e 100644 --- a/openfisca_core/parameters/config.py +++ b/openfisca_core/parameters/config.py @@ -1,21 +1,13 @@ -import warnings -import os import yaml import typing -from openfisca_core.warnings import LibYAMLWarning try: from yaml import CLoader as Loader + except ImportError: - message = [ - "libyaml is not installed in your environment.", - "This can make OpenFisca slower to start.", - "Once you have installed libyaml, run 'pip uninstall pyyaml && pip install pyyaml --no-cache-dir'", - "so that it is used in your Python environment." + os.linesep - ] - warnings.warn(" ".join(message), LibYAMLWarning) - from yaml import Loader # type: ignore # (see https://github.com/python/mypy/issues/1153#issuecomment-455802270) + # (see https://github.com/python/mypy/issues/1153#issuecomment-455802270) + from yaml import SafeLoader as Loader # type: ignore # 'unit' and 'reference' are only listed here for backward compatibility. # It is now recommended to include them in metadata, until a common consensus emerges. diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 4bdc238ce..551dce94c 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -8,7 +8,6 @@ import sys import textwrap import traceback -import warnings import pytest @@ -16,7 +15,6 @@ from openfisca_core.simulation_builder import SimulationBuilder from openfisca_core.tools import assert_near from openfisca_core.types import TaxBenefitSystem -from openfisca_core.warnings import LibYAMLWarning class Options(TypedDict, total = False): @@ -74,17 +72,13 @@ def build_test(params: Dict[str, Any]) -> Test: def import_yaml(): import yaml + try: from yaml import CLoader as Loader + except ImportError: - message = [ - "libyaml is not installed in your environment.", - "This can make your test suite slower to run. Once you have installed libyaml, ", - "run 'pip uninstall pyyaml && pip install pyyaml --no-cache-dir'", - "so that it is used in your Python environment." - ] - warnings.warn(" ".join(message), LibYAMLWarning) from yaml import SafeLoader as Loader + return yaml, Loader diff --git a/openfisca_core/warnings/__init__.py b/openfisca_core/warnings/__init__.py index 9e450c870..5f6da1aec 100644 --- a/openfisca_core/warnings/__init__.py +++ b/openfisca_core/warnings/__init__.py @@ -21,6 +21,5 @@ # # See: https://www.python.org/dev/peps/pep-0008/#imports -from .libyaml_warning import LibYAMLWarning # noqa: F401 from .memory_warning import MemoryConfigWarning # noqa: F401 from .tempfile_warning import TempfileWarning # noqa: F401 diff --git a/openfisca_core/warnings/libyaml_warning.py b/openfisca_core/warnings/libyaml_warning.py deleted file mode 100644 index 361a1688a..000000000 --- a/openfisca_core/warnings/libyaml_warning.py +++ /dev/null @@ -1,5 +0,0 @@ -class LibYAMLWarning(UserWarning): - """ - Custom warning for LibYAML not installed. - """ - pass diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 63fdd4386..9f540955d 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -10,8 +10,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - @pip install --upgrade --no-dependencies openfisca-country-template - @pip install --upgrade --no-dependencies openfisca-extension-template + @pip install --upgrade --no-deps openfisca-country-template + @pip install --upgrade --no-deps openfisca-extension-template ## Run openfisca-core & country/extension template tests. test-code: test-core test-country test-extension From b93292562ce32b85edfdd91472621af40dca511a Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 02:59:11 +0100 Subject: [PATCH 14/39] Fix lint --- openfisca_tasks/lint.mk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 115c6267b..5e1a6f3d7 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -11,7 +11,7 @@ check-syntax-errors: . ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @flake8 $? + @python -m flake8 $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors in the doc. @@ -29,14 +29,14 @@ lint-doc-%: @## able to integrate documentation improvements progresively. @## @$(call print_help,$(subst $*,%,$@:)) - @flake8 --select=D101,D102,D103,DAR openfisca_core/$* - @pylint openfisca_core/$* + @python -m flake8 --select=D101,D102,D103,DAR openfisca_core/$* + @python -m pylint openfisca_core/$* @$(call print_pass,$@:) ## Run static type checkers for type errors. check-types: @$(call print_help,$@:) - @mypy --package openfisca_core --package openfisca_web_api + @python -m mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) ## Run static type checkers for type errors (strict). @@ -48,7 +48,7 @@ lint-typing-strict: \ ## Run static type checkers for type errors (strict). lint-typing-strict-%: @$(call print_help,$(subst $*,%,$@:)) - @mypy \ + @python -m mypy \ --cache-dir .mypy_cache-openfisca_core.$* \ --implicit-reexport \ --strict \ @@ -58,5 +58,5 @@ lint-typing-strict-%: ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") @$(call print_help,$@:) - @autopep8 $? + @python -m autopep8 $? @$(call print_pass,$@:) From 675c4a6d41c45fee05965f0bb23271137d728130 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 03:05:44 +0100 Subject: [PATCH 15/39] Fix tests --- openfisca_tasks/serve.mk | 2 +- openfisca_tasks/test_code.mk | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/openfisca_tasks/serve.mk b/openfisca_tasks/serve.mk index efad0be6c..ced050b24 100644 --- a/openfisca_tasks/serve.mk +++ b/openfisca_tasks/serve.mk @@ -1,6 +1,6 @@ ## Serve the openfisca Web API. api: - @$(call print_help,$@:) + $(call print_help,$@:) @openfisca serve \ --country-package openfisca_country_template \ --extensions openfisca_extension_template diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 9f540955d..aa4c64c53 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -10,8 +10,9 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - @pip install --upgrade --no-deps openfisca-country-template - @pip install --upgrade --no-deps openfisca-extension-template + python -m pip install --upgrade --no-deps openfisca-country-template + python -m pip install --upgrade --no-deps openfisca-extension-template + @$(call print_pass,$@:) ## Run openfisca-core & country/extension template tests. test-code: test-core test-country test-extension @@ -29,14 +30,14 @@ test-code: test-core test-country test-extension @$(call print_pass,$@:) ## Run openfisca-core tests. -test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") +test-core: $(shell python -m pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") @$(call print_help,$@:) - @pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ + @python -m pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - coverage run -m \ + python -m coverage run -m \ ${openfisca} test $? \ ${openfisca_args} @$(call print_pass,$@:) @@ -45,7 +46,8 @@ test-core: $(shell pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 test-country: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - openfisca test ${python_packages}/openfisca_country_template/tests \ + python -m ${openfisca} test \ + ${python_packages}/openfisca_country_template/tests \ --country-package openfisca_country_template \ ${openfisca_args} @$(call print_pass,$@:) @@ -54,7 +56,8 @@ test-country: test-extension: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - openfisca test ${python_packages}/openfisca_extension_template/tests \ + python -m ${openfisca} test \ + ${python_packages}/openfisca_extension_template/tests \ --country-package openfisca_country_template \ --extensions openfisca_extension_template \ ${openfisca_args} @@ -63,4 +66,5 @@ test-extension: ## Print the coverage report. test-cov: @$(call print_help,$@:) - @coverage report + @python -m coverage report + @$(call print_pass,$@:) From 713ae3940f47382b49e8bc2d366372f5c2744480 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 03:52:58 +0100 Subject: [PATCH 16/39] Collect tests --- openfisca_tasks/test_code.mk | 7 ++++--- setup.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index aa4c64c53..10def3092 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -30,7 +30,7 @@ test-code: test-core test-country test-extension @$(call print_pass,$@:) ## Run openfisca-core tests. -test-core: $(shell python -m pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") +test-core: @$(call print_help,$@:) @python -m pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ @@ -38,8 +38,9 @@ test-core: $(shell python -m pytest --quiet --quiet --collect-only 2> /dev/null openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ python -m coverage run -m \ - ${openfisca} test $? \ - ${openfisca_args} + ${openfisca} test \ + ${openfisca_args} \ + $(shell python -m pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") @$(call print_pass,$@:) ## Run country-template tests. diff --git a/setup.py b/setup.py index ff986d081..8c271435a 100644 --- a/setup.py +++ b/setup.py @@ -106,5 +106,5 @@ }, include_package_data = True, # Will read MANIFEST.in install_requires = general_requirements, - packages = find_packages(exclude=['tests*']), + packages = find_packages(), ) From 4c7b4dfddad7118132817ce17f8d5d6d70fe599a Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 04:59:30 +0100 Subject: [PATCH 17/39] Fix core tests path --- openfisca_tasks/test_code.mk | 22 +++++++++++++++------- setup.py | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 10def3092..f4e3f5ab5 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -4,6 +4,15 @@ openfisca = openfisca_core.scripts.openfisca_command ## The path to the installed packages. python_packages = $(shell python -c "import sysconfig; print(sysconfig.get_paths()[\"purelib\"])") +## The path to the templates' tests. +ifeq ($(OS),Windows_NT) + country_tests = ${python_packages}\openfisca_country_template\tests + extension_tests = ${python_packages}\openfisca_extension_template\tests +else + country_tests = ${python_packages}/openfisca_country_template/tests + extension_tests = ${python_packages}/openfisca_extension_template/tests +endif + ## Run all tasks required for testing. install: install-deps install-edit install-test @@ -30,17 +39,16 @@ test-code: test-core test-country test-extension @$(call print_pass,$@:) ## Run openfisca-core tests. -test-core: +test-core: $(shell git ls-files "*test_*.py") @$(call print_help,$@:) @python -m pytest --quiet --capture=no --xdoctest --xdoctest-verbose=0 \ openfisca_core/commons \ openfisca_core/holders \ openfisca_core/types @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - python -m coverage run -m \ - ${openfisca} test \ - ${openfisca_args} \ - $(shell python -m pytest --quiet --quiet --collect-only 2> /dev/null | cut -f 1 -d ":") + python -m coverage run -m ${openfisca} test \ + $? \ + ${openfisca_args} @$(call print_pass,$@:) ## Run country-template tests. @@ -48,7 +56,7 @@ test-country: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ python -m ${openfisca} test \ - ${python_packages}/openfisca_country_template/tests \ + ${country_tests} \ --country-package openfisca_country_template \ ${openfisca_args} @$(call print_pass,$@:) @@ -58,7 +66,7 @@ test-extension: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ python -m ${openfisca} test \ - ${python_packages}/openfisca_extension_template/tests \ + ${extension_tests} \ --country-package openfisca_country_template \ --extensions openfisca_extension_template \ ${openfisca_args} diff --git a/setup.py b/setup.py index 8c271435a..ff986d081 100644 --- a/setup.py +++ b/setup.py @@ -106,5 +106,5 @@ }, include_package_data = True, # Will read MANIFEST.in install_requires = general_requirements, - packages = find_packages(), + packages = find_packages(exclude=['tests*']), ) From c3489f1be768bf18d9264e5520b4da957897df7c Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 05:04:19 +0100 Subject: [PATCH 18/39] Run template tests on Windows --- openfisca_tasks/test_code.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index f4e3f5ab5..375abbbb1 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -19,8 +19,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - python -m pip install --upgrade --no-deps openfisca-country-template - python -m pip install --upgrade --no-deps openfisca-extension-template + @python -m pip install --upgrade --no-deps openfisca-country-template + @python -m pip install --upgrade --no-deps openfisca-extension-template @$(call print_pass,$@:) ## Run openfisca-core & country/extension template tests. From 050a268dde3a6aad0953b7df2be556d6971f80e7 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 05:54:03 +0100 Subject: [PATCH 19/39] Cache openfisca bin --- openfisca_tasks/test_code.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index 375abbbb1..d85dde29a 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -19,8 +19,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - @python -m pip install --upgrade --no-deps openfisca-country-template - @python -m pip install --upgrade --no-deps openfisca-extension-template + pip install --upgrade --no-deps openfisca-country-template + pip install --upgrade --no-deps openfisca-extension-template @$(call print_pass,$@:) ## Run openfisca-core & country/extension template tests. @@ -55,7 +55,7 @@ test-core: $(shell git ls-files "*test_*.py") test-country: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - python -m ${openfisca} test \ + openfisca test \ ${country_tests} \ --country-package openfisca_country_template \ ${openfisca_args} @@ -65,7 +65,7 @@ test-country: test-extension: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - python -m ${openfisca} test \ + openfisca test \ ${extension_tests} \ --country-package openfisca_country_template \ --extensions openfisca_extension_template \ From 5b333bf5354020a7c67df8a8a1f16524f4f5fe3d Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Fri, 2 Dec 2022 07:54:31 +0100 Subject: [PATCH 20/39] Use virtualenv --- openfisca_tasks/test_code.mk | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/openfisca_tasks/test_code.mk b/openfisca_tasks/test_code.mk index d85dde29a..22a3f47f0 100644 --- a/openfisca_tasks/test_code.mk +++ b/openfisca_tasks/test_code.mk @@ -1,16 +1,11 @@ ## The openfisca command module. openfisca = openfisca_core.scripts.openfisca_command -## The path to the installed packages. -python_packages = $(shell python -c "import sysconfig; print(sysconfig.get_paths()[\"purelib\"])") - ## The path to the templates' tests. ifeq ($(OS),Windows_NT) - country_tests = ${python_packages}\openfisca_country_template\tests - extension_tests = ${python_packages}\openfisca_extension_template\tests + tests = $(shell python -c "import os, $(1); print(repr(os.path.join($(1).__path__[0], 'tests')))") else - country_tests = ${python_packages}/openfisca_country_template/tests - extension_tests = ${python_packages}/openfisca_extension_template/tests + tests = $(shell python -c "import $(1); print($(1).__path__[0])")/tests endif ## Run all tasks required for testing. @@ -19,8 +14,8 @@ install: install-deps install-edit install-test ## Enable regression testing with template repositories. install-test: @$(call print_help,$@:) - pip install --upgrade --no-deps openfisca-country-template - pip install --upgrade --no-deps openfisca-extension-template + @python -m pip install --upgrade --no-deps openfisca-country-template + @python -m pip install --upgrade --no-deps openfisca-extension-template @$(call print_pass,$@:) ## Run openfisca-core & country/extension template tests. @@ -55,8 +50,8 @@ test-core: $(shell git ls-files "*test_*.py") test-country: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - openfisca test \ - ${country_tests} \ + python -m ${openfisca} test \ + $(call tests,"openfisca_country_template") \ --country-package openfisca_country_template \ ${openfisca_args} @$(call print_pass,$@:) @@ -65,8 +60,8 @@ test-country: test-extension: @$(call print_help,$@:) @PYTEST_ADDOPTS="$${PYTEST_ADDOPTS} ${pytest_args}" \ - openfisca test \ - ${extension_tests} \ + python -m ${openfisca} test \ + $(call tests,"openfisca_extension_template") \ --country-package openfisca_country_template \ --extensions openfisca_extension_template \ ${openfisca_args} From 1cc90e8ad1bc43496b7bef3cd2790129553a00da Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 12 Dec 2022 09:36:03 +0000 Subject: [PATCH 21/39] Apply suggestions from code review Co-authored-by: Matti Schneider --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff986d081..1dd619cac 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'importlib-metadata < 4.3.0', # Required for Python 3.7 and Flake8 'nptyping == 1.4.4', 'numexpr >= 2.7.0, <= 3.0', - 'numpy >= 1.20.0, < 1.21.0', + 'numpy >= 1.20, < 1.21', 'psutil >= 5.4.7, < 6.0.0', 'pytest >= 4.4.1, < 6.0.0', # For openfisca test 'sortedcontainers == 2.2.2', From 5809252b25c12bc66fbc91641a0f604653da5074 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 12 Dec 2022 11:01:42 +0100 Subject: [PATCH 22/39] Refactor git tag into Makefile --- .github/publish-git-tag.sh | 4 ---- 1 file changed, 4 deletions(-) delete mode 100755 .github/publish-git-tag.sh diff --git a/.github/publish-git-tag.sh b/.github/publish-git-tag.sh deleted file mode 100755 index 4450357cb..000000000 --- a/.github/publish-git-tag.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env bash - -git tag `python setup.py --version` -git push --tags # update the repository version From 3c9de5c218e0ec73b7a56ab61b63271d4684a9ab Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Mon, 12 Dec 2022 11:08:24 +0100 Subject: [PATCH 23/39] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3974b4908..2d134a2c7 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ This package contains the core features of OpenFisca, which are meant to be used ## Environment -OpenFisca runs on Python 3.7. More recent versions should work but are not tested. +OpenFisca is tested against Python 3.8 and 3.7, yet newer versions should work. -OpenFisca also relies strongly on NumPy. The last four minor versions should work, but only the latest/stable is tested. +OpenFisca also depends on NumPy and is tested against v1.20. ## Installation From 046f7f91fd82a6b43d65326adad0a2910cd789e2 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Tue, 13 Dec 2022 16:52:42 +0700 Subject: [PATCH 24/39] Document dependencies inclusion change --- CHANGELOG.md | 54 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c79d9d0de..f09b27129 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +# 39.0.0 [#1168](https://github.com/openfisca/openfisca-core/pull/1168) + +#### Breaking changes + +- Drop support for NumPy < 1.20 +- Drop support for Python < 3.7 + +#### New features + +- Add support for Python 3.8 +- Add CI for Windows + ### 38.0.3 [#1179](https://github.com/openfisca/openfisca-core/pull/1179) #### Bug fix @@ -207,7 +219,7 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). #### New Features - Introduce `rate_from_bracket_indice` method on `RateTaxScaleLike` class - - Allows for the determination of the tax rate based on the tax bracket indice + - Allows for the determination of the tax rate based on the tax bracket index - Introduce `rate_from_tax_base` method on `RateTaxScaleLike` class - Allows for the determination of the tax rate based on the tax base @@ -430,8 +442,8 @@ _Note: this version has been unpublished due to an issue introduced by NumPy upg #### Bug fix - Repair expansion of axes on a variable given as input - - When expanding axes, the expected behavour is to override any input value for the requested variable and period - - As longs as we passed some input for a variable on a period, it was not being overrode, creating a NumPy's error (boradcasting) + - When expanding axes, the expected behaviour is to override any input value for the requested variable and period + - As longs as we passed some input for a variable on a period, it was not being overrode, creating a NumPy's error (broadcasting) - By additionally checking that an input was given, now we make that the array has the correct shape by constructing it with NumPy's tile with a shape equal to the number of the axis expansion count requested. ### 35.3.2 [#992](https://github.com/openfisca/openfisca-core/pull/992) @@ -499,7 +511,7 @@ _Note: this version has been unpublished due to an issue introduced by NumPy upg #### Technical changes -- Improve error message when laoding parameters file to detect the problematic file +- Improve error message when loading parameters file to detect the problematic file ### 35.0.3 [#961](https://github.com/openfisca/openfisca-core/pull/961) @@ -513,7 +525,7 @@ _Note: this version has been unpublished due to an issue introduced by NumPy upg #### Technical changes -- Update dependency: `flask-cors` (`Flask` extension for Cross Origin Resouce Sharing) +- Update dependency: `flask-cors` (`Flask` extension for Cross Origin Resource Sharing) ### 35.0.1 [#968](https://github.com/openfisca/openfisca-core/pull/968) @@ -773,7 +785,7 @@ _Note: this version has been unpublished due to an issue introduced by it. Pleas ### 34.4.4 [#908](https://github.com/openfisca/openfisca-core/pull/908) -- Make parameter cloning return clones that are truly independant from their source +- Make parameter cloning return clones that are truly independent from their source - Before this PR, editing the clone of a parameter tree would change the initial tree - Only impacts reforms that edit parameters tree @@ -2135,7 +2147,7 @@ sum_salary = person.household.sum(salaries) # The sum of the salaries of the pe >>> [4000, 4000, 4000, 4000] # Has the dimension of persons (4) ``` -This is a breaking change, as all the adaptations (such as [this one](https://github.com/openfisca/openfisca-france/blob/18.11.0/openfisca_france/model/prestations/minima_sociaux/rsa.py#L375-L376)) used to overcome these inconsistensies must be removed. +This is a breaking change, as all the adaptations (such as [this one](https://github.com/openfisca/openfisca-france/blob/18.11.0/openfisca_france/model/prestations/minima_sociaux/rsa.py#L375-L376)) used to overcome these inconsistencies must be removed. ## 21.5.0 [#621](https://github.com/openfisca/openfisca-core/pull/621) @@ -2336,7 +2348,7 @@ class housing_occupancy_status(Variable): - When using the Python API (`set_input`), the three following inputs are accepted: - The enum item (e.g. HousingOccupancyStatus.tenant) - The enum string identifier (e.g. "tenant") - - The enum item index, though this is not recommanded. + - The enum item index, though this is not recommended. - If you rely on index, make sure to specify an `__order__` attribute to all your enums to make sure each intem has the right index. See the enum34 [doc](https://pypi.python.org/pypi/enum34/1.1.1). > Example: @@ -2840,8 +2852,8 @@ For more information, check the [documentation](https://openfisca.org/doc/coding #### Technical changes * Refactor the internal representation and the interface of legislation parameters - - The parameters of a legislation are wraped into the classes `Node`, `Parameter`, `Scale`, `Bracket`, `ValuesHistory`, `ValueAtInstant` instead of bare python dict. - - The parameters of a legislation at a given instant are wraped into the classes `NodeAtInstant`, `ValueAtInstant` and tax scales instead of bare python objects. + - The parameters of a legislation are wrapped into the classes `Node`, `Parameter`, `Scale`, `Bracket`, `ValuesHistory`, `ValueAtInstant` instead of bare python dict. + - The parameters of a legislation at a given instant are wrapped into the classes `NodeAtInstant`, `ValueAtInstant` and tax scales instead of bare python objects. - The file `parameters.py` and the classes defined inside are responsible both for loading and accessing the parameters. Before the loading was implemented in `legislationsxml.py` and the other processings were implemented in `legislations.py` - The validation of the XML files was performed against a XML schema defined in `legislation.xsd`. Now the YAML files are loaded with the library `yaml` and then validated in basic Python. @@ -2852,7 +2864,7 @@ For more information, check the [documentation](https://openfisca.org/doc/coding - `Simulation.get_compact_legislation()` -> `Simulation._get_parameters_at_instant()` - `Simulation.get_baseline_compact_legislation()` -> `Simulation._get_baseline_parameters_at_instant()` -* The optionnal parameter `traced_simulation` is removed in function `TaxBenefitSystem.get_compact_legislation()` (now `TaxBenefitSystem.get_parameters_at_instant()`). This parameter had no effect. +* The optional parameter `traced_simulation` is removed in function `TaxBenefitSystem.get_compact_legislation()` (now `TaxBenefitSystem.get_parameters_at_instant()`). This parameter had no effect. * The optional parameter `with_source_file_infos` is removed in functions `TaxBenefitSystem.compute_legislation()` (now `TaxBenefitSystem._compute_parameters()`) and `TaxBenefitSystem.get_legislation()`. This parameter had no effect. @@ -2870,7 +2882,7 @@ For more information, check the [documentation](https://openfisca.org/doc/coding In the preview web API, for variables of type `Enum`: * Accept and recommend to use strings as simulation inputs, instead of the enum indices. - - For instance, `{"housing_occupancy_status": {"2017-01": "Tenant"}}` is now accepted and prefered to `{"housing_occupancy_status": {"2017-01": 0}}`). + - For instance, `{"housing_occupancy_status": {"2017-01": "Tenant"}}` is now accepted and preferred to `{"housing_occupancy_status": {"2017-01": 0}}`). - Using the enum indices as inputs is _still accepted_ for backward compatibility, but _should not_ be encouraged. * Return strings instead of enum indices. - For instance, is `housing_occupancy_status` is calculated for `2017-01`, `{"housing_occupancy_status": {"2017-01": "Tenant"}}` is now returned, instead of `{"housing_occupancy_status": {"2017-01": 0}}`. @@ -2916,7 +2928,7 @@ In the preview web API, for variables of type `Enum`: - This attribute is the legislative reference of a variable. - As previously, this attribute can be a string, or a list of strings. * Rename `Variable` attribute `reference` to `baseline_variable` - - This attibute is, for a variable defined in a reform, the baseline variable the reform variable is replacing. + - This attribute is, for a variable defined in a reform, the baseline variable the reform variable is replacing. * Remove variable attribute `law_reference` * Rename `TaxBenefitSystem.reference` to `TaxBenefitSystem.baseline` * Rename `TaxBenefitSystem.get_reference_compact_legislation` to `TaxBenefitSystem.get_baseline_compact_legislation` @@ -3194,7 +3206,7 @@ These breaking changes only concern variable and tax and benefit system **metada # 9.0.0 -* Make sure identic periods are stringified the same way +* Make sure identical periods are stringified the same way * _Breaking changes_: - Change `periods.period` signature. - It now only accepts strings. @@ -3231,7 +3243,7 @@ These breaking changes only concern variable and tax and benefit system **metada ## 6.1.0 * Move `base.py` content (file usually located in country packages) to core module `formula_toolbox` so that it can be reused by all countries -* Use `AbstractScenario` if no custom scenario is defined for a tax and benefit sytem +* Use `AbstractScenario` if no custom scenario is defined for a tax and benefit system # 6.0.0 @@ -3273,7 +3285,7 @@ These breaking changes only concern variable and tax and benefit system **metada * Improve `openfisca-run-test` script - Make country package detection more robust (it only worked for packages installed in editable mode) - Use spaces instead of commas as separator in the script arguments when loading several extensions or reforms (this is more standard) -* Refactor the `scripts` module to seperate the logic specific to yaml test running from the one that can be re-used by any script which needs to build a tax and benefit system. +* Refactor the `scripts` module to separate the logic specific to yaml test running from the one that can be re-used by any script which needs to build a tax and benefit system. # 5.0.0 @@ -3291,7 +3303,7 @@ These breaking changes only concern variable and tax and benefit system **metada ### 4.3.4 -* Fix occasionnal `NaN` creation in `MarginalRateTaxScale.calc` resulting from `0 * numpy.inf` +* Fix occasional `NaN` creation in `MarginalRateTaxScale.calc` resulting from `0 * numpy.inf` ### 4.3.3 @@ -3350,8 +3362,8 @@ Unlike simple formulas, a `DatedVariable` have several functions. We thus need t ### 4.1.2-Beta * Enable simulation initialization with only legacy roles - * New roles are in this case automatically infered - * Positions are always infered from persons entity id + * New roles are in this case automatically inferred + * Positions are always inferred from persons entity id ### 4.1.1-Beta @@ -3455,7 +3467,7 @@ Unlike simple formulas, a `DatedVariable` have several functions. We thus need t * Metaclasses are not used anymore. * New API for TaxBenefitSystem * Columns are now stored in the TaxBenefitSystem, not in entities. -* New API for rerforms. +* New API for reforms. * XmlBasedTaxBenefitSystem is deprecated, and MultipleXmlBasedTaxBenefitSystem renamed to TaxBenefitSystem ## 1.1.0 – [diff](https://github.com/openfisca/openfisca-core/compare/1.0.0...1.1.0) @@ -3469,7 +3481,7 @@ Unlike simple formulas, a `DatedVariable` have several functions. We thus need t ## 0.5.4 – [diff](https://github.com/openfisca/openfisca-core/compare/0.5.3...0.5.4) * Merge pull request #382 from openfisca/fix-sum-by-entity -* The result size must be the targent entity'one +* The result size must be the target entity's one ## 0.5.3 – [diff](https://github.com/openfisca/openfisca-core/compare/0.5.2...0.5.3) From 67ec505f6b59b3fd8bba0983d8de585ca521cfc7 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 20:26:40 +0100 Subject: [PATCH 25/39] Refactor the deps action --- .github/workflows/_deps.yaml | 55 ++++++++++++++++++++++++++++++++++++ .github/workflows/build.yaml | 29 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/_deps.yaml create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/_deps.yaml b/.github/workflows/_deps.yaml new file mode 100644 index 000000000..1513b523b --- /dev/null +++ b/.github/workflows/_deps.yaml @@ -0,0 +1,55 @@ +name: Install dependencies + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + deps: + runs-on: ${{ input.os }} + name: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }} + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ input.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(input.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + id: restore-deps + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }}-${{ hashFiles('setup.py') }} + restore-keys: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }}- + + - name: Install dependencies + run: | + python -m venv venv + ${{ input.activate_command }} + make install-deps install-dist diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..c88f35513 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,29 @@ +name: OpenFisca-Core / Build + +on: + pull_request: + types: [ assigned, opened, reopened, synchronize, ready_for_review ] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + deps: + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_deps.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} From 6563f1fafddf641704affe9f5c1f87b84e336a64 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 20:28:51 +0100 Subject: [PATCH 26/39] Fix typo in 'input' --- .github/workflows/_deps.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_deps.yaml b/.github/workflows/_deps.yaml index 1513b523b..548f2691e 100644 --- a/.github/workflows/_deps.yaml +++ b/.github/workflows/_deps.yaml @@ -21,8 +21,8 @@ on: jobs: deps: - runs-on: ${{ input.os }} - name: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }} + runs-on: ${{ inputs.os }} + name: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} env: TERM: xterm-256color # To colorize output of make tasks. @@ -33,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ input.python }} + python-version: ${{ inputs.python }} - name: Use zstd for faster cache restore (windows) if: ${{ startsWith(input.os, 'windows') }} @@ -45,11 +45,11 @@ jobs: uses: actions/cache@v3 with: path: venv - key: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }}-${{ hashFiles('setup.py') }} - restore-keys: deps-${{ input.os }}-np${{ input.numpy }}-py${{ input.python }}- + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + restore-keys: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- - name: Install dependencies run: | python -m venv venv - ${{ input.activate_command }} + ${{ inputs.activate_command }} make install-deps install-dist From b497e69367acedee976d1016bc5c06f50f7f09b0 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 20:29:53 +0100 Subject: [PATCH 27/39] Fix typo in 'input' (2/2) --- .github/workflows/_deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_deps.yaml b/.github/workflows/_deps.yaml index 548f2691e..c27489b86 100644 --- a/.github/workflows/_deps.yaml +++ b/.github/workflows/_deps.yaml @@ -36,7 +36,7 @@ jobs: python-version: ${{ inputs.python }} - name: Use zstd for faster cache restore (windows) - if: ${{ startsWith(input.os, 'windows') }} + if: ${{ startsWith(inputs.os, 'windows') }} shell: cmd run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" From 68171eb2bed2bbad1eb20a7905e2a696cdfa212e Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 20:42:43 +0100 Subject: [PATCH 28/39] Refactor the build action --- .github/workflows/_build.yaml | 68 +++++++++++++++++++ .github/workflows/{build.yaml => review.yaml} | 21 +++++- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/_build.yaml rename .github/workflows/{build.yaml => review.yaml} (53%) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml new file mode 100644 index 000000000..d9542b637 --- /dev/null +++ b/.github/workflows/_build.yaml @@ -0,0 +1,68 @@ +name: Build package + +on: + workflow_call: + inputs: + os: + required: true + type: string + + numpy: + required: true + type: string + + python: + required: true + type: string + + activate_command: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + name: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} + needs: [ deps ] + env: + TERM: xterm-256color # To colorize output of make tasks. + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python }} + + - name: Use zstd for faster cache restore (windows) + if: ${{ startsWith(inputs.os, 'windows') }} + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: venv + key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v3 + with: + path: venv/**/[Oo]pen[Ff]isca* + key: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + restore-keys: | + build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}- + build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- + + - name: Cache release + uses: actions/cache@v3 + with: + path: dist + key: release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Build package + run: | + ${{ inputs.activate_command }} + make install-test clean build diff --git a/.github/workflows/build.yaml b/.github/workflows/review.yaml similarity index 53% rename from .github/workflows/build.yaml rename to .github/workflows/review.yaml index c88f35513..658841328 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/review.yaml @@ -1,4 +1,4 @@ -name: OpenFisca-Core / Build +name: OpenFisca-Core / Review on: pull_request: @@ -27,3 +27,22 @@ jobs: numpy: ${{ matrix.numpy }} python: ${{ matrix.python }} activate_command: ${{ matrix.activate_command }} + + build: + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_build.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} From f5897e105b73ba84e294a48723679866684a4f66 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 20:44:44 +0100 Subject: [PATCH 29/39] Fix dependency in job --- .github/workflows/_build.yaml | 1 - .github/workflows/review.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index d9542b637..c7d705d6e 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -23,7 +23,6 @@ jobs: build: runs-on: ${{ inputs.os }} name: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} - needs: [ deps ] env: TERM: xterm-256color # To colorize output of make tasks. diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index 658841328..bb0c97167 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -29,6 +29,7 @@ jobs: activate_command: ${{ matrix.activate_command }} build: + needs: [ deps ] strategy: fail-fast: true matrix: From 566977536e4a6b218ca6d2c77ab08590400e0469 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 21:16:24 +0100 Subject: [PATCH 30/39] Refactor test & lint actions --- .github/workflows/review.yaml | 40 +++++++++++++++++++++++++++++++++++ openfisca_tasks/lint.mk | 8 +++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index bb0c97167..062768cc3 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -47,3 +47,43 @@ jobs: numpy: ${{ matrix.numpy }} python: ${{ matrix.python }} activate_command: ${{ matrix.activate_command }} + + test: + needs: [ deps ] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + lint: + needs: [ deps ] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_lint.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 5e1a6f3d7..8002237f9 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -1,15 +1,15 @@ ## Lint the codebase. -lint: check-syntax-errors check-style lint-doc check-types lint-typing-strict +lint: compile lint-style lint-doc lint-typing lint-typing-strict @$(call print_pass,$@:) ## Compile python files to check for syntax errors. -check-syntax-errors: . +compile: . @$(call print_help,$@:) @python -m compileall -q $? @$(call print_pass,$@:) ## Run linters to check for syntax and style errors. -check-style: $(shell git ls-files "*.py") +lint-style: $(shell git ls-files "*.py") @$(call print_help,$@:) @python -m flake8 $? @$(call print_pass,$@:) @@ -34,7 +34,7 @@ lint-doc-%: @$(call print_pass,$@:) ## Run static type checkers for type errors. -check-types: +lint-typing: @$(call print_help,$@:) @python -m mypy --package openfisca_core --package openfisca_web_api @$(call print_pass,$@:) From 467198b73dc777f7851c2b7db65e0ac9472d94d3 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 21:21:11 +0100 Subject: [PATCH 31/39] Fix jobs dependencies --- .github/workflows/_build.yaml | 67 ----------------------------------- .github/workflows/_deps.yaml | 55 ---------------------------- .github/workflows/review.yaml | 28 +++------------ 3 files changed, 4 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/_build.yaml delete mode 100644 .github/workflows/_deps.yaml diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml deleted file mode 100644 index c7d705d6e..000000000 --- a/.github/workflows/_build.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Build package - -on: - workflow_call: - inputs: - os: - required: true - type: string - - numpy: - required: true - type: string - - python: - required: true - type: string - - activate_command: - required: true - type: string - -jobs: - build: - runs-on: ${{ inputs.os }} - name: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} - env: - TERM: xterm-256color # To colorize output of make tasks. - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python }} - - - name: Use zstd for faster cache restore (windows) - if: ${{ startsWith(inputs.os, 'windows') }} - shell: cmd - run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" - - - name: Cache dependencies - uses: actions/cache@v3 - with: - path: venv - key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} - - - name: Cache build - uses: actions/cache@v3 - with: - path: venv/**/[Oo]pen[Ff]isca* - key: build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - restore-keys: | - build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}- - build-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- - - - name: Cache release - uses: actions/cache@v3 - with: - path: dist - key: release-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Build package - run: | - ${{ inputs.activate_command }} - make install-test clean build diff --git a/.github/workflows/_deps.yaml b/.github/workflows/_deps.yaml deleted file mode 100644 index c27489b86..000000000 --- a/.github/workflows/_deps.yaml +++ /dev/null @@ -1,55 +0,0 @@ -name: Install dependencies - -on: - workflow_call: - inputs: - os: - required: true - type: string - - numpy: - required: true - type: string - - python: - required: true - type: string - - activate_command: - required: true - type: string - -jobs: - deps: - runs-on: ${{ inputs.os }} - name: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }} - env: - TERM: xterm-256color # To colorize output of make tasks. - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python }} - - - name: Use zstd for faster cache restore (windows) - if: ${{ startsWith(inputs.os, 'windows') }} - shell: cmd - run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" - - - name: Cache dependencies - id: restore-deps - uses: actions/cache@v3 - with: - path: venv - key: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}-${{ hashFiles('setup.py') }} - restore-keys: deps-${{ inputs.os }}-np${{ inputs.numpy }}-py${{ inputs.python }}- - - - name: Install dependencies - run: | - python -m venv venv - ${{ inputs.activate_command }} - make install-deps install-dist diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index 062768cc3..d65cf323a 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -9,7 +9,7 @@ concurrency: cancel-in-progress: true jobs: - deps: + setup: strategy: fail-fast: true matrix: @@ -21,27 +21,7 @@ jobs: activate_command: source venv/bin/activate - os: windows-latest activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_deps.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - build: - needs: [ deps ] - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_build.yaml + uses: ./.github/workflows/_before.yaml with: os: ${{ matrix.os }} numpy: ${{ matrix.numpy }} @@ -49,7 +29,7 @@ jobs: activate_command: ${{ matrix.activate_command }} test: - needs: [ deps ] + needs: [ setup ] strategy: fail-fast: true matrix: @@ -69,7 +49,7 @@ jobs: activate_command: ${{ matrix.activate_command }} lint: - needs: [ deps ] + needs: [ setup ] strategy: fail-fast: true matrix: From 0bce283ba0b55606e0507616bc9d23831233c1b1 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 21:26:45 +0100 Subject: [PATCH 32/39] Fix typo in lint-style --- openfisca_tasks/lint.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_tasks/lint.mk b/openfisca_tasks/lint.mk index 8002237f9..99c834371 100644 --- a/openfisca_tasks/lint.mk +++ b/openfisca_tasks/lint.mk @@ -1,5 +1,5 @@ ## Lint the codebase. -lint: compile lint-style lint-doc lint-typing lint-typing-strict +lint: compile lint-doc lint-style lint-typing lint-typing-strict @$(call print_pass,$@:) ## Compile python files to check for syntax errors. From c5c016dcadaf352414e21b09471e4ee9771b321f Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 21:46:59 +0100 Subject: [PATCH 33/39] Add web trigger --- .github/workflows/deploy.yaml | 186 ++++++++++++++++++++++++++++++ .github/workflows/web-trigger.yml | 77 +++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/web-trigger.yml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 000000000..f7d6be538 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,186 @@ +name: OpenFisca-Core / Deploy + +on: + push: + branches: [ master ] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + setup: + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_before.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + test: + needs: [ setup ] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_test.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + lint: + needs: [ setup ] + strategy: + fail-fast: true + matrix: + os: [ubuntu-20.04, windows-latest] + numpy: [1.20.3] + python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + include: + - os: ubuntu-20.04 + activate_command: source venv/bin/activate + - os: windows-latest + activate_command: .\venv\Scripts\activate + uses: ./.github/workflows/_lint.yaml + with: + os: ${{ matrix.os }} + numpy: ${{ matrix.numpy }} + python: ${{ matrix.python }} + activate_command: ${{ matrix.activate_command }} + + + # The idea behind these dependencies is we want to give feedback to + # contributors on the version number only after they have passed all tests, + # so they don't have to do it twice after changes happened to the main branch + # during the time they took to fix the tests. + check-version: + runs-on: ubuntu-20.04 + needs: [ test-core, test-country-template, test-extension-template, lint-files ] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" + + # GitHub Actions does not have a halt job option, to stop from deploying if no functional changes were found. + # We build a separate job to substitute the halt option. + # The `deploy` job is dependent on the output of the `check-for-functional-changes`job. + check-for-functional-changes: + runs-on: ubuntu-20.04 + if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch + needs: [ check-version ] # Last job to run + outputs: + status: ${{ steps.stop-early.outputs.status }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - id: stop-early + run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. + + deploy: + runs-on: ubuntu-20.04 + needs: [ check-for-functional-changes ] + if: needs.check-for-functional-changes.outputs.status == 'success' + env: + PYPI_USERNAME: openfisca-bot + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + CIRCLE_TOKEN: ${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} # Personal API token created in CircleCI to grant full read and write permissions + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Cache deps + uses: actions/cache@v3 + with: + path: venv + key: deps-ubuntu-20.04-1.20.3-3.8.10-${{ hashFiles('setup.py') }} + + - name: Cache build + uses: actions/cache@v3 + with: + path: venv/**/[oO]pen[fF]isca* + key: build-ubuntu-20.04-1.20.3-3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Upload a Python package to PyPi + run: | + source venv/bin/activate + make publish + + - name: Update doc + run: | + curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} + + publish-to-conda: + runs-on: ubuntu-20.04 + needs: [ deploy ] + + steps: + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.7.9 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + channels: conda-forge + activate-environment: true + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Update meta.yaml + run: | + python3 -m pip install requests argparse + # Sleep to allow PyPi to update its API + sleep 60 + python3 .github/get_pypi_info.py -p OpenFisca-Core + + - name: Conda Config + run: | + conda install conda-build anaconda-client + conda info + conda config --set anaconda_upload yes + + - name: Conda build + run: conda build -c conda-forge --token ${{ secrets.ANACONDA_TOKEN }} --user openfisca .conda diff --git a/.github/workflows/web-trigger.yml b/.github/workflows/web-trigger.yml new file mode 100644 index 000000000..89673855d --- /dev/null +++ b/.github/workflows/web-trigger.yml @@ -0,0 +1,77 @@ +name: OpenFisca-Core / Web triggered workflow + +on: + workflow_dispatch: + inputs: + os: + description: Platform + required: true + default: ubuntu-20.04 + type: choice + options: + - ubuntu-20.04 + - ubuntu-latest + - windows-latest + + numpy: + description: NumPy version + required: true + default: 1.20.3 + type: choice + options: + - 1.20.3 + - 1.21.6 + - 1.22.4 + - 1.23.5 + + python: + description: Python version + required: true + default: 3.7.9 + type: choice + options: + - 3.7.9 + - 3.7.16 + - 3.8.10 + - 3.8.16 + - 3.9.13 + - 3.9.16 + - 3.10.9 + - 3.11.1 + + activate_command: + description: Virtualenv activation command + required: true + default: source venv/bin/activate + type: choice + options: + - source venv/bin/activate + - .\venv\Scripts\activate + + +jobs: + setup: + uses: ./.github/workflows/_before.yaml + with: + os: ${{ inputs.os }} + numpy: ${{ inputs.numpy }} + python: ${{ inputs.python }} + activate_command: ${{ inputs.activate_command }} + + test: + needs: [ setup ] + uses: ./.github/workflows/_test.yaml + with: + os: ${{ inputs.os }} + numpy: ${{ inputs.numpy }} + python: ${{ inputs.python }} + activate_command: ${{ inputs.activate_command }} + + lint: + needs: [ setup ] + uses: ./.github/workflows/_lint.yaml + with: + os: ${{ inputs.os }} + numpy: ${{ inputs.numpy }} + python: ${{ inputs.python }} + activate_command: ${{ inputs.activate_command }} From 319d015f9c34b4fcd4b75035e25179f3624f3264 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 21:56:39 +0100 Subject: [PATCH 34/39] Fix deploy in Github Actions --- .github/workflows/deploy.yaml | 27 ++++++++++++++++----------- .github/workflows/review.yaml | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f7d6be538..af6c13a3f 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -68,14 +68,13 @@ jobs: python: ${{ matrix.python }} activate_command: ${{ matrix.activate_command }} - # The idea behind these dependencies is we want to give feedback to # contributors on the version number only after they have passed all tests, # so they don't have to do it twice after changes happened to the main branch # during the time they took to fix the tests. check-version: runs-on: ubuntu-20.04 - needs: [ test-core, test-country-template, test-extension-template, lint-files ] + needs: [ test, lint ] steps: - uses: actions/checkout@v3 @@ -90,12 +89,12 @@ jobs: - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" - # GitHub Actions does not have a halt job option, to stop from deploying if no functional changes were found. - # We build a separate job to substitute the halt option. - # The `deploy` job is dependent on the output of the `check-for-functional-changes`job. + # GitHub Actions does not have a halt job option, to stop from deploying if + # no functional changes were found. We build a separate job to substitute the + # halt option. The `deploy` job is dependent on the output of the + # `check-for-functional-changes`job. check-for-functional-changes: runs-on: ubuntu-20.04 - if: github.ref == 'refs/heads/master' # Only triggered for the `master` branch needs: [ check-version ] # Last job to run outputs: status: ${{ steps.stop-early.outputs.status }} @@ -113,7 +112,7 @@ jobs: - id: stop-early run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. - deploy: + publish-to-pypi: runs-on: ubuntu-20.04 needs: [ check-for-functional-changes ] if: needs.check-for-functional-changes.outputs.status == 'success' @@ -136,15 +135,21 @@ jobs: uses: actions/cache@v3 with: path: venv - key: deps-ubuntu-20.04-1.20.3-3.8.10-${{ hashFiles('setup.py') }} + key: deps-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }} - name: Cache build uses: actions/cache@v3 with: path: venv/**/[oO]pen[fF]isca* - key: build-ubuntu-20.04-1.20.3-3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} + key: build-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} + + - name: Cache release + uses: actions/cache@v3 + with: + path: dist + key: release-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} - - name: Upload a Python package to PyPi + - name: Upload package to PyPi run: | source venv/bin/activate make publish @@ -155,7 +160,7 @@ jobs: publish-to-conda: runs-on: ubuntu-20.04 - needs: [ deploy ] + needs: [ publish-to-pypi ] steps: - uses: conda-incubator/setup-miniconda@v2 diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml index d65cf323a..d26748984 100644 --- a/.github/workflows/review.yaml +++ b/.github/workflows/review.yaml @@ -67,3 +67,24 @@ jobs: numpy: ${{ matrix.numpy }} python: ${{ matrix.python }} activate_command: ${{ matrix.activate_command }} + + # The idea behind these dependencies is we want to give feedback to + # contributors on the version number only after they have passed all tests, + # so they don't have to do it twice after changes happened to the main branch + # during the time they took to fix the tests. + check-version: + runs-on: ubuntu-20.04 + needs: [ test, lint ] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. + + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" From 1fce7fbb3f0aa4ed921bc085857451add90f9861 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 22:02:33 +0100 Subject: [PATCH 35/39] Rename workflows --- .github/workflows/deploy.yaml | 191 ------------------ .github/workflows/review.yaml | 90 --------- .../workflows/{web-trigger.yml => web.yml} | 0 3 files changed, 281 deletions(-) delete mode 100644 .github/workflows/deploy.yaml delete mode 100644 .github/workflows/review.yaml rename .github/workflows/{web-trigger.yml => web.yml} (100%) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index af6c13a3f..000000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,191 +0,0 @@ -name: OpenFisca-Core / Deploy - -on: - push: - branches: [ master ] - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - setup: - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_before.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - test: - needs: [ setup ] - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_test.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - lint: - needs: [ setup ] - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_lint.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - # The idea behind these dependencies is we want to give feedback to - # contributors on the version number only after they have passed all tests, - # so they don't have to do it twice after changes happened to the main branch - # during the time they took to fix the tests. - check-version: - runs-on: ubuntu-20.04 - needs: [ test, lint ] - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - - - name: Check version number has been properly updated - run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" - - # GitHub Actions does not have a halt job option, to stop from deploying if - # no functional changes were found. We build a separate job to substitute the - # halt option. The `deploy` job is dependent on the output of the - # `check-for-functional-changes`job. - check-for-functional-changes: - runs-on: ubuntu-20.04 - needs: [ check-version ] # Last job to run - outputs: - status: ${{ steps.stop-early.outputs.status }} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - - - id: stop-early - run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "::set-output name=status::success" ; fi # The `check-for-functional-changes` job should always succeed regardless of the `has-functional-changes` script's exit code. Consequently, we do not use that exit code to trigger deploy, but rather a dedicated output variable `status`, to avoid a job failure if the exit code is different from 0. Conversely, if the job fails the entire workflow would be marked as `failed` which is disturbing for contributors. - - publish-to-pypi: - runs-on: ubuntu-20.04 - needs: [ check-for-functional-changes ] - if: needs.check-for-functional-changes.outputs.status == 'success' - env: - PYPI_USERNAME: openfisca-bot - PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - CIRCLE_TOKEN: ${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} # Personal API token created in CircleCI to grant full read and write permissions - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - - - name: Cache deps - uses: actions/cache@v3 - with: - path: venv - key: deps-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }} - - - name: Cache build - uses: actions/cache@v3 - with: - path: venv/**/[oO]pen[fF]isca* - key: build-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Cache release - uses: actions/cache@v3 - with: - path: dist - key: release-ubuntu-20.04-np1.20.3-py3.8.10-${{ hashFiles('setup.py') }}-${{ github.sha }} - - - name: Upload package to PyPi - run: | - source venv/bin/activate - make publish - - - name: Update doc - run: | - curl -X POST --header "Content-Type: application/json" -d '{"branch":"master"}' https://circleci.com/api/v1.1/project/github/openfisca/openfisca-doc/build?circle-token=${{ secrets.CIRCLECI_V1_OPENFISCADOC_TOKEN }} - - publish-to-conda: - runs-on: ubuntu-20.04 - needs: [ publish-to-pypi ] - - steps: - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: 3.7.9 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - channels: conda-forge - activate-environment: true - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Update meta.yaml - run: | - python3 -m pip install requests argparse - # Sleep to allow PyPi to update its API - sleep 60 - python3 .github/get_pypi_info.py -p OpenFisca-Core - - - name: Conda Config - run: | - conda install conda-build anaconda-client - conda info - conda config --set anaconda_upload yes - - - name: Conda build - run: conda build -c conda-forge --token ${{ secrets.ANACONDA_TOKEN }} --user openfisca .conda diff --git a/.github/workflows/review.yaml b/.github/workflows/review.yaml deleted file mode 100644 index d26748984..000000000 --- a/.github/workflows/review.yaml +++ /dev/null @@ -1,90 +0,0 @@ -name: OpenFisca-Core / Review - -on: - pull_request: - types: [ assigned, opened, reopened, synchronize, ready_for_review ] - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - setup: - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_before.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - test: - needs: [ setup ] - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_test.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - lint: - needs: [ setup ] - strategy: - fail-fast: true - matrix: - os: [ubuntu-20.04, windows-latest] - numpy: [1.20.3] - python: [3.8.10, 3.7.9] # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - include: - - os: ubuntu-20.04 - activate_command: source venv/bin/activate - - os: windows-latest - activate_command: .\venv\Scripts\activate - uses: ./.github/workflows/_lint.yaml - with: - os: ${{ matrix.os }} - numpy: ${{ matrix.numpy }} - python: ${{ matrix.python }} - activate_command: ${{ matrix.activate_command }} - - # The idea behind these dependencies is we want to give feedback to - # contributors on the version number only after they have passed all tests, - # so they don't have to do it twice after changes happened to the main branch - # during the time they took to fix the tests. - check-version: - runs-on: ubuntu-20.04 - needs: [ test, lint ] - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all the tags - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.8.10 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. If left unspecified, different patch versions could be allocated between jobs, and any such difference would lead to a cache not found error. - - - name: Check version number has been properly updated - run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" diff --git a/.github/workflows/web-trigger.yml b/.github/workflows/web.yml similarity index 100% rename from .github/workflows/web-trigger.yml rename to .github/workflows/web.yml From 51272c3bb6179a4da19d189753d78c7d61f51631 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Tue, 13 Dec 2022 22:03:28 +0100 Subject: [PATCH 36/39] Remove spurious newline --- .github/workflows/web.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 89673855d..0a8302f38 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -48,7 +48,6 @@ on: - source venv/bin/activate - .\venv\Scripts\activate - jobs: setup: uses: ./.github/workflows/_before.yaml From b8434b0c8fec919646601d69f316c67e29e868c6 Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Sun, 22 Jan 2023 21:22:09 +0100 Subject: [PATCH 37/39] Skip web workflow for now --- .github/workflows/web.yml | 76 --------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 .github/workflows/web.yml diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml deleted file mode 100644 index 0a8302f38..000000000 --- a/.github/workflows/web.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: OpenFisca-Core / Web triggered workflow - -on: - workflow_dispatch: - inputs: - os: - description: Platform - required: true - default: ubuntu-20.04 - type: choice - options: - - ubuntu-20.04 - - ubuntu-latest - - windows-latest - - numpy: - description: NumPy version - required: true - default: 1.20.3 - type: choice - options: - - 1.20.3 - - 1.21.6 - - 1.22.4 - - 1.23.5 - - python: - description: Python version - required: true - default: 3.7.9 - type: choice - options: - - 3.7.9 - - 3.7.16 - - 3.8.10 - - 3.8.16 - - 3.9.13 - - 3.9.16 - - 3.10.9 - - 3.11.1 - - activate_command: - description: Virtualenv activation command - required: true - default: source venv/bin/activate - type: choice - options: - - source venv/bin/activate - - .\venv\Scripts\activate - -jobs: - setup: - uses: ./.github/workflows/_before.yaml - with: - os: ${{ inputs.os }} - numpy: ${{ inputs.numpy }} - python: ${{ inputs.python }} - activate_command: ${{ inputs.activate_command }} - - test: - needs: [ setup ] - uses: ./.github/workflows/_test.yaml - with: - os: ${{ inputs.os }} - numpy: ${{ inputs.numpy }} - python: ${{ inputs.python }} - activate_command: ${{ inputs.activate_command }} - - lint: - needs: [ setup ] - uses: ./.github/workflows/_lint.yaml - with: - os: ${{ inputs.os }} - numpy: ${{ inputs.numpy }} - python: ${{ inputs.python }} - activate_command: ${{ inputs.activate_command }} From 76856d6d43c255febecf962e1102d91681e3a389 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 24 Jan 2023 11:49:24 +0100 Subject: [PATCH 38/39] Sync Conda deps with PyPi --- .conda/meta.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 076dd397c..6472d0b36 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -28,16 +28,16 @@ requirements: - python - pip run: - - python >=3.6,<4.0 + - python >=3.7,<4.0 - dpath >=1.5.0,<3.0.0 - nptyping ==1.4.4 - numexpr >=2.7.0,<=3.0 - - numpy >=1.11,<1.21 + - numpy >=1.20,<1.21 - psutil >=5.4.7,<6.0.0 - pytest >=4.4.1,<6.0.0 - PyYAML >=3.10 - sortedcontainers ==2.2.2 - - typing-extensions ==3.10.0.2 + - typing-extensions >=4.0.0,<5.0.0 test: imports: @@ -61,8 +61,8 @@ outputs: host: - python run: - - python >=3.6,<4.0 - - flask ==1.1.2 + - python >=3.7,<4.0 + - flask ==1.1.4 - flask-cors ==3.0.10 - gunicorn >=20.0.0,<21.0.0 - werkzeug >=1.0.0,<2.0.0 @@ -76,9 +76,9 @@ outputs: - python run: - autopep8 >=1.4.0,<1.6.0 - - coverage ==6.0.2 + - coverage >=6.2.0,<7.0.0 - darglint ==1.8.0 - - flake8 >=3.9.0,<4.0.0 + - flake8 >=4.0.0,<4.1.0 - flake8-bugbear >=19.3.0,<20.0.0 - flake8-docstrings ==1.6.0 - flake8-print >=3.1.0,<4.0.0 From 51272ee774d74d0833e35f9ed9d7b0ded8f901de Mon Sep 17 00:00:00 2001 From: Mauko Quiroga Date: Thu, 1 Dec 2022 23:12:28 +0100 Subject: [PATCH 39/39] Bump version --- .conda/meta.yaml | 63 ++++++++++++++++++++++------------------ CHANGELOG.md | 48 +++++++++++++++--------------- openfisca_tasks/serve.mk | 2 +- setup.py | 30 +++++++++---------- 4 files changed, 75 insertions(+), 68 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 6472d0b36..336df12c9 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -28,23 +28,24 @@ requirements: - python - pip run: - - python >=3.7,<4.0 - - dpath >=1.5.0,<3.0.0 - - nptyping ==1.4.4 - - numexpr >=2.7.0,<=3.0 - - numpy >=1.20,<1.21 - - psutil >=5.4.7,<6.0.0 - - pytest >=4.4.1,<6.0.0 - - PyYAML >=3.10 - - sortedcontainers ==2.2.2 - - typing-extensions >=4.0.0,<5.0.0 + - python >= 3.7, < 4.0 + - PyYAML >= 6.0.0, < 7.0.0 + - dpath >= 2.1.0, < 3.0.0 + - importlib-metadata < 4.3.0 + - nptyping == 1.4.4 + - numexpr >= 2.8.0, <= 3.0.0 + - numpy >= 1.20, < 1.21 + - psutil >= 5.9.0, < 6.0.0 + - pytest >= 5.4.0, < 6.0.0 + - sortedcontainers == 2.2.2 + - typing-extensions >= 4.4.0, < 5.0.0 test: imports: - openfisca_core - openfisca_core.commons requires: - - pytest >=4.4.1,<6.0.0 + - pytest >= 5.4.0, < 6.0.0 - pip commands: - pip check @@ -61,11 +62,12 @@ outputs: host: - python run: - - python >=3.7,<4.0 - - flask ==1.1.4 - - flask-cors ==3.0.10 - - gunicorn >=20.0.0,<21.0.0 - - werkzeug >=1.0.0,<2.0.0 + - python >= 3.7, < 4.0 + - markupsafe == 2.0.1 + - flask == 1.1.4 + - flask-cors == 3.0.10 + - gunicorn >= 20.1.0, < 21.0.0 + - werkzeug >= 1.0.0, < 2.0.0 - {{ pin_subpackage('openfisca-core', exact=True) }} - name: openfisca-core-dev @@ -75,18 +77,23 @@ outputs: host: - python run: - - autopep8 >=1.4.0,<1.6.0 - - coverage >=6.2.0,<7.0.0 - - darglint ==1.8.0 - - flake8 >=4.0.0,<4.1.0 - - flake8-bugbear >=19.3.0,<20.0.0 - - flake8-docstrings ==1.6.0 - - flake8-print >=3.1.0,<4.0.0 - - flake8-rst-docstrings ==0.2.3 - - mypy ==0.910 - - openfisca-country-template >=3.10.0,<4.0.0 - - openfisca-extension-template >=1.2.0rc0,<2.0.0 - - pylint ==2.10.2 + - autopep8 >= 1.5.0, < 1.6.0 + - coverage >= 6.5.0, < 7.0.0 + - darglint == 1.8.0 + - flake8 >= 4.0.0, < 4.1.0 + - flake8-bugbear >= 19.8.0, < 20.0.0 + - flake8-docstrings == 1.6.0 + - flake8-print >= 3.1.0, < 4.0.0 + - flake8-rst-docstrings == 0.2.3 + - idna >= 3.4.0, < 4.0.0 + - isort >= 5.11.0, < 6.0.0 + - mypy == 0.910 + - openapi-spec-validator >= 0.5.0, < 0.6.0 + - pycodestyle >= 2.8.0, < 2.9.0 + - pylint == 2.10.2 + - xdoctest >= 1.1.0, < 2.0.0 + - openfisca-country-template >= 5.0.0, < 6.0.0 + - openfisca-extension-template >= 1.3.13, < 2.0.0 - {{ pin_subpackage('openfisca-core-api', exact=True) }} about: diff --git a/CHANGELOG.md b/CHANGELOG.md index f09b27129..8c027d3a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,25 +17,25 @@ #### Bug fix - Do not install dependencies outside the `setup.py` - - Dependencies installed outside the `setup.py` are not taken into account by + - Dependencies installed outside the `setup.py` are not taken into account by `pip`'s dependency resolver. - - In case of conflicting transient dependencies, the last library installed + - In case of conflicting transient dependencies, the last library installed will "impose" its dependency version. - - This makes the installation and build of the library non-deterministic and - prone to unforeseen bugs caused by external changes in dependencies' + - This makes the installation and build of the library non-deterministic and + prone to unforeseen bugs caused by external changes in dependencies' versions. #### Note -A definite way to solve this issue is to clearly separate library dependencies -(with a `virtualenv`) and a universal dependency installer for CI requirements +A definite way to solve this issue is to clearly separate library dependencies +(with a `virtualenv`) and a universal dependency installer for CI requirements (like `pipx`), taking care of: - Always running tests inside the `virtualenv` (for example with `nox`). -- Always building outside of the `virtualenv` (for example with `poetry` +- Always building outside of the `virtualenv` (for example with `poetry` installed by `pipx`). -Moreover, it is indeed even better to have a lock file for dependencies, +Moreover, it is indeed even better to have a lock file for dependencies, using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ### 38.0.2 [#1178](https://github.com/openfisca/openfisca-core/pull/1178) @@ -52,8 +52,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). # 38.0.0 [#989](https://github.com/openfisca/openfisca-core/pull/989) -> Note: Version `38.0.0` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `38.0.0` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### New Features @@ -68,8 +68,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ### 37.0.2 [#1170](https://github.com/openfisca/openfisca-core/pull/1170) -> Note: Version `37.0.2` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `37.0.2` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Technical changes @@ -78,8 +78,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ### 37.0.1 [#1169](https://github.com/openfisca/openfisca-core/pull/1169) -> Note: Version `37.0.1` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `37.0.1` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Technical changes @@ -88,8 +88,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). # 37.0.0 [#1142](https://github.com/openfisca/openfisca-core/pull/1142) -> Note: Version `37.0.0` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `37.0.0` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Deprecations @@ -105,8 +105,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). # 36.0.0 [#1149](https://github.com/openfisca/openfisca-core/pull/1162) -> Note: Version `36.0.0` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `36.0.0` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Breaking changes @@ -117,8 +117,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ## 35.12.0 [#1160](https://github.com/openfisca/openfisca-core/pull/1160) -> Note: Version `35.12.0` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `35.12.0` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### New Features @@ -127,8 +127,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ### 35.11.2 [#1166](https://github.com/openfisca/openfisca-core/pull/1166) -> Note: Version `35.11.2` has been unpublished as `35.11.1` introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `35.11.2` has been unpublished as `35.11.1` introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Technical changes @@ -137,8 +137,8 @@ using `pip freeze`) or with tools providing such features (`pipenv`, etc.). ### 35.11.1 [#1165](https://github.com/openfisca/openfisca-core/pull/1165) -> Note: Version `35.11.1` has been unpublished as it introduced a bug -> preventing users to load a tax-benefit system. Please use versions `38.0.2` +> Note: Version `35.11.1` has been unpublished as it introduced a bug +> preventing users to load a tax-benefit system. Please use versions `38.0.2` > and subsequents. #### Bug fix diff --git a/openfisca_tasks/serve.mk b/openfisca_tasks/serve.mk index ced050b24..efad0be6c 100644 --- a/openfisca_tasks/serve.mk +++ b/openfisca_tasks/serve.mk @@ -1,6 +1,6 @@ ## Serve the openfisca Web API. api: - $(call print_help,$@:) + @$(call print_help,$@:) @openfisca serve \ --country-package openfisca_country_template \ --extensions openfisca_extension_template diff --git a/setup.py b/setup.py index 1dd619cac..1b1307e2b 100644 --- a/setup.py +++ b/setup.py @@ -26,47 +26,47 @@ # functional and integration breaks caused by external code updates. general_requirements = [ - 'PyYAML >= 3.10', - 'dpath >= 1.5.0, < 3.0.0', + 'PyYAML >= 6.0.0, < 7.0.0', + 'dpath >= 2.1.0, < 3.0.0', 'importlib-metadata < 4.3.0', # Required for Python 3.7 and Flake8 'nptyping == 1.4.4', - 'numexpr >= 2.7.0, <= 3.0', + 'numexpr >= 2.8.0, <= 3.0.0', 'numpy >= 1.20, < 1.21', - 'psutil >= 5.4.7, < 6.0.0', - 'pytest >= 4.4.1, < 6.0.0', # For openfisca test + 'psutil >= 5.9.0, < 6.0.0', + 'pytest >= 5.4.0, < 6.0.0', # For openfisca test 'sortedcontainers == 2.2.2', - 'typing-extensions >= 4.0.0, < 5.0.0', + 'typing-extensions >= 4.4.0, < 5.0.0', ] api_requirements = [ 'markupsafe == 2.0.1', # While flask revision < 2 'flask == 1.1.4', 'flask-cors == 3.0.10', - 'gunicorn >= 20.0.0, < 21.0.0', + 'gunicorn >= 20.1.0, < 21.0.0', 'werkzeug >= 1.0.0, < 2.0.0', ] dev_requirements = [ - 'autopep8 >= 1.4.0, < 1.6.0', - 'coverage >= 6.2.0, < 7.0.0', + 'autopep8 >= 1.5.0, < 1.6.0', + 'coverage >= 6.5.0, < 7.0.0', 'darglint == 1.8.0', 'flake8 >= 4.0.0, < 4.1.0', - 'flake8-bugbear >= 19.3.0, < 20.0.0', + 'flake8-bugbear >= 19.8.0, < 20.0.0', 'flake8-docstrings == 1.6.0', 'flake8-print >= 3.1.0, < 4.0.0', 'flake8-rst-docstrings == 0.2.3', 'idna >= 3.4.0, < 4.0.0', - 'isort >= 5.0.0, < 6.0.0', + 'isort >= 5.11.0, < 6.0.0', 'mypy == 0.910', 'openapi-spec-validator >= 0.5.0, < 0.6.0', 'pycodestyle >= 2.8.0, < 2.9.0', 'pylint == 2.10.2', - 'xdoctest >= 1.0.0, < 2.0.0', + 'xdoctest >= 1.1.0, < 2.0.0', ] + api_requirements setup( name = 'OpenFisca-Core', - version = '38.0.3', + version = '39.0.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ @@ -95,8 +95,8 @@ 'web-api': api_requirements, 'dev': dev_requirements, 'ci': [ - 'build >= 0.9.0, < 1.0.0', - 'coveralls >= 3.0.0, < 4.0.0', + 'build >= 0.10.0, < 1.0.0', + 'coveralls >= 3.3.0, < 4.0.0', 'twine >= 4.0.0, < 5.0.0', 'wheel < 1.0.0', ],