diff --git a/.circleci/config.yml b/.circleci/config.yml index d0951f2100..c9f90c832b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,6 +48,44 @@ jobs: COUNTRY_TEMPLATE_PATH=`python -c "import openfisca_country_template; print(openfisca_country_template.CountryTaxBenefitSystem().get_package_metadata()['location'])"` openfisca test $COUNTRY_TEMPLATE_PATH/openfisca_country_template/tests/ + test_docs: + docker: + - image: python:3.7 + + steps: + - checkout + + - run: + name: Checkout docs + command: make test-doc-checkout branch=$CIRCLE_BRANCH + + - restore_cache: + key: v1-py3-{{ .Branch }}-{{ checksum "setup.py" }} + + - restore_cache: + key: v1-py3-docs-{{ .Branch }}-{{ checksum "doc/requirements.txt" }} + + - run: + name: Create a virtualenv + command: | + mkdir -p /tmp/venv/openfisca_doc + python -m venv /tmp/venv/openfisca_doc + echo "source /tmp/venv/openfisca_doc/bin/activate" >> $BASH_ENV + + - run: + name: Install dependencies + command: make test-doc-install + + - save_cache: + key: v1-py3-docs-{{ .Branch }}-{{ checksum "doc/requirements.txt" }} + paths: + - /tmp/venv/openfisca_doc + + - run: + name: Run doc tests + command: make test-doc-build + + check_version: docker: - image: python:3.7 @@ -123,6 +161,7 @@ workflows: build_and_deploy: jobs: - run_tests + - test_docs - check_version - submit_coverage: requires: @@ -130,6 +169,7 @@ workflows: - deploy: requires: - run_tests + - test_docs - check_version filters: branches: diff --git a/.gitignore b/.gitignore index 06bfe7ec27..4b56efc6da 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .vscode/ build/ dist/ +doc/ *.egg-info *.mo *.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index a895e9db65..7bed8ae1d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## 35.5.0 [#1038](https://github.com/openfisca/openfisca-core/pull/1038) + +#### New Features + +- Introduce `openfisca_core.variables.typing` + - Documents the signature of formulas + - Note: as formulas are generated dynamically, documenting them is tricky + +#### Bug Fixes + +- Fix the official doc + - Corrects malformed docstrings + - Fix missing and/ou outdated references + +#### Technical Changes + +- Add tasks to automatically validate that changes do not break the documentation + +#### Documentation + +- Add steps to follow in case the documentation is broken +- Add general documenting guidelines in CONTRIBUTING.md + ### 35.4.2 [#1026](https://github.com/openfisca/openfisca-core/pull/1026) #### Bug fix diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c54c9c6516..8ba10fd606 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,9 +65,138 @@ We strive to deliver great error messages, which means they are: ### Example -> **Terrible**: `unexpected value`. -> **Bad**: `argument must be a string, an int, or a period`. -> **Good**: `Variable {0} has already been set for all months contained in period {1}, and value {2} provided for {1} doesn't match the total ({3}).` -> **Great**: `Inconsistent input: variable {0} has already been set for all months contained in period {1}, and value {2} provided for {1} doesn't match the total ({3}). This error may also be thrown if you try to call set_input twice for the same variable and period. See more at .` +- **Terrible**: `unexpected value`. +- **Bad**: `argument must be a string, an int, or a period`. +- **Good**: `Variable {0} has already been set for all months contained in period {1}, and value {2} provided for {1} doesn't match the total ({3}).` +- **Great**: `Inconsistent input: variable {0} has already been set for all months contained in period {1}, and value {2} provided for {1} doesn't match the total ({3}). This error may also be thrown if you try to call set_input twice for the same variable and period. See more at .` [More information](https://blogs.mulesoft.com/dev/api-dev/api-best-practices-response-handling/). + +## Documentation + +OpenFisca does not yet follow a common convention for docstrings, so you'll find [ReStructuredText](https://www.sphinx-doc.org/en/master/usage/restructuredtext/domains.html), [Google](http://google.github.io/styleguide/pyguide.html#Comments), and [NumPy](https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt) style docstrings. + +Whatever the style you choose, contributors and reusers alike will be more than thankful for the effort you put in documenting your contributions. Here are some general good practices you can follow: + +1. TL;DR: Document your intent :smiley: + +2. When adding a new module with several classes and functions, please structure it in different files, create an `__init__.py` file and document it as follows: + +```py +"""Short summary of your module. + +A longer description of what are your module's motivation, domain, and use +cases. For example, if you decide to create a caching system for OpenFisca, +consisting on different caching mechanisms, you could say that large operations +are expensive for some users, that different caching mechanisms exist, and that +this module implements some of them. + +You can then give examples on how to use your module: + + .. code-block:: python + + this = cache(result) + that = this.flush() + +Better you can write `doctests` that will be run with the test suite: + + >>> from . import Cache + >>> cache = Cache("VFS") + >>> cache.add("ratio", .45) + >>> cache.get("ratio") + .45 + +""" + +from .cache import Cache +from .strategies import Memory, Disk + +__all__ = ["Cache", "Memory", "Disk"] +``` + +3. When adding a new class, you can either document the class itself, the `__init__` method, or both: + +```py +class Cache: + """Implements a new caching system. + + Same as before, you could say this is good because virtuals systems are + great but the need a wrapper to make them work with OpenFisca. + + Document the class attributes —different from the initialisation arguments: + type (str): Type of cache. + path (str): An so on… + + Document the class arguments, here or in the `__init__` method: + type: For example if you need a ``type`` to create the :class:`.Cache`. + + Please note that if you're using Python type annotations, you don't need to + specify types in the documentation, they will be parsed and verified + automatically. + + """ + + def __init__(self, type: str) -> None: + pass + +``` + +4. Finally, when adding methods to your class, or helper functions to your module, it is very important to document their contracts: + +```py +def get(self, key: str) -> Any: + """Again, summary description. + + The long description is optional, as long as the code is easy to + understand. However, there are four key elements to help others understand + what the code does: + + * What it takes as arguments + * What it returns + * What happens if things fail + * Valid examples that can be run! + + For example if we were following the Google style, it would look like this: + + Args: + key: The ``key`` to retrieve from the :obj:`.Cache`. + + Returns: + Whatever we stored, if we stored it (see, no need to specify the type) + + Raises: + :exc:`KeyNotFoundError`: When the ``key`` wasn't found. + + Examples: + >>> cache = Cache() + >>> cache.set("key", "value") + >>> cache.get("key") + "value" + + Note: + Your examples should be simple and illustrative. For more complex + scenarios write a regular test. + + Todo: + * Accept :obj:`int` as ``key``. + * Return None when key is not found. + + .. versionadded:: 1.2.3 + This will help people to undestand the code evolution. + + .. deprecated:: 2.3.4 + This, to have time to adapt their own codebases before the code is + removed forever. + + .. seealso:: + Finally, you can help users by referencing useful documentation of + other code, like :mod:`numpy.linalg`, or even links, like the official + OpenFisca `documentation`_. + + .. _documentation: https://openfisca.org/doc/ + + """ + + pass + +``` diff --git a/Makefile b/Makefile index 30a0578f0d..c139b6aac0 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ -doc = sed -n "/^$1/ { x ; p ; } ; s/\#\#/[⚙]/ ; s/\./.../ ; x" ${MAKEFILE_LIST} +help = sed -n "/^$1/ { x ; p ; } ; s/\#\#/[⚙]/ ; s/\./.../ ; x" ${MAKEFILE_LIST} +repo = https://github.com/openfisca/openfisca-doc +branch = $(shell git branch --show-current) -## Same as `make test. +## Same as `make test`. all: test ## Install project dependencies. install: - @$(call doc,$@:) + @$(call help,$@:) @pip install --upgrade pip twine wheel @pip install --editable .[dev] --upgrade --use-deprecated=legacy-resolver @@ -13,50 +15,106 @@ install: build: setup.py @## This allows us to be sure tests are run against the packaged version @## of openfisca-core, the same we put in the hands of users and reusers. - @$(call doc,$@:) + @$(call help,$@:) @python $? bdist_wheel @find dist -name "*.whl" -exec pip install --force-reinstall {}[dev] \; ## Uninstall project dependencies. uninstall: - @$(call doc,$@:) + @$(call help,$@:) @pip freeze | grep -v "^-e" | sed "s/@.*//" | xargs pip uninstall -y ## Delete builds and compiled python files. clean: \ $(shell ls -d * | grep "build\|dist") \ $(shell find . -name "*.pyc") - @$(call doc,$@:) + @$(call help,$@:) @rm -rf $? ## Compile python files to check for syntax errors. check-syntax-errors: . - @$(call doc,$@:) + @$(call help,$@:) @python -m compileall -q $? ## Run linters to check for syntax and style errors. check-style: $(shell git ls-files "*.py") - @$(call doc,$@:) + @$(call help,$@:) @flake8 $? ## Run code formatters to correct style errors. format-style: $(shell git ls-files "*.py") - @$(call doc,$@:) + @$(call help,$@:) @autopep8 $? ## Run static type checkers for type errors. check-types: openfisca_core openfisca_web_api - @$(call doc,$@:) + @$(call help,$@:) @mypy $? ## Run openfisca-core tests. test: clean check-syntax-errors check-style check-types - @$(call doc,$@:) + @$(call help,$@:) @env PYTEST_ADDOPTS="${PYTEST_ADDOPTS} --cov=openfisca_core" pytest +## Check that the current changes do not break the doc. +test-doc: + @## Usage: + @## + @## make test-doc [branch=BRANCH] + @## + @## Examples: + @## + @## # Will check the current branch in openfisca-doc. + @## make test-doc + @## + @## # Will check "test-doc" in openfisca-doc. + @## make test-doc branch=test-doc + @## + @## # Will check "master" if "asdf1234" does not exist. + @## make test-doc branch=asdf1234 + @## + @$(call help,$@:) + @${MAKE} test-doc-checkout + @${MAKE} test-doc-install + @${MAKE} test-doc-build + +## Update the local copy of the doc. +test-doc-checkout: + @$(call help,$@:) + @[ ! -d doc ] && git clone ${repo} doc || : + @cd doc && { \ + git reset --hard ; \ + git fetch --all ; \ + [ $$(git branch --show-current) != master ] && git checkout master || : ; \ + [ ${branch} != "master" ] \ + && { \ + { \ + git branch -D ${branch} 2> /dev/null ; \ + git checkout ${branch} ; \ + } \ + && git pull --ff-only origin ${branch} \ + || { \ + >&2 echo "[!] The branch '${branch}' doesn't exist, checking out 'master' instead..." ; \ + git pull --ff-only origin master ; \ + } \ + } \ + || git pull --ff-only origin master ; \ + } 1> /dev/null + +## Install doc dependencies. +test-doc-install: + @$(call help,$@:) + @pip install --requirement doc/requirements.txt 1> /dev/null + @pip install --editable .[dev] --upgrade 1> /dev/null + +## Dry-build the doc. +test-doc-build: + @$(call help,$@:) + @sphinx-build -M dummy doc/source doc/build -n -q -W + ## Serve the openfisca Web API. api: - @$(call doc,$@:) + @$(call help,$@:) @openfisca serve \ --country-package openfisca_country_template \ --extensions openfisca_extension_template diff --git a/README.md b/README.md index 8823d4c6b8..7f253c9114 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,76 @@ exec make format-style END ``` +## Documentation + +Yet however OpenFisca does not follow a common convention for docstrings, our current toolchain allows to check whether documentation builds correctly and to update it automatically with each contribution to this repository. + +In the meantime, please take a look at our [contributing guidelines](CONTRIBUTING.md) for some general tips on how to document your contributions, and at our official documentation's [repository](https://github.com/openfisca/openfisca-doc/blob/master/README.md) to in case you want to know how to build it by yourself —and improve it! + +### To verify that the documentation still builds correctly + +You can run: + +```sh +make test-doc +``` + +### If it doesn't, or if the doc is already broken. + +Here's how you can fix it: + +1. Clone the documentation, if not yet done: + +``` +make test-doc-checkout +``` + +2. Install the documentation's dependencies, if not yet done: + +``` +make test-doc-install +``` + +3. Create a branch, both in core and in the doc, to correct the problems: + +``` +git checkout -b fix-doc +sh -c "cd doc && git checkout -b `git branch --show-current`" +``` + +4. Fix the offending problems —they could be in core, in the doc, or in both. + +You can test-drive your fixes by checking that each change works as expected: + +``` +make test-doc-build branch=`git branch --show-current` +``` + +5. Commit at each step, so you don't accidentally lose your progress: + +``` +git add -A && git commit -m "Fix outdated argument for Entity" +sh -c "cd doc && git add -A && git commit -m \"Fix outdated argument for Entity\"" +``` + +6. Once you're done, push your changes and cleanup: + +``` +git push origin `git branch --show-current` +sh -c "cd doc && git push origin `git branch --show-current`" +rm -rf doc +``` + +7. Finally, open a pull request both in [core](https://github.com/openfisca/openfisca-core/compare/master...fix-doc) and in the [doc](https://github.com/openfisca/openfisca-doc/compare/master...fix-doc). + +[CircleCI](.circleci/config.yml) will automatically try to build the documentation from the same branch in both core and the doc (in our example "fix-doc") so we can integrate first our changes to core, and then our changes to the doc. + +If no changes were needed to the doc, then your changes to core will be verified against the production version of the doc. + +If your changes concern only the doc, please take a look at the doc's [README](https://github.com/openfisca/openfisca-doc/blob/master/README.md). + +That's it! 🙌 + ## Serving the API OpenFisca-Core provides a Web-API. It is by default served on the `5000` port. diff --git a/openfisca_core/indexed_enums/enum.py b/openfisca_core/indexed_enums/enum.py index 97b9a285e4..3d9fc08447 100644 --- a/openfisca_core/indexed_enums/enum.py +++ b/openfisca_core/indexed_enums/enum.py @@ -5,7 +5,7 @@ import numpy -from openfisca_core.indexed_enums import config, EnumArray +from . import ENUM_ARRAY_DTYPE, EnumArray class Enum(enum.Enum): @@ -43,8 +43,8 @@ def encode( array into an :any:`EnumArray`. See :any:`EnumArray.decode` for decoding. - :param ndarray array: Array of string identifiers, or of enum items, to - encode. + :param numpy.ndarray array: Array of string identifiers, or of enum + items, to encode. :returns: An :any:`EnumArray` encoding the input array values. :rtype: :any:`EnumArray` @@ -72,7 +72,7 @@ def encode( array = numpy.select( [array == item.name for item in cls], [item.index for item in cls], - ).astype(config.ENUM_ARRAY_DTYPE) + ).astype(ENUM_ARRAY_DTYPE) # Enum items arrays elif isinstance(array, numpy.ndarray) and \ @@ -94,6 +94,6 @@ def encode( array = numpy.select( [array == item for item in cls], [item.index for item in cls], - ).astype(config.ENUM_ARRAY_DTYPE) + ).astype(ENUM_ARRAY_DTYPE) return EnumArray(array, cls) diff --git a/openfisca_core/parameters/helpers.py b/openfisca_core/parameters/helpers.py index 0d0698ca36..75d5a18b73 100644 --- a/openfisca_core/parameters/helpers.py +++ b/openfisca_core/parameters/helpers.py @@ -19,7 +19,7 @@ def load_parameter_file(file_path, name = ''): """ Load parameters from a YAML file (or a directory containing YAML files). - :returns: An instance of :any:`ParameterNode` or :any:`Scale` or :any:`Parameter`. + :returns: An instance of :class:`.ParameterNode` or :class:`.ParameterScale` or :class:`.Parameter`. """ if not os.path.exists(file_path): raise ValueError("{} does not exist".format(file_path)) diff --git a/openfisca_core/parameters/parameter.py b/openfisca_core/parameters/parameter.py index 85d367ae67..62fd3f6766 100644 --- a/openfisca_core/parameters/parameter.py +++ b/openfisca_core/parameters/parameter.py @@ -34,7 +34,7 @@ class Parameter(AtInstantLike): } }) - .. py:attribute:: values_list + .. attribute:: values_list List of the values, in reverse chronological order """ diff --git a/openfisca_core/parameters/parameter_node.py b/openfisca_core/parameters/parameter_node.py index e811dd6f84..1dae81dfb4 100644 --- a/openfisca_core/parameters/parameter_node.py +++ b/openfisca_core/parameters/parameter_node.py @@ -5,7 +5,7 @@ import typing from openfisca_core import commons, parameters, tools -from openfisca_core.parameters import config, helpers, AtInstantLike, Parameter, ParameterNodeAtInstant +from . import config, helpers, AtInstantLike, Parameter, ParameterNodeAtInstant class ParameterNode(AtInstantLike): @@ -112,7 +112,7 @@ def add_child(self, name, child): Add a new child to the node. :param name: Name of the child that must be used to access that child. Should not contain anything that could interfere with the operator `.` (dot). - :param child: The new child, an instance of :any:`Scale` or :any:`Parameter` or :any:`ParameterNode`. + :param child: The new child, an instance of :class:`.ParameterScale` or :class:`.Parameter` or :class:`.ParameterNode`. """ if name in self.children: raise ValueError("{} has already a child named {}".format(self.name, name)) diff --git a/openfisca_core/simulations/simulation.py b/openfisca_core/simulations/simulation.py index 9544925990..5dd2694292 100644 --- a/openfisca_core/simulations/simulation.py +++ b/openfisca_core/simulations/simulation.py @@ -84,6 +84,8 @@ def data_storage_dir(self): # ----- Calculation methods ----- # def calculate(self, variable_name, period): + """Calculate ``variable_name`` for ``period``.""" + if period is not None and not isinstance(period, Period): period = periods.period(period) @@ -311,7 +313,7 @@ def get_array(self, variable_name, period): """ Return the value of ``variable_name`` for ``period``, if this value is alreay in the cache (if it has been set as an input or previously calculated). - Unlike :any:`calculate`, this method *does not* trigger calculations and *does not* use any formula. + Unlike :meth:`.calculate`, this method *does not* trigger calculations and *does not* use any formula. """ if period is not None and not isinstance(period, Period): period = periods.period(period) @@ -319,7 +321,7 @@ def get_array(self, variable_name, period): def get_holder(self, variable_name): """ - Get the :any:`Holder` associated with the variable ``variable_name`` for the simulation + Get the :obj:`.Holder` associated with the variable ``variable_name`` for the simulation """ return self.get_variable_population(variable_name).get_holder(variable_name) diff --git a/openfisca_core/taxbenefitsystems/tax_benefit_system.py b/openfisca_core/taxbenefitsystems/tax_benefit_system.py index 6ca8443492..26e37a7b81 100644 --- a/openfisca_core/taxbenefitsystems/tax_benefit_system.py +++ b/openfisca_core/taxbenefitsystems/tax_benefit_system.py @@ -31,9 +31,9 @@ class TaxBenefitSystem: :param string parameters: Directory containing the YAML parameter files. - .. py:attribute:: parameters + .. attribute:: parameters - :any:`ParameterNode` containing the legislation parameters + :obj:`.ParameterNode` containing the legislation parameters """ _base_tax_benefit_system = None _parameters_at_instant_cache = None @@ -143,9 +143,9 @@ def add_variable(self, variable): """ Adds an OpenFisca variable to the tax and benefit system. - :param Variable variable: The variable to add. Must be a subclass of Variable. + :param .Variable variable: The variable to add. Must be a subclass of Variable. - :raises: :any:`VariableNameConflict` if a variable with the same name have previously been added to the tax and benefit system. + :raises: :exc:`.VariableNameConflictError` if a variable with the same name have previously been added to the tax and benefit system. """ return self.load_variable(variable, update = False) @@ -338,9 +338,9 @@ def get_parameters_at_instant(self, instant): """ Get the parameters of the legislation at a given instant - :param instant: string of the format 'YYYY-MM-DD' or `openfisca_core.periods.Instant` instance. + :param instant: :obj:`str` of the format 'YYYY-MM-DD' or :class:`.Instant` instance. :returns: The parameters of the legislation at a given instant. - :rtype: :any:`ParameterNodeAtInstant` + :rtype: :class:`.ParameterNodeAtInstant` """ if isinstance(instant, Period): instant = instant.start @@ -410,7 +410,7 @@ def get_variables(self, entity = None): """ Gets all variables contained in a tax and benefit system. - :param entity: If set, returns only the variable defined for the given entity. + :param .Entity entity: If set, returns only the variable defined for the given entity. :returns: A dictionnary, indexed by variable names. :rtype: dict diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index f6833c74bb..53d5217249 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -44,11 +44,11 @@ def run_tests(tax_benefit_system, paths, options = None): If `path` is a directory, subdirectories will be recursively explored. - :param TaxBenefitSystem tax_benefit_system: the tax-benefit system to use to run the tests - :param (str/list) paths: A path, or a list of paths, towards the files or directories containing the tests to run. If a path is a directory, subdirectories will be recursively explored. + :param .TaxBenefitSystem tax_benefit_system: the tax-benefit system to use to run the tests + :param str or list paths: A path, or a list of paths, towards the files or directories containing the tests to run. If a path is a directory, subdirectories will be recursively explored. :param dict options: See more details below. - :raises AssertionError: if a test does not pass + :raises :exc:`AssertionError`: if a test does not pass :return: the number of sucessful tests excecuted @@ -58,7 +58,7 @@ def run_tests(tax_benefit_system, paths, options = None): | Key | Type | Role | +===============================+===========+===========================================+ | verbose | ``bool`` | | - +-------------------------------+-----------+ See :any:`openfisca_test` options doc + + +-------------------------------+-----------+ See :any:`openfisca_test` options doc + | | name_filter | ``str`` | | +-------------------------------+-----------+-------------------------------------------+ diff --git a/openfisca_core/tracers/__init__.py b/openfisca_core/tracers/__init__.py index 52f4d618ea..de489ad6d9 100644 --- a/openfisca_core/tracers/__init__.py +++ b/openfisca_core/tracers/__init__.py @@ -23,8 +23,8 @@ from .computation_log import ComputationLog # noqa: F401 from .flat_trace import FlatTrace # noqa: F401 +from .full_tracer import FullTracer # noqa: F401 +from .performance_log import PerformanceLog # noqa: F401 from .simple_tracer import SimpleTracer # noqa: F401 from .trace_node import TraceNode # noqa: F401 -from .performance_log import PerformanceLog # noqa: F401 -from .full_tracer import FullTracer # noqa: F401 from .tracing_parameter_node_at_instant import TracingParameterNodeAtInstant # noqa: F401 diff --git a/openfisca_core/tracers/computation_log.py b/openfisca_core/tracers/computation_log.py index d07b9654c4..c785fd9395 100644 --- a/openfisca_core/tracers/computation_log.py +++ b/openfisca_core/tracers/computation_log.py @@ -5,21 +5,20 @@ import numpy +from .. import tracers from openfisca_core.indexed_enums import EnumArray if typing.TYPE_CHECKING: from numpy.typing import ArrayLike - from openfisca_core.tracers import FullTracer, TraceNode - Array = Union[EnumArray, ArrayLike] class ComputationLog: - _full_tracer: FullTracer + _full_tracer: tracers.FullTracer - def __init__(self, full_tracer: FullTracer) -> None: + def __init__(self, full_tracer: tracers.FullTracer) -> None: self._full_tracer = full_tracer def display( @@ -33,12 +32,12 @@ def display( def _get_node_log( self, - node: TraceNode, + node: tracers.TraceNode, depth: int, aggregate: bool, ) -> List[str]: - def print_line(depth: int, node: TraceNode) -> str: + def print_line(depth: int, node: tracers.TraceNode) -> str: indent = ' ' * depth value = node.value diff --git a/openfisca_core/tracers/flat_trace.py b/openfisca_core/tracers/flat_trace.py index eb8e645472..d51dd2576b 100644 --- a/openfisca_core/tracers/flat_trace.py +++ b/openfisca_core/tracers/flat_trace.py @@ -5,25 +5,24 @@ import numpy +from openfisca_core import tracers from openfisca_core.indexed_enums import EnumArray if typing.TYPE_CHECKING: from numpy.typing import ArrayLike - from openfisca_core.tracers import TraceNode, FullTracer - Array = Union[EnumArray, ArrayLike] Trace = Dict[str, dict] class FlatTrace: - _full_tracer: FullTracer + _full_tracer: tracers.FullTracer - def __init__(self, full_tracer: FullTracer) -> None: + def __init__(self, full_tracer: tracers.FullTracer) -> None: self._full_tracer = full_tracer - def key(self, node: TraceNode) -> str: + def key(self, node: tracers.TraceNode) -> str: name = node.name period = node.period return f"{name}<{period}>" @@ -71,7 +70,7 @@ def serialize( def _get_flat_trace( self, - node: TraceNode, + node: tracers.TraceNode, ) -> Trace: key = self.key(node) diff --git a/openfisca_core/tracers/full_tracer.py b/openfisca_core/tracers/full_tracer.py index b7755683f8..3fa46de5ab 100644 --- a/openfisca_core/tracers/full_tracer.py +++ b/openfisca_core/tracers/full_tracer.py @@ -4,13 +4,7 @@ import typing from typing import Dict, Iterator, List, Optional, Union -from openfisca_core.tracers import ( - ComputationLog, - FlatTrace, - PerformanceLog, - SimpleTracer, - TraceNode, - ) +from .. import tracers if typing.TYPE_CHECKING: from numpy.typing import ArrayLike @@ -22,12 +16,12 @@ class FullTracer: - _simple_tracer: SimpleTracer + _simple_tracer: tracers.SimpleTracer _trees: list - _current_node: Optional[TraceNode] + _current_node: Optional[tracers.TraceNode] def __init__(self) -> None: - self._simple_tracer = SimpleTracer() + self._simple_tracer = tracers.SimpleTracer() self._trees = [] self._current_node = None @@ -45,7 +39,7 @@ def _enter_calculation( variable: str, period: Period, ) -> None: - new_node = TraceNode( + new_node = tracers.TraceNode( name = variable, period = period, parent = self._current_node, @@ -68,7 +62,7 @@ def record_parameter_access( if self._current_node is not None: self._current_node.parameters.append( - TraceNode(name = parameter, period = period, value = value), + tracers.TraceNode(name = parameter, period = period, value = value), ) def _record_start_time( @@ -109,20 +103,20 @@ def stack(self) -> Stack: return self._simple_tracer.stack @property - def trees(self) -> List[TraceNode]: + def trees(self) -> List[tracers.TraceNode]: return self._trees @property - def computation_log(self) -> ComputationLog: - return ComputationLog(self) + def computation_log(self) -> tracers.ComputationLog: + return tracers.ComputationLog(self) @property - def performance_log(self) -> PerformanceLog: - return PerformanceLog(self) + def performance_log(self) -> tracers.PerformanceLog: + return tracers.PerformanceLog(self) @property - def flat_trace(self) -> FlatTrace: - return FlatTrace(self) + def flat_trace(self) -> tracers.FlatTrace: + return tracers.FlatTrace(self) def _get_time_in_sec(self) -> float: return time.time_ns() / (10**9) @@ -136,7 +130,7 @@ def generate_performance_graph(self, dir_path: str) -> None: def generate_performance_tables(self, dir_path: str) -> None: self.performance_log.generate_performance_tables(dir_path) - def _get_nb_requests(self, tree: TraceNode, variable: str) -> int: + def _get_nb_requests(self, tree: tracers.TraceNode, variable: str) -> int: tree_call = tree.name == variable children_calls = sum( self._get_nb_requests(child, variable) @@ -159,7 +153,7 @@ def get_flat_trace(self) -> dict: def get_serialized_flat_trace(self) -> dict: return self.flat_trace.get_serialized_trace() - def browse_trace(self) -> Iterator[TraceNode]: + def browse_trace(self) -> Iterator[tracers.TraceNode]: def _browse_node(node): yield node diff --git a/openfisca_core/tracers/performance_log.py b/openfisca_core/tracers/performance_log.py index cdd66a286b..754d7f8056 100644 --- a/openfisca_core/tracers/performance_log.py +++ b/openfisca_core/tracers/performance_log.py @@ -7,11 +7,9 @@ import os import typing -from openfisca_core.tracers import TraceNode +from .. import tracers if typing.TYPE_CHECKING: - from openfisca_core.tracers import FullTracer - Trace = typing.Dict[str, dict] Calculation = typing.Tuple[str, dict] SortedTrace = typing.List[Calculation] @@ -19,7 +17,7 @@ class PerformanceLog: - def __init__(self, full_tracer: FullTracer) -> None: + def __init__(self, full_tracer: tracers.FullTracer) -> None: self._full_tracer = full_tracer def generate_graph(self, dir_path: str) -> None: @@ -87,10 +85,10 @@ def _aggregate_calculations(calculations: list) -> dict: return { 'calculation_count': calculation_count, - 'calculation_time': TraceNode.round(calculation_time), - 'formula_time': TraceNode.round(formula_time), - 'avg_calculation_time': TraceNode.round(calculation_time / calculation_count), - 'avg_formula_time': TraceNode.round(formula_time / calculation_count), + 'calculation_time': tracers.TraceNode.round(calculation_time), + 'formula_time': tracers.TraceNode.round(formula_time), + 'avg_calculation_time': tracers.TraceNode.round(calculation_time / calculation_count), + 'avg_formula_time': tracers.TraceNode.round(formula_time / calculation_count), } def _groupby(calculation: Calculation) -> str: @@ -114,7 +112,7 @@ def _json(self) -> dict: 'children': children, } - def _json_tree(self, tree: TraceNode) -> dict: + def _json_tree(self, tree: tracers.TraceNode) -> dict: calculation_total_time = tree.calculation_time() children = [self._json_tree(child) for child in tree.children] diff --git a/openfisca_core/tracers/tracing_parameter_node_at_instant.py b/openfisca_core/tracers/tracing_parameter_node_at_instant.py index 9775192740..89d9b8fb01 100644 --- a/openfisca_core/tracers/tracing_parameter_node_at_instant.py +++ b/openfisca_core/tracers/tracing_parameter_node_at_instant.py @@ -5,21 +5,17 @@ import numpy -from openfisca_core.parameters import ( - config, - ParameterNodeAtInstant, - VectorialParameterNodeAtInstant, - ) +from openfisca_core import parameters -if typing.TYPE_CHECKING: - from numpy.typing import ArrayLike +from .. import tracers - from openfisca_core.tracers import FullTracer +ParameterNode = Union[ + parameters.VectorialParameterNodeAtInstant, + parameters.ParameterNodeAtInstant, + ] - ParameterNode = Union[ - VectorialParameterNodeAtInstant, - ParameterNodeAtInstant, - ] +if typing.TYPE_CHECKING: + from numpy.typing import ArrayLike Child = Union[ParameterNode, ArrayLike] @@ -29,7 +25,7 @@ class TracingParameterNodeAtInstant: def __init__( self, parameter_node_at_instant: ParameterNode, - tracer: FullTracer, + tracer: tracers.FullTracer, ) -> None: self.parameter_node_at_instant = parameter_node_at_instant self.tracer = tracer @@ -57,14 +53,14 @@ def get_traced_child( if isinstance( child, - (ParameterNodeAtInstant, VectorialParameterNodeAtInstant), + (parameters.ParameterNodeAtInstant, parameters.VectorialParameterNodeAtInstant), ): return TracingParameterNodeAtInstant(child, self.tracer) if not isinstance(key, str) or \ isinstance( self.parameter_node_at_instant, - VectorialParameterNodeAtInstant, + parameters.VectorialParameterNodeAtInstant, ): # In case of vectorization, we keep the parent node name as, for # instance, rate[status].zone1 is best described as the value of @@ -74,7 +70,7 @@ def get_traced_child( else: name = '.'.join([self.parameter_node_at_instant._name, key]) - if isinstance(child, (numpy.ndarray,) + config.ALLOWED_PARAM_TYPES): + if isinstance(child, (numpy.ndarray,) + parameters.ALLOWED_PARAM_TYPES): self.tracer.record_parameter_access(name, period, child) return child diff --git a/openfisca_core/variables/__init__.py b/openfisca_core/variables/__init__.py index 3decaf8f42..fb36963f7d 100644 --- a/openfisca_core/variables/__init__.py +++ b/openfisca_core/variables/__init__.py @@ -24,3 +24,4 @@ from .config import VALUE_TYPES, FORMULA_NAME_PREFIX # noqa: F401 from .helpers import get_annualized_variable, get_neutralized_variable # noqa: F401 from .variable import Variable # noqa: F401 +from .typing import Formula # noqa: F401 diff --git a/openfisca_core/variables/helpers.py b/openfisca_core/variables/helpers.py index 8dec55d0f8..335a585498 100644 --- a/openfisca_core/variables/helpers.py +++ b/openfisca_core/variables/helpers.py @@ -1,14 +1,14 @@ from __future__ import annotations -import typing import sortedcontainers +from typing import Optional -if typing.TYPE_CHECKING: - from openfisca_core.periods import Period - from openfisca_core.variables import Variable +from openfisca_core.periods import Period +from .. import variables -def get_annualized_variable(variable: Variable, annualization_period: typing.Optional[Period] = None) -> Variable: + +def get_annualized_variable(variable: variables.Variable, annualization_period: Optional[Period] = None) -> variables.Variable: """ Returns a clone of ``variable`` that is annualized for the period ``annualization_period``. When annualized, a variable's formula is only called for a January calculation, and the results for other months are assumed to be identical. diff --git a/openfisca_core/variables/typing.py b/openfisca_core/variables/typing.py new file mode 100644 index 0000000000..892ec0bf9f --- /dev/null +++ b/openfisca_core/variables/typing.py @@ -0,0 +1,16 @@ +from typing import Callable, Union + +import numpy + +from openfisca_core.parameters import ParameterNodeAtInstant +from openfisca_core.periods import Instant, Period +from openfisca_core.populations import Population, GroupPopulation + +#: A collection of :obj:`.Entity` or :obj:`.GroupEntity`. +People = Union[Population, GroupPopulation] + +#: A callable to get the parameters for the given instant. +Params = Callable[[Instant], ParameterNodeAtInstant] + +#: A callable defining a calculation, or a rule, on a system. +Formula = Callable[[People, Period, Params], numpy.ndarray] diff --git a/openfisca_core/variables/variable.py b/openfisca_core/variables/variable.py index 1be200b4be..61a5d9274f 100644 --- a/openfisca_core/variables/variable.py +++ b/openfisca_core/variables/variable.py @@ -10,7 +10,8 @@ from openfisca_core.entities import Entity from openfisca_core.indexed_enums import Enum, EnumArray from openfisca_core.periods import Period -from openfisca_core.variables import config, helpers + +from . import config, helpers class Variable: @@ -19,77 +20,77 @@ class Variable: Main attributes: - .. py:attribute:: name + .. attribute:: name Name of the variable - .. py:attribute:: value_type + .. attribute:: value_type The value type of the variable. Possible value types in OpenFisca are ``int`` ``float`` ``bool`` ``str`` ``date`` and ``Enum``. - .. py:attribute:: entity + .. attribute:: entity `Entity `_ the variable is defined for. For instance : ``Person``, ``Household``. - .. py:attribute:: definition_period + .. attribute:: definition_period `Period `_ the variable is defined for. Possible value: ``MONTH``, ``YEAR``, ``ETERNITY``. - .. py:attribute:: formulas + .. attribute:: formulas Formulas used to calculate the variable - .. py:attribute:: label + .. attribute:: label Description of the variable - .. py:attribute:: reference + .. attribute:: reference Legislative reference describing the variable. - .. py:attribute:: default_value + .. attribute:: default_value `Default value `_ of the variable. Secondary attributes: - .. py:attribute:: baseline_variable + .. attribute:: baseline_variable If the variable has been introduced in a `reform `_ to replace another variable, baseline_variable is the replaced variable. - .. py:attribute:: dtype + .. attribute:: dtype Numpy `dtype `_ used under the hood for the variable. - .. py:attribute:: end + .. attribute:: end `Date `_ when the variable disappears from the legislation. - .. py:attribute:: is_neutralized + .. attribute:: is_neutralized True if the variable is neutralized. Neutralized variables never use their formula, and only return their default values when calculated. - .. py:attribute:: json_type + .. attribute:: json_type JSON type corresponding to the variable. - .. py:attribute:: max_length + .. attribute:: max_length If the value type of the variable is ``str``, max length of the string allowed. ``None`` if there is no limit. - .. py:attribute:: possible_values + .. attribute:: possible_values If the value type of the variable is ``Enum``, contains the values the variable can take. - .. py:attribute:: set_input + .. attribute:: set_input Function used to automatically process variable inputs defined for periods not matching the definition_period of the variable. See more on the `documentation `_. Possible values are ``set_input_dispatch_by_period``, ``set_input_divide_by_period``, or nothing. - .. py:attribute:: unit + .. attribute:: unit Free text field describing the unit of the variable. Only used as metadata. - .. py:attribute:: documentation + .. attribute:: documentation Free multilines text field describing the variable context and usage. """ @@ -308,7 +309,8 @@ def get_formula(self, period = None): If no period is given and the variable has several formula, return the oldest formula. :returns: Formula used to compute the variable - :rtype: function + :rtype: .Formula + """ if not self.formulas: diff --git a/setup.cfg b/setup.cfg index 8c18f3a892..4f98591eeb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,10 +5,11 @@ ; W503/504: We break lines before binary operators (Knuth's style) [flake8] -hang-closing = true -ignore = E128,E251,F403,F405,E501,W503,W504 -in-place = true -rst-roles = any +hang-closing = true +ignore = E128,E251,F403,F405,E501,W503,W504 +in-place = true +rst-roles = any, class, exc, meth, obj +rst-directives = attribute [tool:pytest] addopts = --showlocals --doctest-modules --disable-pytest-warnings diff --git a/setup.py b/setup.py index eb83761916..88e2772a2c 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ 'flake8 >= 3.9.0, < 4.0.0', 'flake8-bugbear >= 19.3.0, < 20.0.0', 'flake8-print >= 3.1.0, < 4.0.0', + 'flake8-rst-docstrings < 1.0.0', 'pytest-cov >= 2.6.1, < 3.0.0', 'mypy >= 0.701, < 0.800', 'openfisca-country-template >= 3.10.0, < 4.0.0', @@ -35,7 +36,7 @@ setup( name = 'OpenFisca-Core', - version = '35.4.2', + version = '35.5.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ diff --git a/tests/fixtures/appclient.py b/tests/fixtures/appclient.py index 32d9e71aee..a140e0f938 100644 --- a/tests/fixtures/appclient.py +++ b/tests/fixtures/appclient.py @@ -6,9 +6,12 @@ @pytest.fixture(scope="module") def test_client(tax_benefit_system): """ This module-scoped fixture creates an API client for the TBS defined in the `tax_benefit_system` - fixture. This `tax_benefit_system` is mutable, so you can add/update variables. Example: + fixture. This `tax_benefit_system` is mutable, so you can add/update variables. + + Example: + + .. code-block:: python - ``` from openfisca_country_template import entities from openfisca_core import periods from openfisca_core.variables import Variable @@ -23,7 +26,7 @@ class new_variable(Variable): tax_benefit_system.add_variable(new_variable) flask_app = app.create_app(tax_benefit_system) - ``` + """ # Create the test API client diff --git a/tests/web_api/test_calculate.py b/tests/web_api/test_calculate.py index 5fb6d702b7..b3415810a7 100644 --- a/tests/web_api/test_calculate.py +++ b/tests/web_api/test_calculate.py @@ -346,7 +346,7 @@ def test_handle_period_mismatch_error(test_client): def test_gracefully_handle_unexpected_errors(test_client): """ Context - ======== + ======= Whenever an exception is raised by the calculation engine, the API will try to handle it and to provide a useful message to the user (4XX). When the @@ -359,7 +359,7 @@ def test_gracefully_handle_unexpected_errors(test_client): Calculate the housing tax due by Bill a thousand years ago. Expected behaviour - ======== + ================== In the `country-template`, Housing Tax is only defined from 2010 onwards. The calculation engine should therefore raise an exception `ParameterNotFound`.