diff --git a/.github/workflows/data-pipeline-ci.yml b/.github/workflows/data-pipeline-ci.yml index fc0cae65b..43c267748 100644 --- a/.github/workflows/data-pipeline-ci.yml +++ b/.github/workflows/data-pipeline-ci.yml @@ -34,3 +34,5 @@ jobs: run: black --check data-pipeline/src/data_pipeline - name: Run Ruff run: ruff data-pipeline/src/data_pipeline + - name: Run Pyright + run: pyright diff --git a/data-pipeline/check.sh b/data-pipeline/check.sh new file mode 100755 index 000000000..23f14f12a --- /dev/null +++ b/data-pipeline/check.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "┏━━━ Running pyright ━━━━━━━━━━━━━━━━━━━" +pyright + +echo "┏━━━ Running black ━━━━━━━━━━━━━━━━━━━" +black src/data_pipeline +black tests + +echo "┏━━━ Running ruff ━━━━━━━━━━━━━━━━━━━" +ruff src/data_pipeline +ruff tests --fix + +echo "┏━━━ Running pytest ━━━━━━━━━━━━━━━━━━━" +pytest -m only diff --git a/data-pipeline/poetry.lock b/data-pipeline/poetry.lock index 7c16f69f9..7c8ed50a5 100644 --- a/data-pipeline/poetry.lock +++ b/data-pipeline/poetry.lock @@ -911,6 +911,17 @@ category = "dev" optional = false python-versions = ">=3.5" +[[package]] +name = "nodeenv" +version = "1.8.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" + +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.25.2" @@ -1256,6 +1267,21 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] +[[package]] +name = "pyright" +version = "1.1.327" +description = "Command line wrapper for pyright" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +nodeenv = ">=1.6.0" + +[package.extras] +all = ["twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] + [[package]] name = "pyspark" version = "3.3.3" @@ -1440,6 +1466,19 @@ dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +[[package]] +name = "setuptools" +version = "68.2.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.8" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -1649,7 +1688,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "cb9f408003a3e06698cf56c3214b01af7784e09a33970322d5901cc9804e928a" +content-hash = "c02345219ed0dad93461e529a731a2bee3c0a63288392c5786869d5ab67d4c11" [metadata.files] aiodns = [ @@ -2509,6 +2548,10 @@ nest-asyncio = [ {file = "nest_asyncio-1.5.8-py3-none-any.whl", hash = "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"}, {file = "nest_asyncio-1.5.8.tar.gz", hash = "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb"}, ] +nodeenv = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] numpy = [ {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, @@ -2858,6 +2901,10 @@ PyJWT = [ {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] +pyright = [ + {file = "pyright-1.1.327-py3-none-any.whl", hash = "sha256:3462cda239e9140276238bbdbd0b59d77406f1c2e14d8cb8c20c8e25639c6b3c"}, + {file = "pyright-1.1.327.tar.gz", hash = "sha256:ba74148ad64f22020dbbed6781c4bdb38ecb8a7ca90dc3c87a4f08d1c0e11592"}, +] pyspark = [ {file = "pyspark-3.3.3.tar.gz", hash = "sha256:384d2ad7090cd1db5b2d2ac497bda409d86ab3a27272833e1a27efadf45e4d2f"}, ] @@ -3097,6 +3144,10 @@ scipy = [ {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, ] +setuptools = [ + {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, + {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/data-pipeline/pyproject.toml b/data-pipeline/pyproject.toml index d9d567bcd..4c169a487 100644 --- a/data-pipeline/pyproject.toml +++ b/data-pipeline/pyproject.toml @@ -19,6 +19,7 @@ pytest = "^7.4.2" ipython = "^8.15.0" ruff = "^0.0.290" black = "^23.9.1" +pyright = "^1.1.327" [build-system] requires = ["poetry-core"] @@ -30,3 +31,19 @@ line-length = 120 [tool.ruff] line-length = 120 +[tool.pyright] +include = [ +"src/data_pipeline/pipeline.py", +"src/data_pipeline/config.py", +"src/data_pipeline/helpers/write_schemas.py", +"src/data_pipeline/helpers/logging.py", +"src/data_pipeline/pipelines/gnomad_v4_variants.py", +"src/data_pipeline/pipelines/gnomad_v4_coverage.py", +# "src/data_pipeline/datasets/gnomad_v4", +"tests" +] +reportMissingImports = true +reportMissingTypeStubs = false +typeCheckingMode = "basic" +pythonVersion = "3.9" + diff --git a/data-pipeline/pytest.ini b/data-pipeline/pytest.ini new file mode 100644 index 000000000..f0593b3a9 --- /dev/null +++ b/data-pipeline/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +testpaths = + tests +addopts = --strict -W ignore -v -s --durations=0 +markers = + only: marked with "only" + long: takes a long time + broken: test is broken diff --git a/data-pipeline/requirements.txt b/data-pipeline/requirements.txt index 3f2112239..d56f745f7 100644 --- a/data-pipeline/requirements.txt +++ b/data-pipeline/requirements.txt @@ -790,6 +790,9 @@ mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0" \ nest-asyncio==1.5.8 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb \ --hash=sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d +nodeenv==1.8.0 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2 \ + --hash=sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec numpy==1.25.2 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2 \ --hash=sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55 \ @@ -1113,6 +1116,9 @@ pygments==2.16.1 ; python_version >= "3.9" and python_version < "4.0" \ pyjwt[crypto]==2.8.0 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 +pyright==1.1.327 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:3462cda239e9140276238bbdbd0b59d77406f1c2e14d8cb8c20c8e25639c6b3c \ + --hash=sha256:ba74148ad64f22020dbbed6781c4bdb38ecb8a7ca90dc3c87a4f08d1c0e11592 pyspark==3.3.3 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:384d2ad7090cd1db5b2d2ac497bda409d86ab3a27272833e1a27efadf45e4d2f pytest==7.4.2 ; python_version >= "3.9" and python_version < "4.0" \ @@ -1337,6 +1343,9 @@ scipy==1.9.3 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c \ --hash=sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95 \ --hash=sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027 +setuptools==68.2.2 ; python_version >= "3.9" and python_version < "4.0" \ + --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ + --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a six==1.16.0 ; python_version >= "3.9" and python_version < "4.0" \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254