From ced94fec5d2c079aaeb964395bd87c2a28d3b89e Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Thu, 28 Sep 2023 15:47:40 +0300 Subject: [PATCH] Celery Integration (#31) * Moved pytest-docker-tools from [tool.poetry.group.test.dependencies] to [tool.poetry.dependencies] * Cleaned pyproject.toml * PYTHONUNBUFFERED=1 and PYTHONDONTWRITEBYTECODE=1 * Minimum Celery version 5.3 instead of 5.3.0b * Set Python 3.11 as default for tox envs: mypy, lint, clean, xdist, parallel * pyupgrade updates * poetry.lock * Added legacy plugin code ( tasks.py} | 0 src/pytest_celery/containers/worker.py | 4 +- src/pytest_celery/plugin.py | 5 +- tests/common/celery4/Dockerfile | 5 +- tox.ini | 13 ++-- 11 files changed, 78 insertions(+), 82 deletions(-) rename src/pytest_celery/components/worker/{common.py => tasks.py} (100%) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index ef0da47f..f01a2881 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -23,10 +23,10 @@ jobs: - name: Install apt packages run: sudo apt update - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6027b39..879dc55d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: rev: v3.13.0 hooks: - id: pyupgrade - args: ["--py37-plus"] + args: ["--py38-plus"] - repo: https://github.com/PyCQA/autoflake rev: v2.2.1 diff --git a/poetry.lock b/poetry.lock index 63eda903..d93ebf62 100644 --- a/poetry.lock +++ b/poetry.lock @@ -579,34 +579,34 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, ] [package.dependencies] @@ -1512,13 +1512,13 @@ cryptography = ">=35.0.0" [[package]] name = "types-redis" -version = "4.6.0.6" +version = "4.6.0.7" description = "Typing stubs for redis" optional = false python-versions = "*" files = [ - {file = "types-redis-4.6.0.6.tar.gz", hash = "sha256:7865a843802937ab2ddca33579c4e255bfe73f87af85824ead7a6729ba92fc52"}, - {file = "types_redis-4.6.0.6-py3-none-any.whl", hash = "sha256:e0e9dcc530623db3a41ec058ccefdcd5c7582557f02ab5f7aa9a27fe10a78d7e"}, + {file = "types-redis-4.6.0.7.tar.gz", hash = "sha256:28c4153ddb5c9d4f10def44a2454673c361d2d5fc3cd867cf3bb1520f3f59a38"}, + {file = "types_redis-4.6.0.7-py3-none-any.whl", hash = "sha256:05b1bf92879b25df20433fa1af07784a0d7928c616dc2ebf9087618db77ccbd0"}, ] [package.dependencies] @@ -1527,13 +1527,13 @@ types-pyOpenSSL = "*" [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] @@ -1549,13 +1549,13 @@ files = [ [[package]] name = "urllib3" -version = "2.0.4" +version = "2.0.5" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, - {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, ] [package.extras] @@ -1625,4 +1625,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = ">= 3.8,<4.0" -content-hash = "3ed2b17cf1ef5a0ef2c2bf26a27e46d1dea2f832314f19e0a44c574f0adc4ec1" +content-hash = "1041b0274288477324fd4404bbb6dec9ba1b4f8dc3e9cac8b3a18bf3c8d79230" diff --git a/pyproject.toml b/pyproject.toml index 7b63af58..38bc4e4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ max-line-length = 120 authors = [ "Thomas Grainger ", "Omer Katz ", - "Tomer Nosrati " + "Tomer Nosrati ", ] classifiers = ["License :: OSI Approved :: BSD License"] description = "Pytest plugin for Celery" @@ -50,8 +50,9 @@ version = "1.0.0a1" [tool.poetry.dependencies] python = ">= 3.8,<4.0" -celery = {version = "^5.3.0b", extras = ["redis"]} +celery = { version = "^5.3", extras = ["redis"] } retry = "^0.9.2" +pytest-docker-tools = "^3.1.3" [tool.poetry.group.dev] @@ -59,20 +60,20 @@ retry = "^0.9.2" black = "*" toml-sort = "^0.22.1" autoflake = [ - {version = "^1.7.0", python = "<3.8.1"}, - {version = "^2.0.0", python = ">=3.8.1,<4.0"}, + { version = "^1.7.0", python = "<3.8.1" }, + { version = "^2.0.0", python = ">=3.8.1,<4.0" }, ] isort = [ - {version = "^5.11.0", python = "<3.8.0"}, - {version = "^5.12.0", python = ">=3.8.0,<4.0"}, + { version = "^5.11.0", python = "<3.8.0" }, + { version = "^5.12.0", python = ">=3.8.0,<4.0" }, ] flake8 = [ - {version = "^5.0.0", python = "<3.8.1"}, - {version = "^6.0.0", python = ">=3.8.1,<4.0"}, + { version = "^5.0.0", python = "<3.8.1" }, + { version = "^6.0.0", python = ">=3.8.1,<4.0" }, ] pre-commit = [ - {version = "^2.21.0", python = "<3.8.0"}, - {version = "^3.1.0", python = ">=3.8.0,<4.0"}, + { version = "^2.21.0", python = "<3.8.0" }, + { version = "^3.1.0", python = ">=3.8.0,<4.0" }, ] mypy = "^1.0.1" types-redis = "^4.5.4.1" @@ -84,7 +85,6 @@ optional = true [tool.poetry.group.test.dependencies] coverage = "^7.0.0" pytest = "^7.2.0" -pytest-docker-tools = "^3.1.3" pytest-sugar = { version = "^0.9.6", python = ">=3.8,<4.0" } pytest-lazy-fixture = "^0.6.3" pytest-cov = "^4.0.0" @@ -98,17 +98,13 @@ optional = true [tool.poetry.group.ci.dependencies] tox = [ - {version = "^3.0.0", python = "<3.8.1"}, - {version = "^4.0.0", python = ">=3.8.1,<4.0"}, + { version = "^3.0.0", python = "<3.8.1" }, + { version = "^4.0.0", python = ">=3.8.1,<4.0" }, ] [tool.poetry.plugins.pytest11] celery = "pytest_celery.plugin" -# [tool.pytest.ini_options] -# addopts = "-p no:celery --runpytest subprocess" -# pytester_example_dir = "examples" - [tool.pytest.ini_options] addopts = [ "--verbose", @@ -116,12 +112,12 @@ addopts = [ "--cov=src", "--cov-report=xml", "--cov-report", - "term" + "term", ] minversion = "7.0" testpaths = "tests" python_classes = "test_*" -xfail_strict=true +xfail_strict = true [tool.mypy] warn_unused_configs = true @@ -130,9 +126,7 @@ follow_imports = "skip" show_error_codes = true disallow_untyped_defs = true ignore_missing_imports = true -files = [ - "src", -] +files = ["src"] [tool.coverage.run] branch = true @@ -141,13 +135,5 @@ source = ["src"] omit = ["tests"] [tool.coverage.report] -exclude_lines = [ - "pragma: no cover", - "if TYPE_CHECKING:", - "except ImportError:" -] -omit = [ - "*/python?.?/*", - "*/site-packages/*", - "*/pypy/*", -] +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "except ImportError:"] +omit = ["*/python?.?/*", "*/site-packages/*", "*/pypy/*"] diff --git a/src/pytest_celery/api/setup.py b/src/pytest_celery/api/setup.py index 5d536280..347ca42f 100644 --- a/src/pytest_celery/api/setup.py +++ b/src/pytest_celery/api/setup.py @@ -20,7 +20,7 @@ def __init__( self._backend_cluster = backend_cluster self._app = app - from pytest_celery.components.worker.common import ping + from pytest_celery.components.worker.tasks import ping self.ping = ping diff --git a/src/pytest_celery/components/worker/Dockerfile b/src/pytest_celery/components/worker/Dockerfile index 86c1c95f..6ead4332 100644 --- a/src/pytest_celery/components/worker/Dockerfile +++ b/src/pytest_celery/components/worker/Dockerfile @@ -16,6 +16,9 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + # Install Python dependencies RUN pip install --no-cache-dir --upgrade pip \ && pip install --no-cache-dir celery[redis]==$PIP_VERSION diff --git a/src/pytest_celery/components/worker/common.py b/src/pytest_celery/components/worker/tasks.py similarity index 100% rename from src/pytest_celery/components/worker/common.py rename to src/pytest_celery/components/worker/tasks.py diff --git a/src/pytest_celery/containers/worker.py b/src/pytest_celery/containers/worker.py index 70125886..275c7f56 100644 --- a/src/pytest_celery/containers/worker.py +++ b/src/pytest_celery/containers/worker.py @@ -63,9 +63,9 @@ def initial_content(cls, worker_tasks: set, worker_signals: Union[set, None] = N @classmethod def _initial_content_worker_tasks(cls, worker_tasks: set) -> dict: - from pytest_celery.components.worker import common + from pytest_celery.components.worker import tasks - worker_tasks.add(common) + worker_tasks.add(tasks) import_string = "" diff --git a/src/pytest_celery/plugin.py b/src/pytest_celery/plugin.py index 29888566..ffdc740d 100644 --- a/src/pytest_celery/plugin.py +++ b/src/pytest_celery/plugin.py @@ -1,6 +1,9 @@ """ -TBD +pytest-celery entry point. """ +# pytest-celery < 1.0.0 infrastructure +from celery.contrib.pytest import * # noqa +# pytest-celery >= 1.0.0 infrastructure from pytest_celery import * # noqa diff --git a/tests/common/celery4/Dockerfile b/tests/common/celery4/Dockerfile index 9e20faa2..32a327f5 100644 --- a/tests/common/celery4/Dockerfile +++ b/tests/common/celery4/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-buster +FROM python:3.10-slim-buster # Create a user to run the worker RUN adduser --disabled-password --gecos "" test_user @@ -16,6 +16,9 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + # Install Python dependencies RUN pip install --no-cache-dir --upgrade pip \ && pip install --no-cache-dir celery[redis]==$PIP_VERSION diff --git a/tox.ini b/tox.ini index a919958f..db71f628 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,8 @@ description = skip_install = true allowlist_externals = poetry, pytest, bash, tox setenv = - PYTHONDONTWRITEBYTECODE=1 + PYTHONUNBUFFERED = 1 + PYTHONDONTWRITEBYTECODE = 1 commands_pre = poetry install --with dev,test commands = @@ -35,11 +36,11 @@ basepython = 3.10: py310 3.11: py311 pypy3: pypy3 - mypy: py310 - lint: py39 - clean: py310 - xdist: py310 - parallel: py310 + mypy: py311 + lint: py311 + clean: py311 + xdist: py311 + parallel: py311 usedevelop = True [testenv:xdist]