Celery Integration (#31)

* Moved pytest-docker-tools from [tool.poetry.group.test.dependencies] to [tool.poetry.dependencies] * Cleaned pyproject.toml * PYTHONUNBUFFERED=1 and PYTHONDONTWRITEBYTECODE=1 * Minimum Celery version 5.3 instead of 5.3.0b * Set Python 3.11 as default for tox envs: mypy, lint, clean, xdist, parallel * pyupgrade updates * poetry.lock * Added legacy plugin code (<v1.0.0) to plugin discovery/installation * Cleanup * Renamed worker.common to worker.tasks * Dockerfile syntax fix
celery · Sep 28, 2023 · ced94fe · ced94fe
1 parent 23da30b
commit ced94fe
Show file tree

Hide file tree

Showing 11 changed files with 78 additions and 82 deletions.
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -23,10 +23,10 @@ jobs:
       - name: Install apt packages
         run: sudo apt update
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install dependencies
         run: |

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
     rev: v3.13.0
     hooks:
       - id: pyupgrade
-        args: ["--py37-plus"]
+        args: ["--py38-plus"]
 
   - repo: https://github.com/PyCQA/autoflake
     rev: v2.2.1

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -39,7 +39,7 @@ max-line-length = 120
 authors = [
     "Thomas Grainger <pytest-celery@graingert.co.uk>",
     "Omer Katz <omer.katz@omerkatz.com>",
-    "Tomer Nosrati <tomer.nosrati@kcg.tech>"
+    "Tomer Nosrati <tomer.nosrati@kcg.tech>",
 ]
 classifiers = ["License :: OSI Approved :: BSD License"]
 description = "Pytest plugin for Celery"
@@ -50,29 +50,30 @@ version = "1.0.0a1"
 
 [tool.poetry.dependencies]
 python = ">= 3.8,<4.0"
-celery = {version = "^5.3.0b", extras = ["redis"]}
+celery = { version = "^5.3", extras = ["redis"] }
 retry = "^0.9.2"
+pytest-docker-tools = "^3.1.3"
 
 [tool.poetry.group.dev]
 
 [tool.poetry.group.dev.dependencies]
 black = "*"
 toml-sort = "^0.22.1"
 autoflake = [
-    {version = "^1.7.0", python = "<3.8.1"},
-    {version = "^2.0.0", python = ">=3.8.1,<4.0"},
+    { version = "^1.7.0", python = "<3.8.1" },
+    { version = "^2.0.0", python = ">=3.8.1,<4.0" },
 ]
 isort = [
-    {version = "^5.11.0", python = "<3.8.0"},
-    {version = "^5.12.0", python = ">=3.8.0,<4.0"},
+    { version = "^5.11.0", python = "<3.8.0" },
+    { version = "^5.12.0", python = ">=3.8.0,<4.0" },
 ]
 flake8 = [
-    {version = "^5.0.0", python = "<3.8.1"},
-    {version = "^6.0.0", python = ">=3.8.1,<4.0"},
+    { version = "^5.0.0", python = "<3.8.1" },
+    { version = "^6.0.0", python = ">=3.8.1,<4.0" },
 ]
 pre-commit = [
-    {version = "^2.21.0", python = "<3.8.0"},
-    {version = "^3.1.0", python = ">=3.8.0,<4.0"},
+    { version = "^2.21.0", python = "<3.8.0" },
+    { version = "^3.1.0", python = ">=3.8.0,<4.0" },
 ]
 mypy = "^1.0.1"
 types-redis = "^4.5.4.1"
@@ -84,7 +85,6 @@ optional = true
 [tool.poetry.group.test.dependencies]
 coverage = "^7.0.0"
 pytest = "^7.2.0"
-pytest-docker-tools = "^3.1.3"
 pytest-sugar = { version = "^0.9.6", python = ">=3.8,<4.0" }
 pytest-lazy-fixture = "^0.6.3"
 pytest-cov = "^4.0.0"
@@ -98,30 +98,26 @@ optional = true
 
 [tool.poetry.group.ci.dependencies]
 tox = [
-    {version = "^3.0.0", python = "<3.8.1"},
-    {version = "^4.0.0", python = ">=3.8.1,<4.0"},
+    { version = "^3.0.0", python = "<3.8.1" },
+    { version = "^4.0.0", python = ">=3.8.1,<4.0" },
 ]
 
 [tool.poetry.plugins.pytest11]
 celery = "pytest_celery.plugin"
 
-# [tool.pytest.ini_options]
-# addopts = "-p no:celery --runpytest subprocess"
-# pytester_example_dir = "examples"
-
 [tool.pytest.ini_options]
 addopts = [
     "--verbose",
     "--capture=no",
     "--cov=src",
     "--cov-report=xml",
     "--cov-report",
-    "term"
+    "term",
 ]
 minversion = "7.0"
 testpaths = "tests"
 python_classes = "test_*"
-xfail_strict=true
+xfail_strict = true
 
 [tool.mypy]
 warn_unused_configs = true
@@ -130,9 +126,7 @@ follow_imports = "skip"
 show_error_codes = true
 disallow_untyped_defs = true
 ignore_missing_imports = true
-files = [
-    "src",
-]
+files = ["src"]
 
 [tool.coverage.run]
 branch = true
@@ -141,13 +135,5 @@ source = ["src"]
 omit = ["tests"]
 
 [tool.coverage.report]
-exclude_lines = [
-    "pragma: no cover",
-    "if TYPE_CHECKING:",
-    "except ImportError:"
-]
-omit = [
-    "*/python?.?/*",
-    "*/site-packages/*",
-    "*/pypy/*",
-]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "except ImportError:"]
+omit = ["*/python?.?/*", "*/site-packages/*", "*/pypy/*"]
diff --git a/src/pytest_celery/api/setup.py b/src/pytest_celery/api/setup.py
@@ -20,7 +20,7 @@ def __init__(
         self._backend_cluster = backend_cluster
         self._app = app
 
-        from pytest_celery.components.worker.common import ping
+        from pytest_celery.components.worker.tasks import ping
 
         self.ping = ping
 

diff --git a/src/pytest_celery/components/worker/Dockerfile b/src/pytest_celery/components/worker/Dockerfile
@@ -16,6 +16,9 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL
 ENV WORKER_NAME=$CELERY_WORKER_NAME
 ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
 # Install Python dependencies
 RUN pip install --no-cache-dir --upgrade pip \
     && pip install --no-cache-dir celery[redis]==$PIP_VERSION

diff --git a/...pytest_celery/components/worker/common.py → src/pytest_celery/components/worker/tasks.py b/...pytest_celery/components/worker/common.py → src/pytest_celery/components/worker/tasks.py
diff --git a/src/pytest_celery/containers/worker.py b/src/pytest_celery/containers/worker.py
@@ -63,9 +63,9 @@ def initial_content(cls, worker_tasks: set, worker_signals: Union[set, None] = N
 
     @classmethod
     def _initial_content_worker_tasks(cls, worker_tasks: set) -> dict:
-        from pytest_celery.components.worker import common
+        from pytest_celery.components.worker import tasks
 
-        worker_tasks.add(common)
+        worker_tasks.add(tasks)
 
         import_string = ""
 

diff --git a/src/pytest_celery/plugin.py b/src/pytest_celery/plugin.py
@@ -1,6 +1,9 @@
 """
-TBD
+pytest-celery entry point.
 """
 
+# pytest-celery < 1.0.0 infrastructure
+from celery.contrib.pytest import *  # noqa
 
+# pytest-celery >= 1.0.0 infrastructure
 from pytest_celery import *  # noqa
diff --git a/tests/common/celery4/Dockerfile b/tests/common/celery4/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.10-slim-buster
 
 # Create a user to run the worker
 RUN adduser --disabled-password --gecos "" test_user
@@ -16,6 +16,9 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL
 ENV WORKER_NAME=$CELERY_WORKER_NAME
 ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
 # Install Python dependencies
 RUN pip install --no-cache-dir --upgrade pip \
     && pip install --no-cache-dir celery[redis]==$PIP_VERSION

diff --git a/tox.ini b/tox.ini
@@ -22,7 +22,8 @@ description =
 skip_install = true
 allowlist_externals = poetry, pytest, bash, tox
 setenv =
-    PYTHONDONTWRITEBYTECODE=1
+    PYTHONUNBUFFERED = 1
+    PYTHONDONTWRITEBYTECODE = 1
 commands_pre =
     poetry install --with dev,test
 commands =
@@ -35,11 +36,11 @@ basepython =
     3.10: py310
     3.11: py311
     pypy3: pypy3
-    mypy: py310
-    lint: py39
-    clean: py310
-    xdist: py310
-    parallel: py310
+    mypy: py311
+    lint: py311
+    clean: py311
+    xdist: py311
+    parallel: py311
 usedevelop = True
 
 [testenv:xdist]