From 8ace44c83ecbb3059d4c00273b45cd28803eacd0 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 24 Mar 2024 14:07:52 +0200 Subject: [PATCH] Hotfix: New default worker tasks --- docs/userguide/default-tasks.rst | 110 ++++++++++++++ docs/userguide/index.rst | 1 + src/pytest_celery/__init__.py | 2 +- src/pytest_celery/vendors/worker/Dockerfile | 3 +- src/pytest_celery/vendors/worker/tasks.py | 139 ++++++++++++++++++ .../integration/vendors/test_default_tasks.py | 49 ++++++ tests/tasks.py | 24 +-- .../vendors/test_worker/test_default_tasks.py | 45 ++++++ 8 files changed, 348 insertions(+), 25 deletions(-) create mode 100644 docs/userguide/default-tasks.rst create mode 100644 tests/integration/vendors/test_default_tasks.py create mode 100644 tests/unit/vendors/test_worker/test_default_tasks.py diff --git a/docs/userguide/default-tasks.rst b/docs/userguide/default-tasks.rst new file mode 100644 index 00000000..8f28c136 --- /dev/null +++ b/docs/userguide/default-tasks.rst @@ -0,0 +1,110 @@ +.. _default-tasks: + +================ + Built-in Tasks +================ + +:Release: |version| +:Date: |today| + +The plugin provides a list of built-in celery tasks that can be used out of the box. This page will +list all the available tasks. + +.. tip:: + + The tasks injected into the workers that use the default volume with: + + .. code-block:: python + + volumes={"{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME}, + +.. contents:: + :local: + :depth: 1 + +add +=== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- add ------------- + :end-before: # ------------- add_replaced ------------- + +add_replaced +============ + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- add_replaced ------------- + :end-before: # ------------- fail ------------- + +fail +==== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- fail ------------- + :end-before: # ------------- identity ------------- + +identity +======== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- identity ------------- + :end-before: # ------------- noop ------------- + +noop +==== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- noop ------------- + :end-before: # ------------- ping ------------- + +ping +==== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- ping ------------- + :end-before: # ------------- sleep ------------- + +sleep +===== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- sleep ------------- + :end-before: # ------------- xsum ------------- + +xsum +==== + +.. versionadded:: 1.0.0 + +.. literalinclude:: ../../src/pytest_celery/vendors/worker/tasks.py + :language: python + :caption: pytest_celery.vendors.worker.tasks + :start-after: # ------------- xsum ------------- diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index 680d43f5..598d3907 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -19,6 +19,7 @@ of the plugin before diving into the advanced features. app-conf utils-module tasks + default-tasks signals celery-bug-report examples/index diff --git a/src/pytest_celery/__init__.py b/src/pytest_celery/__init__.py index 6f9774ae..cff98d75 100644 --- a/src/pytest_celery/__init__.py +++ b/src/pytest_celery/__init__.py @@ -107,7 +107,7 @@ from pytest_celery.vendors.worker.fixtures import default_worker_tasks from pytest_celery.vendors.worker.fixtures import default_worker_utils_module from pytest_celery.vendors.worker.fixtures import default_worker_volume - from pytest_celery.vendors.worker.tasks import ping + from pytest_celery.vendors.worker.tasks import * from pytest_celery.vendors.worker.volume import WorkerInitialContent diff --git a/src/pytest_celery/vendors/worker/Dockerfile b/src/pytest_celery/vendors/worker/Dockerfile index 071a3fd0..cce7671a 100644 --- a/src/pytest_celery/vendors/worker/Dockerfile +++ b/src/pytest_celery/vendors/worker/Dockerfile @@ -23,7 +23,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 RUN pip install --no-cache-dir --upgrade \ pip \ celery[redis,pymemcache]${WORKER_VERSION:+==$WORKER_VERSION} \ - pytest-celery@git+https://github.com/celery/pytest-celery.git + pytest-celery@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@hotfix +# pytest-celery@git+https://github.com/celery/pytest-celery.git@hotfix # The workdir must be /app WORKDIR /app diff --git a/src/pytest_celery/vendors/worker/tasks.py b/src/pytest_celery/vendors/worker/tasks.py index 254477e0..701fe68f 100644 --- a/src/pytest_celery/vendors/worker/tasks.py +++ b/src/pytest_celery/vendors/worker/tasks.py @@ -4,13 +4,152 @@ This module is part of the :ref:`built-in-worker` vendor. """ +from __future__ import annotations + +import time +from typing import Any +from typing import Iterable + +import celery.utils +from celery import Task from celery import shared_task +# ------------- add ------------- +@shared_task +def add(x: int | float, y: int | float, z: int | float | None = None) -> int | float: + """Pytest-celery internal task. + + This task adds two or three numbers together. + + Args: + x (int | float): The first number. + y (int | float): The second number. + z (int | float | None, optional): The third number. Defaults to None. + + Returns: + int | float: The sum of the numbers. + """ + if z: + return x + y + z + else: + return x + y + + +# ------------- add_replaced ------------- +@shared_task(bind=True) +def add_replaced( + self: Task, + x: int | float, + y: int | float, + z: int | float | None = None, + *, + queue: str | None = None, +) -> None: + """Pytest-celery internal task. + + This task replaces itself with the add task for the given arguments. + + Args: + x (int | float): The first number. + y (int | float): The second number. + z (int | float | None, optional): The third number. Defaults to None. + + Raises: + Ignore: Always raises Ignore. + """ + queue = queue or "celery" + raise self.replace(add.s(x, y, z).set(queue=queue)) + + +# ------------- fail ------------- +@shared_task +def fail(*args: tuple) -> None: + """Pytest-celery internal task. + + This task raises a RuntimeError with the given arguments. + + Args: + *args (tuple): Arguments to pass to the RuntimeError. + + Raises: + RuntimeError: Always raises a RuntimeError. + """ + args = (("Task expected to fail",) + args,) + raise RuntimeError(*args) + + +# ------------- identity ------------- +@shared_task +def identity(x: Any) -> Any: + """Pytest-celery internal task. + + This task returns the input as is. + + Args: + x (Any): Any value. + + Returns: + Any: The input value. + """ + return x + + +# ------------- noop ------------- +@shared_task +def noop(*args: tuple, **kwargs: dict) -> None: + """Pytest-celery internal task. + + This is a no-op task that does nothing. + + Returns: + None: Always returns None. + """ + return celery.utils.noop(*args, **kwargs) + + +# ------------- ping ------------- @shared_task def ping() -> str: """Pytest-celery internal task. Used to check if the worker is up and running. + + Returns: + str: Always returns "pong". """ return "pong" + + +# ------------- sleep ------------- +@shared_task +def sleep(seconds: float = 1, **kwargs: dict) -> bool: + """Pytest-celery internal task. + + This task sleeps for the given number of seconds. + + Args: + seconds (float, optional): The number of seconds to sleep. Defaults to 1. + **kwargs (dict): Additional keyword arguments. + + Returns: + bool: Always returns True. + """ + time.sleep(seconds, **kwargs) + return True + + +# ------------- xsum ------------- +@shared_task +def xsum(nums: Iterable) -> int: + """Pytest-celery internal task. + + This task sums a list of numbers, but also supports nested lists. + + Args: + nums (Iterable): A list of numbers or nested lists. + + Returns: + int: The sum of the numbers. + """ + return sum(sum(num) if isinstance(num, Iterable) else num for num in nums) diff --git a/tests/integration/vendors/test_default_tasks.py b/tests/integration/vendors/test_default_tasks.py new file mode 100644 index 00000000..797ced29 --- /dev/null +++ b/tests/integration/vendors/test_default_tasks.py @@ -0,0 +1,49 @@ +import pytest + +from pytest_celery import RESULT_TIMEOUT +from pytest_celery import CeleryTestSetup +from pytest_celery import add +from pytest_celery import add_replaced +from pytest_celery import fail +from pytest_celery import identity +from pytest_celery import noop +from pytest_celery import ping +from pytest_celery import sleep +from pytest_celery import xsum + + +class test_default_tasks: + def test_add(self, celery_setup: CeleryTestSetup): + assert add.s(1, 2).apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) == 3 + + def test_add_replaced(self, celery_setup: CeleryTestSetup): + queue = celery_setup.worker.worker_queue + add_replaced.s(1, 2, queue=queue).apply_async(queue=queue) + celery_setup.worker.assert_log_exists("ignored") + + def test_fail(self, celery_setup: CeleryTestSetup): + with pytest.raises(RuntimeError): + fail.s().apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) + + def test_identity(self, celery_setup: CeleryTestSetup): + assert identity.s(1).apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) == 1 + + def test_noop(self, celery_setup: CeleryTestSetup): + assert noop.s().apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) is None + + def test_ping(self, celery_setup: CeleryTestSetup): + assert ping.s().apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) == "pong" + + def test_sleep(self, celery_setup: CeleryTestSetup): + assert sleep.s().apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) is True + + def test_xsum(self, celery_setup: CeleryTestSetup): + assert xsum.s([1, 2, 3]).apply_async(queue=celery_setup.worker.worker_queue).get(timeout=RESULT_TIMEOUT) == 6 + + def test_xsum_nested_list(self, celery_setup: CeleryTestSetup): + assert ( + xsum.s([[1, 2], [3, 4], [5, 6]]) + .apply_async(queue=celery_setup.worker.worker_queue) + .get(timeout=RESULT_TIMEOUT) + == 21 + ) diff --git a/tests/tasks.py b/tests/tasks.py index fe5eb78f..53797a90 100644 --- a/tests/tasks.py +++ b/tests/tasks.py @@ -1,31 +1,9 @@ -import time - -import celery.utils from celery import Task from celery import shared_task from celery import signature from celery.canvas import Signature - -@shared_task -def noop(*args, **kwargs) -> None: - return celery.utils.noop(*args, **kwargs) - - -@shared_task -def identity(x): - return x - - -@shared_task -def sleep(seconds: float = 1, **kwargs) -> True: - time.sleep(seconds, **kwargs) - return True - - -@shared_task -def add(x, y): - return x + y +from pytest_celery.vendors.worker.tasks import * # noqa @shared_task diff --git a/tests/unit/vendors/test_worker/test_default_tasks.py b/tests/unit/vendors/test_worker/test_default_tasks.py new file mode 100644 index 00000000..703389d3 --- /dev/null +++ b/tests/unit/vendors/test_worker/test_default_tasks.py @@ -0,0 +1,45 @@ +from unittest.mock import patch + +import pytest +from celery.exceptions import Ignore + +from pytest_celery import add +from pytest_celery import add_replaced +from pytest_celery import fail +from pytest_celery import identity +from pytest_celery import noop +from pytest_celery import ping +from pytest_celery import sleep +from pytest_celery import xsum + + +class test_default_tasks: + def test_add(self): + assert add(1, 2) == 3 + + def test_add_replaced(self): + with patch("pytest_celery.add_replaced.replace", side_effect=Ignore): + with pytest.raises(Ignore): + add_replaced(1, 2) + + def test_fail(self): + with pytest.raises(RuntimeError): + fail() + + def test_identity(self): + assert identity(1) == 1 + + def test_noop(self): + assert noop() is None + + def test_ping(self): + assert ping() == "pong" + + def test_sleep(self): + assert sleep() is True + + def test_xsum(self): + assert xsum([1, 2, 3]) == 6 + + def test_xsum_nested_list(self): + assert xsum([[1, 2], [3, 4], [5, 6]]) == 21