Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: Support for dynamic worker args #203

Merged
merged 2 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/userguide/examples/django.rst
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ and `container <https://github.com/Jc2k/pytest-docker-tools?tab=readme-ov-file#c
},
wrapper_class=DjangoWorkerContainer,
timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)

In this case, we also mount the project directory to ``/src`` in the container, so that we can install the project
Expand Down
1 change: 1 addition & 0 deletions examples/django/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ def worker_queue(cls) -> str:
},
wrapper_class=DjangoWorkerContainer,
timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)


Expand Down
3 changes: 0 additions & 3 deletions src/pytest_celery/api/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,6 @@ def command(cls, *args: str) -> list[str]:
the first element, followed by the command-line arguments.
"""

# To be used with pytest_docker_tools.container using the command
# kwarg with the class method as value
# e.g. command=MyContainer.command()
raise NotImplementedError("CeleryTestContainer.command")

def teardown(self) -> None:
Expand Down
15 changes: 14 additions & 1 deletion src/pytest_celery/vendors/redis/backend/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,23 @@
network="{default_pytest_celery_network.name}",
wrapper_class=RedisContainer,
timeout=REDIS_CONTAINER_TIMEOUT,
command=RedisContainer.command("--maxclients", "100000"),
command=fxtr("default_redis_backend_command"),
)


@pytest.fixture

Check warning on line 57 in src/pytest_celery/vendors/redis/backend/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/redis/backend/fixtures.py#L57

Added line #L57 was not covered by tests
def default_redis_backend_command(default_redis_backend_cls: type[RedisContainer]) -> list[str]:
"""Command to run the container.

Args:
default_redis_backend_cls (type[RedisContainer]): See also: :ref:`vendor-class`.

Returns:
list[str]: Docker CMD instruction.
"""
return default_redis_backend_cls.command("--maxclients", "100000")

Check warning on line 67 in src/pytest_celery/vendors/redis/backend/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/redis/backend/fixtures.py#L67

Added line #L67 was not covered by tests


@pytest.fixture
def default_redis_backend_env(default_redis_backend_cls: type[RedisContainer]) -> dict:
"""Environment variables for this vendor.
Expand Down
15 changes: 14 additions & 1 deletion src/pytest_celery/vendors/redis/broker/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,23 @@
network="{default_pytest_celery_network.name}",
wrapper_class=RedisContainer,
timeout=REDIS_CONTAINER_TIMEOUT,
command=RedisContainer.command("--maxclients", "100000"),
command=fxtr("default_redis_broker_command"),
)


@pytest.fixture

Check warning on line 57 in src/pytest_celery/vendors/redis/broker/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/redis/broker/fixtures.py#L57

Added line #L57 was not covered by tests
def default_redis_broker_command(default_redis_broker_cls: type[RedisContainer]) -> list[str]:
"""Command to run the container.

Args:
default_redis_broker_cls (type[RedisContainer]): See also: :ref:`vendor-class`.

Returns:
list[str]: Docker CMD instruction.
"""
return default_redis_broker_cls.command("--maxclients", "100000")

Check warning on line 67 in src/pytest_celery/vendors/redis/broker/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/redis/broker/fixtures.py#L67

Added line #L67 was not covered by tests


@pytest.fixture
def default_redis_broker_env(default_redis_broker_cls: type[RedisContainer]) -> dict:
"""Environment variables for this vendor.
Expand Down
16 changes: 16 additions & 0 deletions src/pytest_celery/vendors/worker/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,22 @@
dependencies of your project.
"""

@classmethod

Check warning on line 35 in src/pytest_celery/vendors/worker/container.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/worker/container.py#L35

Added line #L35 was not covered by tests
def command(cls, *args: str) -> list[str]:
args = args or tuple()
return [

Check warning on line 38 in src/pytest_celery/vendors/worker/container.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/worker/container.py#L37-L38

Added lines #L37 - L38 were not covered by tests
"celery",
"-A",
"app",
"worker",
f"--loglevel={cls.log_level()}",
"-n",
f"{cls.worker_name()}@%h",
"-Q",
f"{cls.worker_queue()}",
*args,
]

def _wait_port(self, port: str) -> int:
# Not needed for worker container
raise NotImplementedError
Expand Down
16 changes: 15 additions & 1 deletion src/pytest_celery/vendors/worker/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
wrapper_class=CeleryWorkerContainer,
timeout=DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)

celery_base_worker_image = build(
Expand Down Expand Up @@ -161,6 +162,19 @@
return default_worker_container_session_cls.worker_queue()


@pytest.fixture

Check warning on line 165 in src/pytest_celery/vendors/worker/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/worker/fixtures.py#L165

Added line #L165 was not covered by tests
def default_worker_command(default_worker_container_cls: type[CeleryWorkerContainer]) -> list[str]:
"""Command to run the container.

Args:
default_worker_container_cls (type[CeleryWorkerContainer]): See also: :ref:`vendor-class`.

Returns:
list[str]: Docker CMD instruction.
"""
return default_worker_container_cls.command()

Check warning on line 175 in src/pytest_celery/vendors/worker/fixtures.py

View check run for this annotation

Codecov / codecov/patch

src/pytest_celery/vendors/worker/fixtures.py#L175

Added line #L175 was not covered by tests


@pytest.fixture
def default_worker_env(
default_worker_container_cls: type[CeleryWorkerContainer],
Expand Down Expand Up @@ -201,7 +215,7 @@

.. note::

Move volumes may be added additionally.
More volumes may be added additionally.

Args:
default_worker_container_cls (type[CeleryWorkerContainer]): See also: :ref:`vendor-class`.
Expand Down
1 change: 1 addition & 0 deletions tests/integration/api/custom_setup/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ def default_worker_container_session_cls() -> type[CeleryWorkerContainer]:
volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
wrapper_class=Celery5WorkerContainer,
timeout=DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)


Expand Down
1 change: 1 addition & 0 deletions tests/integration/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,5 @@ def default_worker_container_session_cls() -> type[CeleryWorkerContainer]:
volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
wrapper_class=IntegrationWorkerContainer,
timeout=DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)
8 changes: 8 additions & 0 deletions tests/integration/vendors/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ def default_worker_app_module(self, request: pytest.FixtureRequest) -> ModuleTyp
def test_replacing_app_module(self, container: CeleryWorkerContainer, default_worker_app_module: ModuleType):
assert container.app_module() == default_worker_app_module

class test_replacing_command:
@pytest.fixture
def default_worker_command(self, default_worker_command: list[str]) -> list[str]:
return [*default_worker_command, "--pool", "solo"]

def test_replacing_command(self, container: CeleryWorkerContainer):
assert container.logs().count("solo") == 1


class test_base_test_worker:
def test_config(self, celery_setup_worker: CeleryTestWorker):
Expand Down
1 change: 1 addition & 0 deletions tests/smoke/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def default_worker_container_session_cls() -> type[CeleryWorkerContainer]:
volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
wrapper_class=SmokeWorkerContainer,
timeout=DEFAULT_WORKER_CONTAINER_TIMEOUT,
command=fxtr("default_worker_command"),
)


Expand Down
24 changes: 24 additions & 0 deletions tests/smoke/test_worker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
from __future__ import annotations

import pytest

from pytest_celery import CeleryTestSetup


@pytest.mark.parametrize(
"pool",
[
"solo",
"prefork",
"threads",
],
)
class test_replacing_pool:
@pytest.fixture
def default_worker_command(self, default_worker_command: list[str], pool: str) -> list[str]:
return [*default_worker_command, "--pool", pool]

def test_pool_from_celery_banner(self, celery_setup: CeleryTestSetup, pool: str):
if pool == "threads":
pool = "thread"
celery_setup.worker.assert_log_exists(pool)
Loading