Refactored Dependecies Management (#225)

* Removed "-n auto" from examples CI

* Bugfix: Added pytest-celery package to built-in Celery worker Dockerfile

* Refactored Dependecies Management

* Disabled Memcached automatic detection due to "Experimental Status"

.github/workflows/examples.yml

@@ -11,12 +11,16 @@ on:
       - "**.txt"
       - ".github/workflows/examples.yml"
       - "**.toml"
+      - 'src/pytest_celery/vendors/worker/**'
+      - 'Dockerfile'
   pull_request:
     paths:
       - "**.py"
       - "**.txt"
       - "**.toml"
       - ".github/workflows/examples.yml"
+      - 'src/pytest_celery/vendors/worker/**'
+      - 'Dockerfile'
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -53,7 +57,7 @@ jobs:
         working-directory: examples/myworker
         timeout-minutes: 10
         run: |
-          pytest -vv tests -n auto
+          pytest -vv tests
 
   range:
     runs-on: ${{ matrix.os }}
@@ -84,9 +88,9 @@ jobs:
 
       - name: Run tests
         working-directory: examples/range
-        timeout-minutes: 10
+        timeout-minutes: 30
         run: |
-          pytest -vv tests -n auto
+          pytest -vv tests
 
   rabbitmq_management:
     runs-on: ${{ matrix.os }}
@@ -119,7 +123,7 @@ jobs:
         working-directory: examples/rabbitmq_management
         timeout-minutes: 10
         run: |
-          pytest -vv tests -n auto
+          pytest -vv tests
 
   django:
     runs-on: ${{ matrix.os }}
@@ -158,7 +162,7 @@ jobs:
         timeout-minutes: 10
         run: |
           export DJANGO_SETTINGS_MODULE=proj.settings
-          pytest -vv tests -n auto
+          pytest -vv tests
 
   myutils:
     runs-on: ${{ matrix.os }}
@@ -191,4 +195,4 @@ jobs:
         working-directory: examples/myutils
         timeout-minutes: 10
         run: |
-          pytest -vv tests -n auto
+          pytest -vv tests

.github/workflows/parallel-support.yml

@@ -62,7 +62,7 @@ jobs:
           poetry install --only ci
 
       - name: Run tox for all environments in parallel
-        timeout-minutes: 15
+        timeout-minutes: 30
         run: |
           tox -e xdist
 
@@ -104,6 +104,6 @@ jobs:
           poetry install --only ci
 
       - name: Run tox for all environments in parallel
-        timeout-minutes: 15
+        timeout-minutes: 30
         run: |
           tox -e parallel

.github/workflows/python-package.yml

@@ -12,13 +12,17 @@ on:
       - ".github/workflows/python-package.yml"
       - "**.toml"
       - "tox.ini"
+      - 'src/pytest_celery/vendors/worker/**'
+      - 'Dockerfile'
   pull_request:
     paths:
       - "**.py"
       - "**.txt"
       - "**.toml"
       - ".github/workflows/python-package.yml"
       - "tox.ini"
+      - 'src/pytest_celery/vendors/worker/**'
+      - 'Dockerfile'
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -159,6 +163,6 @@ jobs:
           poetry install --only ci
 
       - name: Run tox for "${{ matrix.python-version }}-smoke"
-        timeout-minutes: 15
+        timeout-minutes: 30
         run: |
           tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -n auto --reruns 2 --rerun-except AssertionError

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       # Tell poetry to not use a virtual environment
       - poetry config virtualenvs.create false
     post_install:
-      - poetry install --with docs
+      - poetry install -E "all" --with docs
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:

docs/faq.rst

@@ -167,7 +167,7 @@ To avoid these, you may:
 1. Increase the resources available to Docker.
 2. Use the :pypi:`pytest-rerunfailures` pytest plugin to retry failed tests with:
 
-.. code-block:: bash
+.. code-block:: console
 
     --reruns 5 --reruns-delay 60 --rerun-except AssertionError
 

docs/getting-started/first-steps.rst

@@ -148,7 +148,7 @@ This simple test then,
 
 Will run against all of the (enabled) possible combinations of the environment matrix.
 
-.. code-block:: bash
+.. code-block:: console
 
     pytest tests/test_example.py
     ======================================================================= test session starts ===================================
@@ -165,7 +165,7 @@ With each iteration having its own isolated environment.
 RabbitMQ Broker Iteration Breakdown
 ###################################
 
-.. code-block:: bash
+.. code-block:: console
 
     docker ps
     CONTAINER ID   IMAGE             COMMAND                  CREATED          STATUS          PORTS                                                                     NAMES
@@ -175,7 +175,7 @@ RabbitMQ Broker Iteration Breakdown
 
 With the worker configured correctly for its broker and backend.
 
-.. code-block:: bash
+.. code-block:: console
 
      -------------- celery_test_worker@0ffb4e75b5e4 v5.3.6 (emerald-rush)
     --- ***** -----
@@ -193,7 +193,7 @@ With the worker configured correctly for its broker and backend.
 
 With more verbose test logs.
 
-.. code-block:: bash
+.. code-block:: console
 
     ============================= test session starts ==============================
     ...
@@ -215,7 +215,7 @@ With more verbose test logs.
 Redis Broker Iteration Breakdown
 ################################
 
-.. code-block:: bash
+.. code-block:: console
 
     docker ps
     CONTAINER ID   IMAGE          COMMAND                  CREATED          STATUS          PORTS                     NAMES
@@ -225,7 +225,7 @@ Redis Broker Iteration Breakdown
 
 With the worker configured correctly for its broker and backend.
 
-.. code-block:: bash
+.. code-block:: console
 
      -------------- celery_test_worker@37e8ea35206f v5.3.6 (emerald-rush)
     --- ***** -----
@@ -243,7 +243,7 @@ With the worker configured correctly for its broker and backend.
 
 With more verbose test logs.
 
-.. code-block:: bash
+.. code-block:: console
 
     ============================= test session starts ==============================
     ...
@@ -299,7 +299,7 @@ Or, testing the :ref:`default redis broker <redis-broker>` at the node level.
 Remember, each test case is isolated. This means that both of these tests
 can run in parallel, and **each will be assigned its own container instance.**
 
-.. code-block:: bash
+.. code-block:: console
 
     pytest tests/test_example.py -n auto
     ======================================================================= test session starts ===================================
@@ -312,7 +312,7 @@ can run in parallel, and **each will be assigned its own container instance.**
 
     ======================================================================== 2 passed in 1.72s ====================================
 
-.. code-block:: bash
+.. code-block:: console
 
     docker ps
     CONTAINER ID   IMAGE          COMMAND                  CREATED         STATUS         PORTS                     NAMES

docs/getting-started/introduction.rst

@@ -9,7 +9,7 @@
 
 .. contents::
     :local:
-    :depth: 1
+    :depth: 2
 
 What is Pytest Celery?
 ======================
@@ -254,4 +254,6 @@ Quick Jump
         - :ref:`FAQ <faq>`
         - :ref:`API Reference <apiref>`
 
+.. _installation:
+
 .. include:: ../includes/installation.txt

docs/getting-started/vendors.rst

@@ -43,12 +43,13 @@ Experimental brokers may be functional but are not confirmed to be
 production ready.
 
 Enabled means that it is automatically added to the test setup matrix
-when running the test suite.
+when running the test suite :ref:`if the vendor dependencies are installed <installation>`.
 
 .. warning::
 
     Enabling a new vendor will automatically add it globally to every test suite that relies
-    on the default configurations. Be careful when enabling new vendors.
+    on the default vendors detection. Be careful when enabling new vendors and make sure they are
+    stable and production ready.
 
 .. _built-in-worker:
 

docs/includes/installation.txt

@@ -1,20 +1,68 @@
-.. _installation:
-
 Installation
 ============
 
-You can install Pytest Celery via the Python Package Index (PyPI).
+The **pytest-celery** plugin can be easily installed via the Python Package Index (PyPI) using :command:`pip`.
+
+Installing the pytest-celery package
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To install using :command:`pip`:
+To install the latest version of **pytest-celery**, run the following command:
 
 .. code-block:: console
 
-    $ pip install -U pytest-celery
+    pip install -U pytest-celery
+
+This command installs **pytest-celery** along with its required dependencies.
+
+This will include:
+
+- Latest version of :pypi:`celery`.
+- RabbitMQ broker via :pypi:`kombu`, installed as a dependency of Celery.
+
+Installing pytest-celery vendors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The plugin detects which vendor dependencies are installed in the test environment to configure
+the default configurations automatically. This means that just by installing the matching dependencies,
+the plugin will allow extending the default configurations, up to the supported built-in :ref:`vendors`.
+
+.. warning::
+
+    If you don't install any vendor (e.g. no extras and no manual installation), the plugin will result in an
+    empty setup matrix and might not be fully functional.
+
+To install the vendors, you may either install all of the dependencies manually, or use the following extras:
+
+- ``all``: Installs all vendors.
+- ``redis``: Installs Redis vendor, providing **broker** and **result backend** components.
+- ``memcached``: Installs Memcached vendor, providing a **result backend** component.
+
+The following extra is installed by default:
+
+- ``rabbitmq``: Installs RabbitMQ vendor, providing a **broker** component.
+
+To install **pytest-celery** with the built-in :ref:`vendors`, replace ``<extra>`` with the name of the vendor.
+
+.. code-block:: console
+
+    pip install -U "pytest-celery[<extra>]"
+
+RabbitMQ & Redis combo
+----------------------
+
+.. code-block:: console
+
+    pip install -U "pytest-celery[redis]"
+
+This will configure the plugin to generate all possible setups using only RabbitMQ and Redis vendors.
+
+All vendors
+-----------
+
+.. code-block:: console
 
-The :pypi:`celery` package will be installed by default with the ``redis`` and ``pymemcache``
-`extras <https://docs.celeryq.dev/en/stable/getting-started/introduction.html#transports-and-backends>`_.
+    pip install -U "pytest-celery[all]"
 
-With git
-~~~~~~~~
+This will configure the plugin to generate all possible setups.
 
-Please see the :ref:`Contributing <contributing>` section.
+This approach allows you to tailor the installation to your project's specific needs by including only the necessary optional vendors.

examples/django/requirements.txt

@@ -1,5 +1,5 @@
 sqlalchemy>=1.2.18
 django>=2.2.1
 pytest-django>=4.7.0
-git+https://github.com/celery/pytest-celery.git
 pytest-xdist>=3.5.0
+pytest-celery[all]@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@bugfix

examples/django/tests/DjangoWorker.Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.11-bookworm
 RUN adduser --disabled-password --gecos "" test_user
 
 # Install system dependencies
-RUN apt-get update && apt-get install -y build-essential
+RUN apt-get update && apt-get install -y build-essential git
 
 # Set arguments
 ARG CELERY_LOG_LEVEL=INFO

examples/myutils/requirements.txt

@@ -1,3 +1,3 @@
 pytest>=7.4.4
-git+https://github.com/celery/pytest-celery.git
 pytest-xdist>=3.5.0
+pytest-celery[all]@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@bugfix

examples/myworker/requirements.txt

@@ -1,3 +1,3 @@
 pytest>=7.4.4
-git+https://github.com/celery/pytest-celery.git
 pytest-xdist>=3.5.0
+pytest-celery[all]@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@bugfix

examples/rabbitmq_management/requirements.txt

@@ -1,3 +1,3 @@
 pytest>=7.4.4
-git+https://github.com/celery/pytest-celery.git
 pytest-xdist>=3.5.0
+pytest-celery[all]@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@bugfix

examples/range/requirements.txt

@@ -1,4 +1,4 @@
 pytest>=7.4.4
-git+https://github.com/celery/pytest-celery.git
 pytest-xdist>=3.5.0
 pytest-subtests>=0.11.0
+pytest-celery[all]@git+https://github.com/Katz-Consulting-Group/pytest-celery.git@bugfix