diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml new file mode 100644 index 00000000..1f502e9f --- /dev/null +++ b/.github/workflows/examples.yml @@ -0,0 +1,62 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: examples + +on: + push: + branches: ["main"] + paths: + - "**.py" + - "**.txt" + - ".github/workflows/examples.yml" + - "**.toml" + pull_request: + paths: + - "**.py" + - "**.txt" + - "**.toml" + - ".github/workflows/examples.yml" + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + django: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + os: ["ubuntu-latest"] + + steps: + - name: Install apt packages + if: startsWith(matrix.os, 'ubuntu-') + run: | + sudo apt update + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/setup.py' + - name: Install dependencies + working-directory: examples/django + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Migrations + working-directory: examples/django + run: | + ./manage.py migrate + + - name: Run tests + working-directory: examples/django + timeout-minutes: 5 + run: | + export DJANGO_SETTINGS_MODULE=proj.settings + pytest -vv tests -n auto diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c1d91364..bc35179b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,7 @@ repos: - id: check-ast - id: check-case-conflict - id: check-docstring-first + exclude: "examples/django/proj/settings.py" - id: check-json - id: check-merge-conflict - id: check-shebang-scripts-are-executable diff --git a/examples/django/demoapp/__init__.py b/examples/django/demoapp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/django/demoapp/migrations/0001_initial.py b/examples/django/demoapp/migrations/0001_initial.py new file mode 100644 index 00000000..0bc1aa42 --- /dev/null +++ b/examples/django/demoapp/migrations/0001_initial.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.1 on 2019-05-24 21:37 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Widget", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=140)), + ], + ), + ] diff --git a/examples/django/demoapp/migrations/__init__.py b/examples/django/demoapp/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/django/demoapp/models.py b/examples/django/demoapp/models.py new file mode 100644 index 00000000..1f7d09ea --- /dev/null +++ b/examples/django/demoapp/models.py @@ -0,0 +1,5 @@ +from django.db import models + + +class Widget(models.Model): + name = models.CharField(max_length=140) diff --git a/examples/django/demoapp/tasks.py b/examples/django/demoapp/tasks.py new file mode 100644 index 00000000..fe681a75 --- /dev/null +++ b/examples/django/demoapp/tasks.py @@ -0,0 +1,32 @@ +# Create your tasks here + +from celery import shared_task + +from .models import Widget + + +@shared_task +def add(x, y): + return x + y + + +@shared_task +def mul(x, y): + return x * y + + +@shared_task +def xsum(numbers): + return sum(numbers) + + +@shared_task +def count_widgets(): + return Widget.objects.count() + + +@shared_task +def rename_widget(widget_id, name): + w = Widget.objects.get(id=widget_id) + w.name = name + w.save() diff --git a/examples/django/demoapp/views.py b/examples/django/demoapp/views.py new file mode 100644 index 00000000..60f00ef0 --- /dev/null +++ b/examples/django/demoapp/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/examples/django/manage.py b/examples/django/manage.py new file mode 100755 index 00000000..71bd74d7 --- /dev/null +++ b/examples/django/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/examples/django/proj/__init__.py b/examples/django/proj/__init__.py new file mode 100644 index 00000000..5568b6d7 --- /dev/null +++ b/examples/django/proj/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/examples/django/proj/celery.py b/examples/django/proj/celery.py new file mode 100644 index 00000000..a9c583d9 --- /dev/null +++ b/examples/django/proj/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") + +app = Celery("proj") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f"Request: {self.request!r}") diff --git a/examples/django/proj/settings.py b/examples/django/proj/settings.py new file mode 100644 index 00000000..d6b2580d --- /dev/null +++ b/examples/django/proj/settings.py @@ -0,0 +1,138 @@ +import os + +# ^^^ The above is required if you want to import from the celery +# library. If you don't have this then `from celery.schedules import` +# becomes `proj.celery.schedules` in Python 2.x since it allows +# for relative imports by default. + +# Celery settings + +CELERY_BROKER_URL = "amqp://guest:guest@localhost" + +#: Only add pickle to this list if your broker is secured +#: from unwanted access (see userguide/security.html) +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_RESULT_BACKEND = "db+sqlite:///results.sqlite" +CELERY_TASK_SERIALIZER = "json" + + +""" +Django settings for proj project. + +Generated by 'django-admin startproject' using Django 2.2.1. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.2/ref/settings/ +""" + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "l!t+dmzf97rt9s*yrsux1py_1@odvz1szr&6&m!f@-nxq6k%%p" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "demoapp", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "proj.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "proj.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.2/howto/static-files/ + +STATIC_URL = "/static/" diff --git a/examples/django/proj/urls.py b/examples/django/proj/urls.py new file mode 100644 index 00000000..42bdf2e5 --- /dev/null +++ b/examples/django/proj/urls.py @@ -0,0 +1,13 @@ +# Uncomment the next two lines to enable the admin: +# from django.contrib import admin +# admin.autodiscover() + +urlpatterns = [ + # Examples: + # url(r'^$', 'proj.views.home', name='home'), + # url(r'^proj/', include('proj.foo.urls')), + # Uncomment the admin/doc line below to enable admin documentation: + # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + # Uncomment the next line to enable the admin: + # url(r'^admin/', include(admin.site.urls)), +] diff --git a/examples/django/proj/wsgi.py b/examples/django/proj/wsgi.py new file mode 100644 index 00000000..d48b8110 --- /dev/null +++ b/examples/django/proj/wsgi.py @@ -0,0 +1,30 @@ +""" +WSGI config for proj project. + +This module contains the WSGI application used by Django's development server +and any production WSGI deployments. It should expose a module-level variable +named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover +this application via the ``WSGI_APPLICATION`` setting. + +Usually you will have the standard Django WSGI application here, but it also +might make sense to replace the whole Django WSGI application with a custom one +that later delegates to the Django one. For example, you could introduce WSGI +middleware here, or combine a Django application with an application of another +framework. + +""" + +import os + +# This application object is used by any WSGI server configured to use this +# file. This includes Django's development server, if the WSGI_APPLICATION +# setting points here. +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "proj.settings") + +application = get_wsgi_application() + +# Apply WSGI middleware here. +# from helloworld.wsgi import HelloWorldApplication +# application = HelloWorldApplication(application) diff --git a/examples/django/pytest.ini b/examples/django/pytest.ini new file mode 100644 index 00000000..de6463ec --- /dev/null +++ b/examples/django/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +DJANGO_SETTINGS_MODULE = 'proj.settings' diff --git a/examples/django/requirements.txt b/examples/django/requirements.txt new file mode 100644 index 00000000..c11b1014 --- /dev/null +++ b/examples/django/requirements.txt @@ -0,0 +1,7 @@ + +sqlalchemy>=1.2.18 +django>=2.2.1 +pytest-django>=4.7.0 +# pytest-celery>=1.0.0 +git+https://github.com/celery/pytest-celery.git +pytest-xdist>=3.5.0 diff --git a/examples/django/tests/DjangoWorker.Dockerfile b/examples/django/tests/DjangoWorker.Dockerfile new file mode 100644 index 00000000..9c4edd56 --- /dev/null +++ b/examples/django/tests/DjangoWorker.Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11-bookworm + +# Create a user to run the worker +RUN adduser --disabled-password --gecos "" test_user + +# Install system dependencies +RUN apt-get update && apt-get install -y build-essential + +# Set arguments +ARG CELERY_LOG_LEVEL=INFO +ARG CELERY_WORKER_NAME=celery_dev_worker +ARG CELERY_WORKER_QUEUE=celery +ENV LOG_LEVEL=$CELERY_LOG_LEVEL +ENV WORKER_NAME=$CELERY_WORKER_NAME +ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE + +# Install packages +WORKDIR /src + +COPY --chown=test_user:test_user requirements.txt . +RUN pip install --no-cache-dir --upgrade pip +RUN pip install -r ./requirements.txt + +# Switch to the test_user +USER test_user + +# Start the celery worker +CMD celery -A proj worker --loglevel=$LOG_LEVEL -n $WORKER_NAME@%h -Q $WORKER_QUEUE diff --git a/examples/django/tests/__init__.py b/examples/django/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/django/tests/conftest.py b/examples/django/tests/conftest.py new file mode 100644 index 00000000..08373b66 --- /dev/null +++ b/examples/django/tests/conftest.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import os +from typing import Any + +import celery +import pytest +from pytest_docker_tools import build +from pytest_docker_tools import container +from pytest_docker_tools import fxtr + +from pytest_celery import CeleryWorkerContainer +from pytest_celery import defaults + + +class DjangoWorkerContainer(CeleryWorkerContainer): + @property + def client(self) -> Any: + return self + + @classmethod + def version(cls) -> str: + return celery.__version__ + + @classmethod + def log_level(cls) -> str: + return "INFO" + + @classmethod + def worker_name(cls) -> str: + return "django_tests_worker" + + @classmethod + def worker_queue(cls) -> str: + return "celery" + + +worker_image = build( + path=".", + dockerfile="tests/DjangoWorker.Dockerfile", + tag="pytest-celery/django:example", + buildargs=DjangoWorkerContainer.buildargs(), +) + + +default_worker_container = container( + image="{worker_image.id}", + environment=fxtr("default_worker_env"), + network="{default_pytest_celery_network.name}", + volumes={ + # Volume: Worker /app + "{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME, + # Mount: source + os.path.abspath(os.getcwd()): { + "bind": "/src", + "mode": "rw", + }, + }, + wrapper_class=DjangoWorkerContainer, + timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT, +) + + +@pytest.fixture +def default_worker_container_cls() -> type[CeleryWorkerContainer]: + return DjangoWorkerContainer + + +@pytest.fixture(scope="session") +def default_worker_container_session_cls() -> type[CeleryWorkerContainer]: + return DjangoWorkerContainer diff --git a/examples/django/tests/test_tasks.py b/examples/django/tests/test_tasks.py new file mode 100644 index 00000000..56117d36 --- /dev/null +++ b/examples/django/tests/test_tasks.py @@ -0,0 +1,10 @@ +from demoapp.tasks import add +from demoapp.tasks import count_widgets + + +def test_add(celery_setup): + assert add.s(1, 2).delay().get() == 3 + + +def test_count_widgets(celery_setup): + assert count_widgets.s().delay().get() == 0