pax_global_header00006660000000000000000000000064151414154330014513gustar00rootroot0000000000000052 comment=29f923a58aadf8222a77d3e7d67cafd28b3a3f84 django-tasks-db-0.12.0/000077500000000000000000000000001514141543300145435ustar00rootroot00000000000000django-tasks-db-0.12.0/.github/000077500000000000000000000000001514141543300161035ustar00rootroot00000000000000django-tasks-db-0.12.0/.github/workflows/000077500000000000000000000000001514141543300201405ustar00rootroot00000000000000django-tasks-db-0.12.0/.github/workflows/ci.yml000066400000000000000000000104741514141543300212640ustar00rootroot00000000000000name: CI on: pull_request: push: branches: - master tags: - "*" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [windows-latest, macos-latest, ubuntu-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] django-version: ["4.2", "5.2", "6.0"] exclude: - django-version: "6.0" python-version: "3.10" - django-version: "6.0" python-version: "3.11" - django-version: "4.2" python-version: "3.14" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v5 with: path: ~/.cache/pip key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }} - uses: taiki-e/install-action@just - name: Install dependencies run: | pip install --upgrade pip pip install -e . --group dev pip install Django~=${{ matrix.django-version }} - name: Lint run: just lint - name: Run fast tests if: ${{ !cancelled() }} run: just test-fast -v2 - name: Run tests if: ${{ !cancelled() }} run: just test -v2 --keepdb test-postgres: runs-on: ubuntu-latest services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres ports: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 strategy: fail-fast: false matrix: django-version: ["5.2", "6.0"] steps: - uses: actions/checkout@v6 - name: Set up Python 3.14 uses: actions/setup-python@v6 with: python-version: "3.14" - uses: actions/cache@v5 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} - uses: taiki-e/install-action@just - name: Install dependencies run: | pip install --upgrade pip pip install -e . --group dev --group postgres pip install Django~=${{ matrix.django-version }} - name: Run tests run: just test -v2 env: DATABASE_URL: postgres://postgres:postgres@localhost/postgres test-mysql: runs-on: ubuntu-latest services: mysql: image: mysql:8.4 env: MYSQL_ROOT_PASSWORD: django MYSQL_DATABASE: django ports: - 3306:3306 strategy: fail-fast: false matrix: django-version: ["5.2", "6.0"] steps: - uses: actions/checkout@v6 - name: Set up Python 3.14 uses: actions/setup-python@v6 with: python-version: "3.14" - uses: actions/cache@v5 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} - uses: taiki-e/install-action@just - name: Install dependencies run: | pip install --upgrade pip pip install -e . --group dev --group mysql pip install Django~=${{ matrix.django-version }} - name: Run tests run: just test -v2 env: DATABASE_URL: mysql://root:django@127.0.0.1/django build: permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing runs-on: ubuntu-latest needs: - test - test-postgres - test-mysql steps: - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.14" - name: Install dependencies run: | python -m pip install --upgrade pip build - name: Build package run: python -m build - name: Save built package uses: actions/upload-artifact@v6 with: name: package path: dist - name: Publish to PyPi if: ${{ github.ref_type == 'tag' }} uses: pypa/gh-action-pypi-publish@release/v1 with: print-hash: true django-tasks-db-0.12.0/.gitignore000066400000000000000000000067031514141543300165410ustar00rootroot00000000000000# Created by https://www.toptal.com/developers/gitignore/api/python # Edit at https://www.toptal.com/developers/gitignore?templates=python ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py *.sqlite3 *.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ### Python Patch ### # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration poetry.toml # ruff .ruff_cache/ # LSP config files pyrightconfig.json # End of https://www.toptal.com/developers/gitignore/api/python # Editor config files .vscode django-tasks-db-0.12.0/CONTRIBUTING.md000066400000000000000000000026751514141543300170060ustar00rootroot00000000000000# Contributing Found a bug? Want to fix an open issue? Got an idea for an improvement? Please contribute! **All** contributions are welcome, from absolutely anyone. Just open a PR, Issue or Discussion (as relevant) - no need to ask beforehand. If you're going to work on an issue, it's a good idea to say so on the issue, to make sure work isn't duplicated. ## Development set up Fork, then clone the repo: ```sh git clone git@github.com:your-username/django-tasks-db.git ``` Set up a venv: ```sh python -m venv .venv source .venv/bin/activate python -m pip install -e --group dev ``` > [!TIP] > To include support for a specific database, you can stack group flags (e.g. `--group dev --group postgres`). This is optional. Then you can run the tests with the [just](https://just.systems/man/en/) command runner: ```sh just test ``` If you don't have `just` installed, you can look in the `justfile` for the commands that are run. To help with testing on different databases, there's a `docker-compose.yml` file to run PostgreSQL and MySQL in Docker, as well as some additional `just` commands for testing: ```sh just start-dbs just test-postgres just test-mysql just test-sqlite # To run all of the above: just test-dbs ``` Due to database worker process' tests, tests cannot run using an in-memory database, which means tests run quite slow locally. If you're not modifying the worker, and want you tests run run quicker, run: ```sh just test-fast ``` django-tasks-db-0.12.0/LICENSE000066400000000000000000000030171514141543300155510ustar00rootroot00000000000000Copyright (c) Jake Howard and individual contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. django-tasks-db-0.12.0/README.md000066400000000000000000000050041514141543300160210ustar00rootroot00000000000000# Django Tasks DB [![CI](https://github.com/RealOrangeOne/django-tasks-db/actions/workflows/ci.yml/badge.svg)](https://github.com/RealOrangeOne/django-tasks-db/actions/workflows/ci.yml) ![PyPI](https://img.shields.io/pypi/v/django-tasks-db.svg) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/django-tasks-db.svg) ![PyPI - Status](https://img.shields.io/pypi/status/django-tasks-db.svg) ![PyPI - License](https://img.shields.io/pypi/l/django-tasks-db.svg) A [Django Tasks](https://docs.djangoproject.com/en/stable/topics/tasks/) backend which uses Django's ORM to store tasks in the database. ## Installation ``` python -m pip install django-tasks ``` First, add `django_tasks_db` to your `INSTALLED_APPS`: ```python INSTALLED_APPS = [ # ... "django_tasks_db", ] ``` Finally, add it to your `TASKS` configuration: ```python TASKS = { "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["default"] } } ``` ## Usage ### Worker You can run the `db_worker` command to run tasks as they're created. Check the `--help` for more options. ```shell ./manage.py db_worker ``` In `DEBUG`, the worker will automatically reload when code is changed (or by using `--reload`). This is not recommended in production environments as tasks may not be stopped cleanly. ### Pruning old tasks After a while, tasks may start to build up in your database. This can be managed using the `prune_db_task_results` management command, which deletes completed tasks according to the given retention policy. Check the `--help` for the available options. ### Customizing the task id By default, the database worker uses `uuid.uuid4` to generate a task id. This can be customized using the `id_function` option: ```python TASKS = { "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "OPTIONS": { "id_function": "uuid.uuid7" } } } ``` The `id_function` must return a UUID (either `uuid.UUID` or string representation). Additionally, the PostgreSQL-specific [`RandomUUID`](https://docs.djangoproject.com/en/stable/ref/contrib/postgres/functions/#django.contrib.postgres.functions.RandomUUID) or other database expressions are supported on Django 6.0+. ## Contributing See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute. Note: Prior to `0.12.0`, this backend was included in [`django-tasks`](https://github.com/RealOrangeOne/django-tasks/). Whilst the commit history was cleaned up, it's still quite messy. Don't look too closely. django-tasks-db-0.12.0/django_tasks_db/000077500000000000000000000000001514141543300176575ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/__init__.py000066400000000000000000000003371514141543300217730ustar00rootroot00000000000000# ruff: noqa: E402 import django_stubs_ext django_stubs_ext.monkeypatch() import importlib.metadata __version__ = importlib.metadata.version(__name__) from .backend import DatabaseBackend __all__ = ["DatabaseBackend"] django-tasks-db-0.12.0/django_tasks_db/admin.py000066400000000000000000000031071514141543300213220ustar00rootroot00000000000000from typing import Any from django.contrib import admin from django.http import HttpRequest from django.utils.html import format_html from .models import DBTaskResult @admin.register(DBTaskResult) class DBTaskResultAdmin(admin.ModelAdmin): list_display = ( "id", "task_name", "status", "enqueued_at", "started_at", "finished_at", "priority", "queue_name", ) list_filter = ("status", "priority", "queue_name") ordering = ["-enqueued_at"] def has_add_permission( self, request: HttpRequest, obj: DBTaskResult | None = None ) -> bool: return False def has_delete_permission( self, request: HttpRequest, obj: DBTaskResult | None = None ) -> bool: return False def has_change_permission( self, request: HttpRequest, obj: DBTaskResult | None = None ) -> bool: return False def get_readonly_fields( self, request: HttpRequest, obj: DBTaskResult | None = None ) -> list[str]: fields = [f.name for f in self.model._meta.fields] fields[fields.index("traceback")] = "formatted_traceback" return fields def get_fields( self, request: HttpRequest, obj: DBTaskResult | None = None ) -> list[Any]: return self.get_readonly_fields(request, obj) @admin.display(description="traceback") def formatted_traceback(self, obj: DBTaskResult) -> str: if not obj.traceback: return "-" return format_html( "
{}
", obj.traceback, ) django-tasks-db-0.12.0/django_tasks_db/apps.py000066400000000000000000000003761514141543300212020ustar00rootroot00000000000000from django.apps import AppConfig class TasksAppConfig(AppConfig): name = "django_tasks_db" label = "django_tasks_database" verbose_name = "Tasks Database Backend" def ready(self) -> None: from . import signal_handlers # noqa django-tasks-db-0.12.0/django_tasks_db/backend.py000066400000000000000000000124751514141543300216310ustar00rootroot00000000000000from collections.abc import Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, TypeVar from django import VERSION from django.apps import apps from django.core import checks from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db.models import Expression from django.utils.module_loading import import_string from django.utils.version import PY311 from django_tasks.backends.base import BaseTaskBackend from django_tasks.base import Task from django_tasks.base import TaskResult as BaseTaskResult from django_tasks.exceptions import TaskResultDoesNotExist from django_tasks.signals import task_enqueued from django_tasks.utils import normalize_json from typing_extensions import ParamSpec if TYPE_CHECKING: from .models import DBTaskResult T = TypeVar("T") P = ParamSpec("P") @dataclass(frozen=True, slots=PY311, kw_only=True) # type: ignore[literal-required] class TaskResult(BaseTaskResult[T]): db_result: "DBTaskResult" class DatabaseBackend(BaseTaskBackend): supports_async_task = True supports_get_result = True supports_defer = True supports_priority = True def __init__(self, alias: str, params: dict) -> None: from .models import DBTaskResult super().__init__(alias, params) if id_function := self.options.get("id_function"): if callable(id_function): self.id_function = id_function else: self.id_function = import_string(id_function) else: # Fall back to the default defined on the model self.id_function = DBTaskResult._meta.pk.default def _get_id(self) -> Any: result_id = self.id_function() if VERSION < (6, 0) and isinstance(result_id, Expression): raise ImproperlyConfigured( "id_function cannot be a database expression until Django 6.0" ) return result_id def _task_to_db_task( self, task: Task[P, T], args: P.args, # type:ignore[valid-type] kwargs: P.kwargs, # type:ignore[valid-type] ) -> "DBTaskResult": from .models import DBTaskResult return DBTaskResult.objects.create( id=self._get_id(), args_kwargs=normalize_json({"args": args, "kwargs": kwargs}), priority=task.priority, task_path=task.module_path, queue_name=task.queue_name, run_after=task.run_after, # type: ignore[misc] backend_name=self.alias, ) async def _atask_to_db_task( self, task: Task[P, T], args: P.args, # type:ignore[valid-type] kwargs: P.kwargs, # type:ignore[valid-type] ) -> "DBTaskResult": from .models import DBTaskResult return await DBTaskResult.objects.acreate( id=self._get_id(), args_kwargs=normalize_json({"args": args, "kwargs": kwargs}), priority=task.priority, task_path=task.module_path, queue_name=task.queue_name, run_after=task.run_after, # type: ignore[misc] backend_name=self.alias, ) def enqueue( self, task: Task[P, T], args: P.args, # type:ignore[valid-type] kwargs: P.kwargs, # type:ignore[valid-type] ) -> TaskResult[T]: self.validate_task(task) db_result = self._task_to_db_task(task, args, kwargs) task_enqueued.send(type(self), task_result=db_result.task_result) return db_result.task_result async def _asend_task_enqueued_signal(self, task_result: TaskResult) -> None: if VERSION < (5, 0): from asgiref.sync import sync_to_async await sync_to_async(task_enqueued.send, thread_sensitive=True)( type(self), task_result=task_result ) else: await task_enqueued.asend(type(self), task_result=task_result) async def aenqueue( self, task: Task[P, T], args: P.args, # type:ignore[valid-type] kwargs: P.kwargs, # type:ignore[valid-type] ) -> TaskResult[T]: self.validate_task(task) db_result = await self._atask_to_db_task(task, args, kwargs) await self._asend_task_enqueued_signal(db_result.task_result) return db_result.task_result def get_result(self, result_id: str) -> TaskResult: from .models import DBTaskResult try: return DBTaskResult.objects.get(id=result_id).task_result except (DBTaskResult.DoesNotExist, ValidationError) as e: raise TaskResultDoesNotExist(result_id) from e async def aget_result(self, result_id: str) -> TaskResult: from .models import DBTaskResult try: return (await DBTaskResult.objects.aget(id=result_id)).task_result except (DBTaskResult.DoesNotExist, ValidationError) as e: raise TaskResultDoesNotExist(result_id) from e def check(self, **kwargs: Any) -> Iterable[checks.CheckMessage]: yield from super().check(**kwargs) backend_name = self.__class__.__name__ if not apps.is_installed("django_tasks_db"): yield checks.Error( f"{backend_name} configured as django_tasks_db backend, but database app not installed", "Insert 'django_tasks_db' in INSTALLED_APPS", ) django-tasks-db-0.12.0/django_tasks_db/compat.py000066400000000000000000000003561514141543300215200ustar00rootroot00000000000000try: from django.tasks.base import Task as DjangoTask except ImportError: DjangoTask = None from django_tasks.base import Task __all__ = ["TASK_CLASSES"] TASK_CLASSES = (Task, DjangoTask) if DjangoTask is not None else (Task,) django-tasks-db-0.12.0/django_tasks_db/management/000077500000000000000000000000001514141543300217735ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/management/commands/000077500000000000000000000000001514141543300235745ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/management/commands/db_worker.py000066400000000000000000000275131514141543300261340ustar00rootroot00000000000000import logging import math import os import random import signal import sys import time from argparse import ArgumentParser, ArgumentTypeError, BooleanOptionalAction from types import FrameType from django.conf import settings from django.core.exceptions import SuspiciousOperation from django.core.management.base import BaseCommand from django.db import close_old_connections from django.db.utils import OperationalError from django.utils.autoreload import DJANGO_AUTORELOAD_ENV, run_with_reloader from django_tasks import DEFAULT_TASK_BACKEND_ALIAS, task_backends from django_tasks.base import DEFAULT_TASK_QUEUE_NAME, TaskContext from django_tasks.exceptions import InvalidTaskBackendError from django_tasks.signals import task_finished, task_started from django_tasks.utils import get_random_id from django_tasks_db.backend import DatabaseBackend from django_tasks_db.models import DBTaskResult from django_tasks_db.utils import exclusive_transaction logger = logging.getLogger("django_tasks_db") class Worker: def __init__( self, *, queue_names: list[str], interval: float, batch: bool, backend_name: str, startup_delay: bool, max_tasks: int | None, worker_id: str, ): self.queue_names = queue_names self.process_all_queues = "*" in queue_names self.interval = interval self.batch = batch self.backend_name = backend_name self.startup_delay = startup_delay self.max_tasks = max_tasks self.running = True self.running_task = False self._run_tasks = 0 self.worker_id = worker_id def shutdown(self, signum: int, frame: FrameType | None) -> None: if not self.running: logger.warning( "Received %s - terminating current task.", signal.strsignal(signum) ) self.reset_signals() sys.exit(1) logger.warning( "Received %s - shutting down gracefully... (press Ctrl+C again to force)", signal.strsignal(signum), ) self.running = False if not self.running_task: # If we're not currently running a task, exit immediately. # This is useful if we're currently in a `sleep`. sys.exit(0) def configure_signals(self) -> None: signal.signal(signal.SIGINT, self.shutdown) signal.signal(signal.SIGTERM, self.shutdown) if hasattr(signal, "SIGQUIT"): signal.signal(signal.SIGQUIT, self.shutdown) def reset_signals(self) -> None: signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) if hasattr(signal, "SIGQUIT"): signal.signal(signal.SIGQUIT, signal.SIG_DFL) def run(self) -> None: logger.info( "Starting worker worker_id=%s queues=%s", self.worker_id, ",".join(self.queue_names), ) if self.startup_delay and self.interval: # Add a random small delay before starting to avoid a thundering herd time.sleep(random.random()) # noqa: S311 while self.running: tasks = DBTaskResult.objects.ready().filter(backend_name=self.backend_name) if not self.process_all_queues: tasks = tasks.filter(queue_name__in=self.queue_names) # During this transaction, all "ready" tasks are locked. Therefore, it's important # it be as efficient as possible. with exclusive_transaction(tasks.db): try: task_result = tasks.get_locked() except OperationalError as e: # Ignore locked databases and keep trying. # It should unlock eventually. if "is locked" in e.args[0]: task_result = None else: raise if task_result is not None: # "claim" the task, so it isn't run by another worker process task_result.claim(self.worker_id) if task_result is not None: self.run_task(task_result) if self.batch and task_result is None: # If we're running in "batch" mode, terminate the loop (and thus the worker) logger.info( "No more tasks to run for worker_id=%s - exiting gracefully.", self.worker_id, ) return None if self.max_tasks is not None and self._run_tasks >= self.max_tasks: logger.info( "Run maximum tasks (%d) on worker=%s - exiting gracefully.", self._run_tasks, self.worker_id, ) return None # Emulate Django's request behaviour and check for expired # database connections periodically. close_old_connections() # If ctrl-c has just interrupted a task, self.running was cleared, # and we should not sleep, but rather exit immediately. if self.running and not task_result: # Wait before checking for another task time.sleep(self.interval) def run_task(self, db_task_result: DBTaskResult) -> None: """ Run the given task, marking it as successful or failed. """ try: self.running_task = True task = db_task_result.task task_result = db_task_result.task_result backend_type = task.get_backend() task_started.send(sender=backend_type, task_result=task_result) if task.takes_context: return_value = task.call( TaskContext(task_result=task_result), *task_result.args, **task_result.kwargs, ) else: return_value = task.call(*task_result.args, **task_result.kwargs) # Setting the return and success value inside the error handling, # So errors setting it (eg JSON encode) can still be recorded db_task_result.set_successful(return_value) task_finished.send( sender=backend_type, task_result=db_task_result.task_result ) except BaseException as e: db_task_result.set_failed(e) try: sender = type(db_task_result.task.get_backend()) task_result = db_task_result.task_result except (ImportError, SuspiciousOperation): logger.exception("Task id=%s failed unexpectedly", db_task_result.id) else: task_finished.send( sender=sender, task_result=task_result, ) finally: self.running_task = False self._run_tasks += 1 def valid_backend_name(val: str) -> str: try: backend = task_backends[val] except InvalidTaskBackendError as e: raise ArgumentTypeError(e.args[0]) from e if not isinstance(backend, DatabaseBackend): raise ArgumentTypeError(f"Backend '{val}' is not a database backend") return val def valid_interval(val: str) -> float: num = float(val) if not math.isfinite(num): raise ArgumentTypeError("Must be a finite floating point value") if num < 0: raise ArgumentTypeError("Must be greater than zero") return num def valid_max_tasks(val: str) -> int: num = int(val) if num < 0: raise ArgumentTypeError("Must be greater than zero") return num def validate_worker_id(val: str) -> str: if not val: raise ArgumentTypeError("Worker id must not be empty") if len(val) > 64: raise ArgumentTypeError("Worker ids must be shorter than 64 characters") return val class Command(BaseCommand): help = "Run a database background worker" def add_arguments(self, parser: ArgumentParser) -> None: parser.add_argument( "--queue-name", nargs="?", default=DEFAULT_TASK_QUEUE_NAME, type=str, help="The queues to process. Separate multiple with a comma. To process all queues, use '*' (default: %(default)r)", ) parser.add_argument( "--interval", nargs="?", default=1, type=valid_interval, help="The interval (in seconds) to wait, when there are no tasks in the queue, before checking for tasks again (default: %(default)r)", ) parser.add_argument( "--batch", action="store_true", help="Process all outstanding tasks, then exit. Can be used in combination with --max-tasks.", ) parser.add_argument( "--reload", action=BooleanOptionalAction, default=settings.DEBUG, help="Reload the worker on code changes. Not recommended for production as tasks may not be stopped cleanly (default: DEBUG)", ) parser.add_argument( "--backend", nargs="?", default=DEFAULT_TASK_BACKEND_ALIAS, type=valid_backend_name, dest="backend_name", help="The backend to operate on (default: %(default)r)", ) parser.add_argument( "--no-startup-delay", action="store_false", dest="startup_delay", help="Don't add a small delay at startup.", ) parser.add_argument( "--max-tasks", nargs="?", default=None, type=valid_max_tasks, help="If provided, the maximum number of tasks the worker will execute before exiting.", ) parser.add_argument( "--worker-id", nargs="?", type=validate_worker_id, help="Worker id. MUST be unique across worker pool (default: auto-generate)", default=get_random_id(), ) def configure_logging(self, verbosity: int) -> None: tasks_logger = logging.getLogger("django_tasks") if verbosity == 0: tasks_logger.setLevel(logging.CRITICAL) logger.setLevel(logging.CRITICAL) elif verbosity == 1: tasks_logger.setLevel(logging.INFO) logger.setLevel(logging.INFO) else: tasks_logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) # If no handler is configured, the logs won't show, # regardless of the set level. if not tasks_logger.hasHandlers(): tasks_logger.addHandler(logging.StreamHandler(self.stdout)) if not logger.hasHandlers(): logger.addHandler(logging.StreamHandler(self.stdout)) def handle( self, *, verbosity: int, queue_name: str, interval: float, batch: bool, backend_name: str, startup_delay: bool, reload: bool, max_tasks: int | None, worker_id: str, **options: dict, ) -> None: self.configure_logging(verbosity) if reload and batch: logger.warning( "Warning: --reload and --batch cannot be specified together. Disabling autoreload." ) reload = False worker = Worker( queue_names=queue_name.split(","), interval=interval, batch=batch, backend_name=backend_name, startup_delay=startup_delay, max_tasks=max_tasks, worker_id=worker_id, ) if reload: if os.environ.get(DJANGO_AUTORELOAD_ENV) == "true": # Only the child process should configure its signals worker.configure_signals() run_with_reloader(worker.run) else: worker.configure_signals() worker.run() django-tasks-db-0.12.0/django_tasks_db/management/commands/prune_db_task_results.py000066400000000000000000000102541514141543300305510ustar00rootroot00000000000000import logging from argparse import ArgumentParser, ArgumentTypeError from datetime import timedelta from django.core.management.base import BaseCommand from django.db.models import Q from django.utils import timezone from django_tasks import ( DEFAULT_TASK_BACKEND_ALIAS, DEFAULT_TASK_QUEUE_NAME, task_backends, ) from django_tasks.base import TaskResultStatus from django_tasks.exceptions import InvalidTaskBackendError from django_tasks_db.backend import DatabaseBackend from django_tasks_db.models import DBTaskResult logger = logging.getLogger("django_tasks_db.prune_db_task_results") def valid_backend_name(val: str) -> DatabaseBackend: try: backend = task_backends[val] except InvalidTaskBackendError as e: raise ArgumentTypeError(e.args[0]) from e if not isinstance(backend, DatabaseBackend): raise ArgumentTypeError(f"Backend '{val}' is not a database backend") return backend def valid_positive_int(val: str) -> int: num = int(val) if num < 0: raise ArgumentTypeError("Must be greater than zero") return num class Command(BaseCommand): help = "Prune finished database task results" def add_arguments(self, parser: ArgumentParser) -> None: parser.add_argument( "--backend", nargs="?", default=DEFAULT_TASK_BACKEND_ALIAS, type=valid_backend_name, dest="backend", help="The backend to operate on (default: %(default)r)", ) parser.add_argument( "--queue-name", nargs="?", default=DEFAULT_TASK_QUEUE_NAME, type=str, help="The queues to process. Separate multiple with a comma. To process all queues, use '*' (default: %(default)r)", ) parser.add_argument( "--min-age-days", nargs="?", default=14, type=valid_positive_int, help="The minimum age (in days) of a finished task result to be pruned (default: %(default)r)", ) parser.add_argument( "--failed-min-age-days", nargs="?", default=None, type=valid_positive_int, help="The minimum age (in days) of a failed task result to be pruned (default: min-age-days)", ) parser.add_argument( "--dry-run", action="store_true", help="Don't delete the task results, just show how many would be deleted", ) def configure_logging(self, verbosity: int) -> None: if verbosity == 0: logger.setLevel(logging.WARNING) elif verbosity == 1: logger.setLevel(logging.INFO) else: logger.setLevel(logging.DEBUG) # If no handler is configured, the logs won't show, # regardless of the set level. if not logger.hasHandlers(): logger.addHandler(logging.StreamHandler(self.stdout)) def handle( self, *, verbosity: int, backend: DatabaseBackend, min_age_days: int, failed_min_age_days: int | None, queue_name: str, dry_run: bool, **options: dict, ) -> None: self.configure_logging(verbosity) min_age = timezone.now() - timedelta(days=min_age_days) failed_min_age = ( (timezone.now() - timedelta(days=failed_min_age_days)) if failed_min_age_days else None ) results = DBTaskResult.objects.finished().filter(backend_name=backend.alias) queue_names = queue_name.split(",") if "*" not in queue_names: results = results.filter(queue_name__in=queue_names) if failed_min_age is None: results = results.filter(finished_at__lte=min_age) else: results = results.filter( Q(status=TaskResultStatus.SUCCESSFUL, finished_at__lte=min_age) | Q(status=TaskResultStatus.FAILED, finished_at__lte=failed_min_age) ) if dry_run: logger.info("Would delete %d task result(s)", results.count()) else: deleted, _ = results.delete() logger.info("Deleted %d task result(s)", deleted) django-tasks-db-0.12.0/django_tasks_db/migrations/000077500000000000000000000000001514141543300220335ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/migrations/0001_initial.py000066400000000000000000000036001514141543300244750ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-06-08 09:10 import uuid from django.db import migrations, models import django_tasks_db.models class Migration(migrations.Migration): initial = True dependencies = [] operations = [ migrations.CreateModel( name="DBTaskResult", fields=[ ( "id", models.UUIDField( default=uuid.uuid4, editable=False, primary_key=True, serialize=False, ), ), ( "status", models.CharField( choices=[ ("NEW", "New"), ("RUNNING", "Running"), ("FAILED", "Failed"), ("COMPLETE", "Complete"), ], default="NEW", max_length=8, ), ), ("args_kwargs", models.JSONField()), ("priority", models.PositiveSmallIntegerField(default=0)), ("task_path", models.TextField()), ("queue_name", models.TextField(default="default")), ("backend_name", models.TextField()), ("run_after", models.DateTimeField(null=True)), ("result", models.JSONField(default=None, null=True)), ], options={ "ordering": [ models.OrderBy(models.F("priority"), descending=True), models.OrderBy( models.F("run_after"), descending=True, nulls_last=True ), ], }, bases=(django_tasks_db.models.GenericBase, models.Model), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0002_alter_dbtaskresult_options.py000066400000000000000000000013201514141543300305130ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-06-08 15:24 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0001_initial"), ] operations = [ migrations.AlterModelOptions( name="dbtaskresult", options={ "ordering": [ models.OrderBy(models.F("priority"), descending=True), models.OrderBy( models.F("run_after"), descending=True, nulls_last=True ), ], "verbose_name": "Task Result", "verbose_name_plural": "Task Results", }, ), ] 0003_dbtaskresult_enqueued_at_dbtaskresult_finished_at.py000066400000000000000000000013141514141543300352010ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/migrations# Generated by Django 5.0.6 on 2024-06-13 10:24 import django.utils.timezone from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0002_alter_dbtaskresult_options"), ] operations = [ migrations.AddField( model_name="dbtaskresult", name="enqueued_at", field=models.DateTimeField( auto_now_add=True, default=django.utils.timezone.now ), preserve_default=False, ), migrations.AddField( model_name="dbtaskresult", name="finished_at", field=models.DateTimeField(null=True), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0004_dbtaskresult_started_at.py000066400000000000000000000007311514141543300277720ustar00rootroot00000000000000# Generated by Django 5.0.6 on 2024-07-04 09:31 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ( "django_tasks_database", "0003_dbtaskresult_enqueued_at_dbtaskresult_finished_at", ), ] operations = [ migrations.AddField( model_name="dbtaskresult", name="started_at", field=models.DateTimeField(null=True), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0005_alter_dbtaskresult_priority_and_more.py000066400000000000000000000015151514141543300325560ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-07-10 15:48 from django import VERSION from django.db import migrations, models if VERSION >= (5, 1): constraint = models.CheckConstraint( condition=models.Q(("priority__range", (-100, 100))), name="priority_range" ) else: constraint = models.CheckConstraint( check=models.Q(("priority__range", (-100, 100))), name="priority_range" ) class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0004_dbtaskresult_started_at"), ] operations = [ migrations.AlterField( model_name="dbtaskresult", name="priority", field=models.IntegerField(default=0), ), migrations.AddConstraint( model_name="dbtaskresult", constraint=constraint, ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0006_alter_dbtaskresult_args_kwargs_and_more.py000066400000000000000000000050571514141543300332150ustar00rootroot00000000000000# Generated by Django 4.2.14 on 2024-08-02 13:17 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0005_alter_dbtaskresult_priority_and_more"), ] operations = [ migrations.AlterField( model_name="dbtaskresult", name="args_kwargs", field=models.JSONField(verbose_name="args kwargs"), ), migrations.AlterField( model_name="dbtaskresult", name="backend_name", field=models.TextField(verbose_name="backend name"), ), migrations.AlterField( model_name="dbtaskresult", name="enqueued_at", field=models.DateTimeField(auto_now_add=True, verbose_name="enqueued at"), ), migrations.AlterField( model_name="dbtaskresult", name="finished_at", field=models.DateTimeField(null=True, verbose_name="finished at"), ), migrations.AlterField( model_name="dbtaskresult", name="priority", field=models.IntegerField(default=0, verbose_name="priority"), ), migrations.AlterField( model_name="dbtaskresult", name="queue_name", field=models.TextField(default="default", verbose_name="queue name"), ), migrations.AlterField( model_name="dbtaskresult", name="result", field=models.JSONField(default=None, null=True, verbose_name="result"), ), migrations.AlterField( model_name="dbtaskresult", name="run_after", field=models.DateTimeField(null=True, verbose_name="run after"), ), migrations.AlterField( model_name="dbtaskresult", name="started_at", field=models.DateTimeField(null=True, verbose_name="started at"), ), migrations.AlterField( model_name="dbtaskresult", name="status", field=models.CharField( choices=[ ("NEW", "New"), ("RUNNING", "Running"), ("FAILED", "Failed"), ("COMPLETE", "Complete"), ], default="NEW", max_length=8, verbose_name="status", ), ), migrations.AlterField( model_name="dbtaskresult", name="task_path", field=models.TextField(verbose_name="task path"), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0007_add_separate_results_fields.py000066400000000000000000000013401514141543300305740ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-08-23 14:38 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0006_alter_dbtaskresult_args_kwargs_and_more"), ] operations = [ migrations.AddField( model_name="dbtaskresult", name="exception_data", field=models.JSONField( default=None, null=True, verbose_name="exception data" ), ), migrations.AddField( model_name="dbtaskresult", name="return_value", field=models.JSONField( default=None, null=True, verbose_name="return value" ), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0008_separate_results_field.py000066400000000000000000000031221514141543300276020ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-08-23 14:38 from django.db import migrations, models from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps def separate_results_field( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") # If a task succeeded, the result is its return value DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="SUCCEEDED" ).update(return_value=models.F("result")) # If a task failed, the result is the exception data (or nothing) DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="FAILED" ).update(exception_data=models.F("result")) def merge_results_field( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") # If a task succeeded, the result is its return value DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="SUCCEEDED" ).update(result=models.F("return_value")) # If a task failed, the result is the exception data (or nothing) DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="FAILED" ).update(result=models.F("exception_data")) class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0007_add_separate_results_fields"), ] operations = [migrations.RunPython(separate_results_field, merge_results_field)] django-tasks-db-0.12.0/django_tasks_db/migrations/0009_remove_results_field.py000066400000000000000000000005401514141543300272750ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-08-23 14:51 from django.db import migrations class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0008_separate_results_field"), ] operations = [ migrations.RemoveField( model_name="dbtaskresult", name="result", ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0010_alter_dbtaskresult_status.py000066400000000000000000000013231514141543300303450ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-11-21 13:46 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0009_remove_results_field"), ] operations = [ migrations.AlterField( model_name="dbtaskresult", name="status", field=models.CharField( choices=[ ("NEW", "New"), ("RUNNING", "Running"), ("FAILED", "Failed"), ("SUCCEEDED", "Succeeded"), ], default="NEW", max_length=9, verbose_name="status", ), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0011_rename_complete_status.py000066400000000000000000000020141514141543300276050ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-08-23 14:38 from django.db import migrations from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps def separate_results_field( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="COMPLETE" ).update(status="SUCCEEDED") def merge_results_field( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="SUCCEEDED" ).update(status="COMPLETE") class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0010_alter_dbtaskresult_status"), ] operations = [migrations.RunPython(separate_results_field, merge_results_field)] django-tasks-db-0.12.0/django_tasks_db/migrations/0012_add_separate_exception_fields.py000066400000000000000000000013061514141543300310670ustar00rootroot00000000000000# Generated by Django 5.1.1 on 2024-11-22 16:32 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0011_rename_complete_status"), ] operations = [ migrations.AddField( model_name="dbtaskresult", name="exception_class_path", field=models.TextField(default="", verbose_name="exception class path"), preserve_default=False, ), migrations.AddField( model_name="dbtaskresult", name="traceback", field=models.TextField(default="", verbose_name="traceback"), preserve_default=False, ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0013_separate_exception_fields.py000066400000000000000000000017541514141543300302670ustar00rootroot00000000000000# Generated by Django 4.2.13 on 2024-08-23 14:38 from django.db import migrations, models from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models.functions import Coalesce def separate_exception_fields( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.using(schema_editor.connection.alias).update( exception_class_path=Coalesce( models.F("exception_data__exc_type"), models.Value("", models.JSONField()) ), traceback=Coalesce( models.F("exception_data__exc_traceback"), models.Value("", models.JSONField()), ), ) class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0012_add_separate_exception_fields"), ] operations = [migrations.RunPython(separate_exception_fields)] django-tasks-db-0.12.0/django_tasks_db/migrations/0014_remove_dbtaskresult_exception_data.py000066400000000000000000000005521514141543300322060ustar00rootroot00000000000000# Generated by Django 5.1.1 on 2024-11-22 16:32 from django.db import migrations class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0013_separate_exception_fields"), ] operations = [ migrations.RemoveField( model_name="dbtaskresult", name="exception_data", ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0015_correctly_order_run_after.py000066400000000000000000000012531514141543300303210ustar00rootroot00000000000000# Generated by Django 5.1.5 on 2025-04-27 12:00 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0014_remove_dbtaskresult_exception_data"), ] operations = [ migrations.AlterModelOptions( name="dbtaskresult", options={ "ordering": [ models.OrderBy(models.F("priority"), descending=True), models.OrderBy(models.F("run_after"), nulls_last=True), ], "verbose_name": "Task Result", "verbose_name_plural": "Task Results", }, ), ] 0016_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py000066400000000000000000000073001514141543300366760ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/migrations# Generated by Django 5.2.2 on 2025-06-13 14:56 import datetime from django.conf import settings from django.db import migrations, models from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps def update_status(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.filter(status="NEW").update(status="READY") def revert_status_rename( apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.filter(status="READY").update(status="NEW") class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0015_correctly_order_run_after"), ] operations = [ migrations.AlterModelOptions( name="dbtaskresult", options={ "ordering": [ models.OrderBy(models.F("priority"), descending=True), models.OrderBy(models.F("run_after")), ], "verbose_name": "Task Result", "verbose_name_plural": "Task Results", }, ), migrations.AlterField( model_name="dbtaskresult", name="backend_name", field=models.CharField(max_length=32, verbose_name="backend name"), ), migrations.AlterField( model_name="dbtaskresult", name="queue_name", field=models.CharField( default="default", max_length=32, verbose_name="queue name" ), ), migrations.AlterField( model_name="dbtaskresult", name="run_after", field=models.DateTimeField( default=datetime.datetime( 9999, 1, 1, 0, 0, tzinfo=datetime.timezone.utc if settings.USE_TZ else None, ), verbose_name="run after", ), preserve_default=False, ), migrations.AlterField( model_name="dbtaskresult", name="status", field=models.CharField( choices=[ ("READY", "Ready"), ("RUNNING", "Running"), ("FAILED", "Failed"), ("SUCCEEDED", "Succeeded"), ], default="READY", max_length=9, verbose_name="status", ), ), migrations.AddField( model_name="dbtaskresult", name="worker_ids", field=models.JSONField(default=list, verbose_name="worker id"), ), migrations.RunPython(update_status, revert_status_rename), migrations.AddIndex( model_name="dbtaskresult", index=models.Index( models.F("status"), models.OrderBy(models.F("priority"), descending=True), models.OrderBy(models.F("run_after")), condition=models.Q(("status", "READY")), name="django_task_new_ordering_idx", ), ), migrations.AddIndex( model_name="dbtaskresult", index=models.Index( fields=["queue_name"], name="django_task_queue_n_99c321_idx" ), ), migrations.AddIndex( model_name="dbtaskresult", index=models.Index( fields=["backend_name"], name="django_task_backend_071754_idx" ), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0017_dbtaskresult_metadata.py000066400000000000000000000007701514141543300274270ustar00rootroot00000000000000# Generated by Django 6.0rc1 on 2025-12-17 08:46 from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ ( "django_tasks_database", "0016_remove_dbtaskresult_django_task_new_ordering_idx_and_more", ), ] operations = [ migrations.AddField( model_name="dbtaskresult", name="metadata", field=models.JSONField(default=dict, verbose_name="metadata"), ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/0018_rename_succeeded_status.py000066400000000000000000000026771514141543300277470ustar00rootroot00000000000000# Generated by Django 6.0 on 2026-01-09 from django.db import migrations, models from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.migrations.state import StateApps def forwards(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="SUCCEEDED" ).update(status="SUCCESSFUL") def backwards(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult") DBTaskResult.objects.using(schema_editor.connection.alias).filter( status="SUCCESSFUL" ).update(status="SUCCEEDED") class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0017_dbtaskresult_metadata"), ] operations = [ migrations.AlterField( model_name="dbtaskresult", name="status", field=models.CharField( choices=[ ("READY", "Ready"), ("RUNNING", "Running"), ("FAILED", "Failed"), ("SUCCESSFUL", "Successful"), ], default="READY", max_length=10, verbose_name="status", ), ), migrations.RunPython(forwards, backwards), ] 0019_rename_django_task_new_ordering_idx_tasks_db_new_ordering_idx_and_more.py000066400000000000000000000010331514141543300413410ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/migrations# Generated by Django 6.0.2 on 2026-02-06 16:30 from django.db import migrations class Migration(migrations.Migration): dependencies = [ ("django_tasks_database", "0018_rename_succeeded_status"), ] operations = [ migrations.RenameIndex( model_name="dbtaskresult", new_name="tasks_db_new_ordering_idx", old_name="django_task_new_ordering_idx", ), migrations.RemoveField( model_name="dbtaskresult", name="metadata", ), ] django-tasks-db-0.12.0/django_tasks_db/migrations/__init__.py000066400000000000000000000000001514141543300241320ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/models.py000066400000000000000000000176301514141543300215230ustar00rootroot00000000000000import datetime import logging import uuid from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar import django from django.conf import settings from django.core.exceptions import SuspiciousOperation from django.db import models from django.db.models import F, Q from django.db.models.constraints import CheckConstraint from django.utils import timezone from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from django_tasks.base import ( DEFAULT_TASK_PRIORITY, DEFAULT_TASK_QUEUE_NAME, TASK_MAX_PRIORITY, TASK_MIN_PRIORITY, Task, TaskError, TaskResultStatus, ) from django_tasks.utils import ( get_exception_traceback, get_module_path, ) from typing_extensions import ParamSpec from .compat import TASK_CLASSES from .utils import normalize_uuid, retry logger = logging.getLogger("django_tasks_db") T = TypeVar("T") P = ParamSpec("P") if TYPE_CHECKING: from .backend import TaskResult class GenericBase(Generic[P, T]): pass else: class GenericBase: """ https://code.djangoproject.com/ticket/33174 """ def __class_getitem__(cls, _): return cls def get_date_max() -> datetime.datetime: return datetime.datetime( 9999, 1, 1, tzinfo=datetime.timezone.utc if settings.USE_TZ else None ) class DBTaskResultQuerySet(models.QuerySet): def ready(self) -> "DBTaskResultQuerySet": """ Return tasks which are ready to be processed. """ return self.filter( status=TaskResultStatus.READY, ).filter( models.Q(run_after=get_date_max()) | models.Q(run_after__lte=timezone.now()) ) def successful(self) -> "DBTaskResultQuerySet": return self.filter(status=TaskResultStatus.SUCCESSFUL) def failed(self) -> "DBTaskResultQuerySet": return self.filter(status=TaskResultStatus.FAILED) def running(self) -> "DBTaskResultQuerySet": return self.filter(status=TaskResultStatus.RUNNING) def finished(self) -> "DBTaskResultQuerySet": return self.failed() | self.successful() @retry() def get_locked(self) -> Optional["DBTaskResult"]: """ Get a job, locking the row and accounting for deadlocks. """ return self.select_for_update(skip_locked=True).first() class DBTaskResult(GenericBase[P, T], models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) status = models.CharField( _("status"), choices=TaskResultStatus.choices, default=TaskResultStatus.READY, max_length=max(len(value) for value in TaskResultStatus.values), ) enqueued_at = models.DateTimeField(_("enqueued at"), auto_now_add=True) started_at = models.DateTimeField(_("started at"), null=True) finished_at = models.DateTimeField(_("finished at"), null=True) args_kwargs = models.JSONField(_("args kwargs")) priority = models.IntegerField(_("priority"), default=DEFAULT_TASK_PRIORITY) task_path = models.TextField(_("task path")) worker_ids = models.JSONField(_("worker id"), default=list) queue_name = models.CharField( _("queue name"), default=DEFAULT_TASK_QUEUE_NAME, max_length=32 ) backend_name = models.CharField(_("backend name"), max_length=32) run_after = models.DateTimeField(_("run after")) return_value = models.JSONField(_("return value"), default=None, null=True) exception_class_path = models.TextField(_("exception class path")) traceback = models.TextField(_("traceback")) objects = DBTaskResultQuerySet.as_manager() class Meta: ordering = [F("priority").desc(), F("run_after").asc()] verbose_name = _("Task Result") verbose_name_plural = _("Task Results") indexes = [ models.Index( "status", *ordering, name="tasks_db_new_ordering_idx", condition=Q(status=TaskResultStatus.READY), ), models.Index(fields=["queue_name"]), models.Index(fields=["backend_name"]), ] if django.VERSION >= (5, 1): constraints = [ CheckConstraint( condition=Q(priority__range=(TASK_MIN_PRIORITY, TASK_MAX_PRIORITY)), name="priority_range", ) ] else: constraints = [ CheckConstraint( check=Q(priority__range=(TASK_MIN_PRIORITY, TASK_MAX_PRIORITY)), name="priority_range", ) ] @property def task(self) -> Task[P, T]: task = import_string(self.task_path) if not isinstance(task, TASK_CLASSES): raise SuspiciousOperation( f"Task {self.id} does not point to a Task ({self.task_path})" ) return task.using( # type: ignore[no-any-return] priority=self.priority, queue_name=self.queue_name, run_after=None if self.run_after == get_date_max() else self.run_after, backend=self.backend_name, ) @property def task_result(self) -> "TaskResult[T]": from .backend import TaskResult task_result: TaskResult[T] = TaskResult( db_result=self, task=self.task, id=normalize_uuid(self.id), status=TaskResultStatus[self.status], enqueued_at=self.enqueued_at, started_at=self.started_at, last_attempted_at=self.started_at, finished_at=self.finished_at, args=self.args_kwargs["args"], kwargs=self.args_kwargs["kwargs"], backend=self.backend_name, errors=[], worker_ids=self.worker_ids, ) if self.status == TaskResultStatus.FAILED: task_result.errors.append( TaskError( exception_class_path=self.exception_class_path, traceback=self.traceback, ) ) object.__setattr__(task_result, "_return_value", self.return_value) return task_result @property def task_name(self) -> str: # If the function for an existing task is no longer available, it'll either raise an # ImportError or ModuleNotFoundError (a subclass of ImportError). try: return self.task.name except ImportError: pass try: return self.task_path.rsplit(".", 1)[1] except IndexError: return self.task_path @retry(backoff_delay=0) def claim(self, worker_id: str) -> None: """ Mark as job as being run """ self.status = TaskResultStatus.RUNNING self.started_at = timezone.now() self.worker_ids = [*self.worker_ids, worker_id] self.save(update_fields=["status", "started_at", "worker_ids"]) @retry() def set_successful(self, return_value: Any) -> None: self.status = TaskResultStatus.SUCCESSFUL self.finished_at = timezone.now() self.return_value = return_value self.exception_class_path = "" self.traceback = "" self.save( update_fields=[ "status", "return_value", "finished_at", "exception_class_path", "traceback", ] ) @retry() def set_failed(self, exc: BaseException) -> None: self.status = TaskResultStatus.FAILED self.finished_at = timezone.now() self.exception_class_path = get_module_path(type(exc)) self.traceback = get_exception_traceback(exc) self.return_value = None self.save( update_fields=[ "status", "return_value", "finished_at", "exception_class_path", "traceback", ] ) django-tasks-db-0.12.0/django_tasks_db/py.typed000066400000000000000000000000001514141543300213440ustar00rootroot00000000000000django-tasks-db-0.12.0/django_tasks_db/signal_handlers.py000066400000000000000000000005441514141543300233710ustar00rootroot00000000000000from typing import Any from django.db.models.signals import pre_save from django.dispatch import receiver from .models import DBTaskResult, get_date_max @receiver(pre_save, sender=DBTaskResult) def set_run_after(sender: Any, instance: DBTaskResult, **kwargs: Any) -> None: if instance.run_after is None: instance.run_after = get_date_max() django-tasks-db-0.12.0/django_tasks_db/utils.py000066400000000000000000000053561514141543300214020ustar00rootroot00000000000000import time from collections.abc import Callable, Generator from contextlib import contextmanager from functools import wraps from typing import Any, TypeVar from uuid import UUID import django from django.db import transaction from django.db.backends.base.base import BaseDatabaseWrapper from typing_extensions import ParamSpec T = TypeVar("T") P = ParamSpec("P") def connection_requires_manual_exclusive_transaction( connection: BaseDatabaseWrapper, ) -> bool: """ Determine whether the backend requires manual transaction handling. Extracted from `exclusive_transaction` for unit testing purposes. """ if connection.vendor != "sqlite": return False if django.VERSION < (5, 1): return True if not hasattr(connection, "transaction_mode"): # Manually called to set `transaction_mode` connection.get_connection_params() return connection.transaction_mode != "EXCLUSIVE" # type:ignore[attr-defined,no-any-return] @contextmanager def exclusive_transaction(using: str | None = None) -> Generator[Any, Any, Any]: """ Wrapper around `transaction.atomic` which ensures transactions on SQLite are exclusive. This functionality is built-in to Django 5.1+. """ connection: BaseDatabaseWrapper = transaction.get_connection(using) if connection_requires_manual_exclusive_transaction(connection): with connection.cursor() as c: c.execute("BEGIN EXCLUSIVE") try: yield finally: c.execute("COMMIT") else: with transaction.atomic(using=using): yield def normalize_uuid(val: str | UUID) -> str: """ Normalize a UUID into its dashed representation. This works around engines like MySQL which don't store values in a uuid field, and thus drops the dashes. """ if isinstance(val, str): val = UUID(val) return str(val) def retry(*, retries: int = 3, backoff_delay: float = 0.1) -> Callable: """ Retry the given code `retries` times, raising the final error. `backoff_delay` can be used to add a delay between attempts. """ def wrapper(f: Callable[P, T]) -> Callable[P, T]: @wraps(f) def inner_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: # type:ignore[return] for attempt in range(1, retries + 1): try: return f(*args, **kwargs) except KeyboardInterrupt: # Let the user ctrl-C out of the program without a retry raise except BaseException: if attempt == retries: raise time.sleep(backoff_delay) return inner_wrapper return wrapper django-tasks-db-0.12.0/docker-compose.yml000066400000000000000000000004621514141543300202020ustar00rootroot00000000000000services: postgresql: image: postgres:16-alpine environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres ports: - 15432:5432 mysql: image: mysql:8.4 environment: MYSQL_ROOT_PASSWORD: django MYSQL_DATABASE: django ports: - 13306:3306 django-tasks-db-0.12.0/justfile000066400000000000000000000021351514141543300163140ustar00rootroot00000000000000# Recipes @default: just --list test *ARGS: python -m manage check python -m manage makemigrations --dry-run --check --noinput python -m coverage run --source=django_tasks_db -m manage test --shuffle --noinput {{ ARGS }} python -m coverage report python -m coverage html test-fast *ARGS: python -m manage test --shuffle --noinput --settings tests.settings_fast {{ ARGS }} format: python -m ruff check django_tasks_db tests --fix python -m ruff format django_tasks_db tests lint: python -m ruff check django_tasks_db tests python -m ruff format django_tasks_db tests --check python -m mypy django_tasks_db tests start-dbs: docker-compose up -d test-sqlite *ARGS: python -m manage test --shuffle --noinput {{ ARGS }} test-postgres *ARGS: DATABASE_URL=postgres://postgres:postgres@localhost:15432/postgres python -m manage test --shuffle --noinput {{ ARGS }} test-mysql *ARGS: DATABASE_URL=mysql://root:django@127.0.0.1:13306/django python -m manage test --shuffle --noinput {{ ARGS }} test-dbs *ARGS: start-dbs test-postgres test-mysql test-sqlite django-tasks-db-0.12.0/manage.py000077500000000000000000000003701514141543300163500ustar00rootroot00000000000000#!/usr/bin/env python import os import sys if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") from django.core.management import execute_from_command_line execute_from_command_line(sys.argv) django-tasks-db-0.12.0/pyproject.toml000066400000000000000000000054231514141543300174630ustar00rootroot00000000000000[build-system] build-backend = "setuptools.build_meta" requires = [ "setuptools", ] [project] name = "django-tasks-db" description = "An ORM-based backend for Django Tasks" authors = [ {name = "Jake Howard"}, ] version = "0.12.0" readme = "README.md" license = "BSD-3-Clause" license-files = ["LICENSE"] classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Environment :: Web Environment", "Framework :: Django", "Framework :: Django", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", "Framework :: Django :: 5.1", "Framework :: Django :: 5.2", "Framework :: Django :: 6.0", "Intended Audience :: Developers", "Operating System :: OS Independent", "Natural Language :: English", "Topic :: Internet :: WWW/HTTP", "Typing :: Typed" ] requires-python = ">=3.10" dependencies = [ "Django>=4.2", "typing_extensions", "django-stubs-ext", "django-tasks>=0.12.0" ] [project.urls] Source = "https://github.com/RealOrangeOne/django-tasks-db" Issues = "https://github.com/RealOrangeOne/django-tasks-db/issues" Changelog = "https://github.com/RealOrangeOne/django-tasks-db/releases" [dependency-groups] dev = [ "ruff", "coverage", "django-stubs[compatible-mypy]", "dj-database-url", ] mysql = [ "mysqlclient" ] postgres = [ "psycopg[binary]", ] [tool.ruff.lint] select = ["E", "F", "I", "W", "N", "B", "A", "C4", "T20", "DJ", "S", "UP"] ignore = ["E501", "DJ008"] [tool.ruff.lint.per-file-ignores] "tests/db_worker_test_settings.py" = ["F403", "F405"] "tests/settings_fast.py" = ["F403", "F405"] "tests/**.py" = ["S101", "S603", "S105"] [tool.mypy] plugins = ["mypy_django_plugin.main"] warn_unused_ignores = true warn_return_any = true show_error_codes = true strict_optional = true implicit_optional = true disallow_subclassing_any = true disallow_untyped_calls = true disallow_untyped_defs = true disallow_incomplete_defs = true disallow_untyped_decorators = true check_untyped_defs = true ignore_missing_imports = true [tool.django-stubs] django_settings_module = "tests.settings" [tool.coverage.run] branch = true [tool.coverage.report] show_missing = true ignore_errors = true exclude_also = [ # Don't complain if tests don't hit defensive assertion code: "raise AssertionError", "raise NotImplementedError", # Don't complain about abstract methods, they aren't run: "@(abc.)?abstractmethod", # Nor complain about type checking "if TYPE_CHECKING:", ] django-tasks-db-0.12.0/tests/000077500000000000000000000000001514141543300157055ustar00rootroot00000000000000django-tasks-db-0.12.0/tests/__init__.py000066400000000000000000000000001514141543300200040ustar00rootroot00000000000000django-tasks-db-0.12.0/tests/db_worker_test_settings.py000066400000000000000000000005041514141543300232130ustar00rootroot00000000000000from .settings import * TASKS = {"default": {"BACKEND": "django_tasks_db.DatabaseBackend"}} # Force the test DB to be used if "sqlite" in DATABASES["default"]["ENGINE"]: DATABASES["default"]["NAME"] = DATABASES["default"]["TEST"]["NAME"] else: DATABASES["default"]["NAME"] = "test_" + DATABASES["default"]["NAME"] django-tasks-db-0.12.0/tests/settings.py000066400000000000000000000014471514141543300201250ustar00rootroot00000000000000import os import sys import dj_database_url BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) IN_TEST = "IN_TEST" in os.environ or (len(sys.argv) > 1 and sys.argv[1] == "test") ALLOWED_HOSTS = ["*"] INSTALLED_APPS = [ "django_tasks_db", "tests", ] SECRET_KEY = "abcde12345" DEFAULT_AUTO_FIELD = "django.db.models.AutoField" DATABASES = { "default": dj_database_url.config( default="sqlite:///" + os.path.join(BASE_DIR, "db.sqlite3") ) } if "sqlite" in DATABASES["default"]["ENGINE"]: DATABASES["default"]["TEST"] = {"NAME": os.path.join(BASE_DIR, "db-test.sqlite3")} USE_TZ = True TASKS = { "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["default", "queue-1"], } } if not IN_TEST: DEBUG = True django-tasks-db-0.12.0/tests/settings_fast.py000066400000000000000000000002361514141543300211350ustar00rootroot00000000000000from .settings import * # Unset custom test settings to use in-memory DB if "sqlite" in DATABASES["default"]["ENGINE"]: del DATABASES["default"]["TEST"] django-tasks-db-0.12.0/tests/tasks.py000066400000000000000000000027041514141543300174070ustar00rootroot00000000000000import time from typing import Any from django_tasks import TaskContext, task @task() def noop_task(*args: Any, **kwargs: Any) -> None: return None @task def noop_task_from_bare_decorator(*args: Any, **kwargs: Any) -> None: return None @task() async def noop_task_async(*args: Any, **kwargs: Any) -> None: return None @task() def calculate_meaning_of_life() -> int: return 42 @task() def failing_task_value_error() -> None: raise ValueError("This task failed due to ValueError") @task() def failing_task_system_exit() -> None: raise SystemExit("This task failed due to SystemExit") @task() def failing_task_keyboard_interrupt() -> None: raise KeyboardInterrupt("This task failed due to KeyboardInterrupt") @task() def complex_exception() -> None: raise ValueError(ValueError("This task failed")) @task() def complex_return_value() -> Any: # Return something which isn't JSON serializable nor picklable return lambda: True @task() def exit_task() -> None: exit(1) @task() def hang() -> None: """ Do nothing for 5 minutes """ time.sleep(300) @task() def sleep_for(seconds: float) -> None: time.sleep(seconds) @task(takes_context=True) def get_task_id(context: TaskContext) -> str: return context.task_result.id @task(takes_context=True) def test_context(context: TaskContext, attempt: int) -> None: assert isinstance(context, TaskContext) assert context.attempt == attempt django-tasks-db-0.12.0/tests/tests.py000066400000000000000000001667151514141543300174410ustar00rootroot00000000000000import contextlib import copy import logging import os import signal import subprocess import sys import time import uuid import warnings from collections import Counter from collections.abc import Iterator, Sequence from contextlib import redirect_stderr from datetime import datetime, timedelta from functools import partial from io import StringIO from typing import Any, cast from unittest import mock, skipIf import django from django import VERSION from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.core.management import call_command, execute_from_command_line from django.db import connection, connections, transaction from django.db.models import QuerySet from django.db.utils import IntegrityError, OperationalError from django.test import SimpleTestCase, TransactionTestCase, override_settings from django.test.testcases import _deferredSkip # type:ignore[attr-defined] from django.utils import timezone from django_tasks import ( TaskResultStatus, default_task_backend, task_backends, ) from django_tasks.base import Task from django_tasks.exceptions import InvalidTaskError, TaskResultDoesNotExist from django_tasks.signals import task_enqueued from django_tasks.utils import get_random_id from django_tasks_db import DatabaseBackend, compat from django_tasks_db.management.commands.prune_db_task_results import ( logger as prune_db_tasks_logger, ) from django_tasks_db.models import DBTaskResult from django_tasks_db.utils import ( connection_requires_manual_exclusive_transaction, exclusive_transaction, normalize_uuid, ) from tests import tasks as test_tasks def skipIfInMemoryDB() -> Any: # noqa:N802 return _deferredSkip( lambda: connection.vendor == "sqlite" and connection.is_in_memory_db(), # type:ignore[attr-defined] "Tests cannot run on in-memory DB", "skipIfInMemoryDB", ) class DatabaseBackendTestCase(TransactionTestCase): @contextlib.contextmanager def _capture_task_enqueued_signal( self, ) -> Iterator[list[tuple[type[DatabaseBackend], Any]]]: received: list[tuple[type[DatabaseBackend], Any]] = [] def capture_signal( sender: type[DatabaseBackend], task_result: Any, **kwargs: Any ) -> None: received.append((sender, task_result)) task_enqueued.connect( capture_signal, dispatch_uid="db_enqueue_signal", weak=False ) try: yield received finally: task_enqueued.disconnect(capture_signal, dispatch_uid="db_enqueue_signal") def get_task_count_in_new_connection(self) -> int: """ See what other connections see """ new_connection = connections.create_connection("default") try: with new_connection.cursor() as c: c.execute(str(DBTaskResult.objects.values("id").query)) return len(c.fetchall()) finally: new_connection.close() def test_using_correct_backend(self) -> None: self.assertEqual(default_task_backend, task_backends["default"]) self.assertIsInstance(task_backends["default"], DatabaseBackend) self.assertEqual(default_task_backend.alias, "default") self.assertEqual(default_task_backend.options, {}) def test_enqueue_task(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: with self.subTest(task), self.assertNumQueries(1): with self._capture_task_enqueued_signal() as received: result = cast(Task, task).enqueue(1, two=3) self.assertEqual(uuid.UUID(result.id).version, 4) self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) self.assertEqual(result.attempts, 0) self.assertEqual(received, [(DatabaseBackend, result)]) async def test_enqueue_task_async(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: with self.subTest(task): with self._capture_task_enqueued_signal() as received: result = await cast(Task, task).aenqueue() self.assertEqual(uuid.UUID(result.id).version, 4) self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) self.assertEqual(result.attempts, 0) self.assertEqual(received, [(DatabaseBackend, result)]) def test_get_result(self) -> None: with self.assertNumQueries(1): result = default_task_backend.enqueue(test_tasks.noop_task, [], {}) with self.assertNumQueries(1): new_result = default_task_backend.get_result(result.id) self.assertEqual(result, new_result) async def test_get_result_async(self) -> None: result = await default_task_backend.aenqueue(test_tasks.noop_task, [], {}) new_result = await default_task_backend.aget_result(result.id) self.assertEqual(result, new_result) def test_refresh_result(self) -> None: result = default_task_backend.enqueue( test_tasks.calculate_meaning_of_life, (), {} ) DBTaskResult.objects.all().update( status=TaskResultStatus.SUCCESSFUL, started_at=timezone.now(), finished_at=timezone.now(), return_value=42, worker_ids=[get_random_id()], ) self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) with self.assertNumQueries(1): result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) self.assertEqual(result.attempts, 1) async def test_refresh_result_async(self) -> None: result = await default_task_backend.aenqueue( test_tasks.calculate_meaning_of_life, (), {} ) await DBTaskResult.objects.all().aupdate( status=TaskResultStatus.SUCCESSFUL, started_at=timezone.now(), finished_at=timezone.now(), return_value=42, worker_ids=[get_random_id()], ) self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) await result.arefresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) self.assertEqual(result.attempts, 1) def test_get_missing_result(self) -> None: with self.assertRaises(TaskResultDoesNotExist): default_task_backend.get_result(str(uuid.uuid4())) async def test_async_get_missing_result(self) -> None: with self.assertRaises(TaskResultDoesNotExist): await default_task_backend.aget_result(str(uuid.uuid4())) def test_invalid_uuid(self) -> None: with self.assertRaises(TaskResultDoesNotExist): default_task_backend.get_result("123") async def test_async_invalid_uuid(self) -> None: with self.assertRaises(TaskResultDoesNotExist): await default_task_backend.aget_result("123") def test_invalid_task_path(self) -> None: db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [["exit", "1"]], "kwargs": {}}, task_path="subprocess.check_output", backend_name="default", ) with self.assertRaisesMessage( SuspiciousOperation, f"Task {db_task_result.id} does not point to a Task ({db_task_result.task_path})", ): _ = db_task_result.task def test_missing_task_path(self) -> None: db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [], "kwargs": {}}, task_path="missing.func", backend_name="default", ) with self.assertRaises(ImportError): _ = db_task_result.task def test_task_name(self) -> None: for task_path, expected_task_name in [ ("tests.tasks.noop_task", "noop_task"), ("tests.tasks.task_not_found", "task_not_found"), ("tests.tasks.module_not_found.module_not_found", "module_not_found"), ("unexpected_function", "unexpected_function"), ]: with self.subTest(task_path): db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [], "kwargs": {}}, task_path=task_path, backend_name="default", ) self.assertEqual(db_task_result.task_name, expected_task_name) def test_check(self) -> None: errors = list(default_task_backend.check()) self.assertEqual(len(errors), 0, errors) @override_settings(INSTALLED_APPS=[]) def test_database_backend_app_missing(self) -> None: errors = list(default_task_backend.check()) self.assertEqual(len(errors), 1) self.assertIn("django_tasks_db", errors[0].hint) # type:ignore[arg-type] def test_priority_range_check(self) -> None: with self.assertRaises(IntegrityError): DBTaskResult.objects.create( task_path="", backend_name="default", priority=-101, args_kwargs={} ) with self.assertRaises(IntegrityError): DBTaskResult.objects.create( task_path="", backend_name="default", priority=101, args_kwargs={} ) # Django accepts the float, but only stores an int result = DBTaskResult.objects.create( task_path="", backend_name="default", priority=3.1, args_kwargs={} ) result.refresh_from_db() self.assertEqual(result.priority, 3) DBTaskResult.objects.create( task_path="", backend_name="default", priority=100, args_kwargs={} ) DBTaskResult.objects.create( task_path="", backend_name="default", priority=-100, args_kwargs={} ) DBTaskResult.objects.create( task_path="", backend_name="default", priority=0, args_kwargs={} ) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", } } ) def test_doesnt_wait_until_transaction_commit(self) -> None: with transaction.atomic(): result = test_tasks.noop_task.enqueue() self.assertIsNotNone(result.enqueued_at) self.assertEqual(DBTaskResult.objects.count(), 1) # SQLite locks the table during this transaction if connection.vendor != "sqlite": self.assertEqual(self.get_task_count_in_new_connection(), 0) if connection.vendor != "sqlite": self.assertEqual(self.get_task_count_in_new_connection(), 1) def test_enqueue_logs(self) -> None: with self.assertLogs("django_tasks", level="DEBUG") as captured_logs: result = test_tasks.noop_task.enqueue() self.assertEqual(len(captured_logs.output), 1) self.assertIn("enqueued", captured_logs.output[0]) self.assertIn(result.id, captured_logs.output[0]) def test_index_scan_for_ready(self) -> None: test_tasks.noop_task.enqueue() # Quickly duplicate tasks db_task = DBTaskResult.objects.get() db_task.id = None DBTaskResult.objects.bulk_create([copy.copy(db_task) for _ in range(5000)]) # Update query plan for certain databases if connection.vendor == "postgresql": with connection.cursor() as c: c.execute(f"ANALYZE {DBTaskResult._meta.db_table};") elif connection.vendor == "mysql": with connection.cursor() as c: c.execute(f"ANALYZE TABLE {DBTaskResult._meta.db_table};") plan = DBTaskResult.objects.ready().explain() if connection.vendor == "postgresql": self.assertIn("tasks_db_new_ordering_idx", plan) elif connection.vendor == "sqlite": self.assertIn("USING INDEX tasks_db_new_ordering_idx", plan) elif connection.vendor == "mysql": self.assertIn("Index lookup", plan) self.assertIn("using tasks_db_new_ordering_idx", plan) else: self.fail("Unknown database engine") def test_run_after_tz(self) -> None: for use_tz in [True, False]: with self.subTest(use_tz=use_tz): with override_settings(USE_TZ=use_tz): result = test_tasks.noop_task.enqueue() self.assertIsNone( DBTaskResult.objects.get(id=result.id).task.run_after ) def test_run_after_null_0016_migration(self) -> None: from datetime import timezone for use_tz in [True, False]: with self.subTest(use_tz=use_tz): with override_settings(USE_TZ=use_tz): result = test_tasks.noop_task.enqueue() db_result = DBTaskResult.objects.get(id=result.id) # Literal taken from migration db_result.run_after = datetime( 9999, 1, 1, tzinfo=timezone.utc if use_tz else None, ) with warnings.catch_warnings(): warnings.filterwarnings( "ignore", module="django.db", category=RuntimeWarning ) db_result.save() self.assertIsNone( DBTaskResult.objects.get(id=result.id).task.run_after ) def test_validate_on_enqueue(self) -> None: with override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["unknown_queue"], } } ): task_with_custom_queue_name = test_tasks.noop_task.using( queue_name="unknown_queue" ) with self.assertRaisesMessage( InvalidTaskError, "Queue 'unknown_queue' is not valid for backend" ): task_with_custom_queue_name.enqueue() async def test_validate_on_aenqueue(self) -> None: with override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["unknown_queue"], } } ): task_with_custom_queue_name = test_tasks.noop_task.using( queue_name="unknown_queue" ) with self.assertRaisesMessage( InvalidTaskError, "Queue 'unknown_queue' is not valid for backend" ): await task_with_custom_queue_name.aenqueue() def test_custom_id_function(self) -> None: for id_function in ["uuid.uuid1", uuid.uuid1]: with self.subTest(id_function): with override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "OPTIONS": {"id_function": id_function}, } } ): result = test_tasks.noop_task.enqueue() self.assertEqual(uuid.UUID(result.id).version, 1) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "OPTIONS": {"id_function": "missing.function"}, } } ) def test_unknown_id_function(self) -> None: with self.assertRaises(ImportError): test_tasks.noop_task.enqueue() @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "OPTIONS": { "id_function": "django.contrib.postgres.functions.RandomUUID" }, } } ) @skipIf(connection.vendor != "postgresql", "RandomUUID only works on postgres") @skipIf(VERSION < (6, 0), "DB expressions are only supported on 6.0+") def test_postgres_db_id_function(self) -> None: with self.assertNumQueries(1) as c: result = test_tasks.noop_task.enqueue() self.assertIn("GEN_RANDOM_UUID", c.captured_queries[0]["sql"]) self.assertEqual(uuid.UUID(result.id).version, 4) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "OPTIONS": {"id_function": "django.db.models.functions.Now"}, } } ) @skipIf(VERSION >= (6, 0), "DB expressions are supported on 6.0+") def test_postgres_id_function_expression(self) -> None: with self.assertRaisesMessage( ImproperlyConfigured, "id_function cannot be a database expression until Django 6.0", ): test_tasks.noop_task.enqueue() @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["default", "queue-1"], }, "dummy": {"BACKEND": "django_tasks.backends.dummy.DummyBackend"}, } ) class DatabaseBackendWorkerTestCase(TransactionTestCase): worker_id = get_random_id() run_worker = staticmethod( partial( call_command, "db_worker", verbosity=0, batch=True, interval=0, startup_delay=False, worker_id=worker_id, ) ) def tearDown(self) -> None: logger = logging.getLogger("django_tasks_db") tasks_logger = logging.getLogger("django_tasks") # Reset the logger after every run, to ensure the correct `stdout` is used for handler in logger.handlers: logger.removeHandler(handler) for handler in tasks_logger.handlers: tasks_logger.removeHandler(handler) def test_run_enqueued_task(self) -> None: for task in [ test_tasks.noop_task, # test_tasks.noop_task_async, ]: with self.subTest(task): result = cast(Task, task).enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) self.assertEqual(result.status, TaskResultStatus.READY) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() self.assertEqual(result.status, TaskResultStatus.READY) self.assertEqual(result.attempts, 0) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) self.assertEqual(result.attempts, 1) self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_batch_processes_all_tasks(self) -> None: for _ in range(3): test_tasks.noop_task.enqueue() test_tasks.failing_task_value_error.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 4) with self.assertNumQueries(27 if connection.vendor == "mysql" else 23): self.run_worker() self.assertEqual(DBTaskResult.objects.ready().count(), 0) self.assertEqual(DBTaskResult.objects.successful().count(), 3) self.assertEqual(DBTaskResult.objects.failed().count(), 1) def test_no_tasks(self) -> None: with self.assertNumQueries(3): self.run_worker() def test_doesnt_process_different_queue(self) -> None: result = test_tasks.noop_task.using(queue_name="queue-1").enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(3): self.run_worker() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker(queue_name=result.task.queue_name) self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_process_all_queues(self) -> None: test_tasks.noop_task.using(queue_name="queue-1").enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(3): self.run_worker() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker(queue_name="*") self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_failing_task(self) -> None: result = test_tasks.failing_task_value_error.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() self.assertEqual(result.status, TaskResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore self.assertGreaterEqual(result.finished_at, result.started_at) # type: ignore self.assertEqual(result.status, TaskResultStatus.FAILED) with self.assertRaisesMessage(ValueError, "Task failed"): result.return_value # noqa: B018 self.assertEqual(result.errors[0].exception_class, ValueError) traceback = result.errors[0].traceback self.assertTrue( traceback and traceback.endswith("ValueError: This task failed due to ValueError\n"), traceback, ) self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_complex_exception(self) -> None: result = test_tasks.complex_exception.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() self.assertEqual(result.status, TaskResultStatus.READY) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore self.assertGreaterEqual(result.finished_at, result.started_at) # type: ignore self.assertEqual(result.status, TaskResultStatus.FAILED) with self.assertRaisesMessage(ValueError, "Task failed"): result.return_value # noqa: B018 self.assertEqual(result.errors[0].exception_class, ValueError) self.assertIn( 'ValueError(ValueError("This task failed"))', result.errors[0].traceback ) self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_complex_return_value(self) -> None: result = test_tasks.complex_return_value.enqueue() self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertIsNone(result._return_value) self.assertEqual(result.errors[0].exception_class, TypeError) self.assertIn("is not JSON serializable", result.errors[0].traceback) def test_doesnt_process_different_backend(self) -> None: result = test_tasks.failing_task_value_error.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(3): self.run_worker(backend_name="dummy") self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker(backend_name=result.backend) self.assertEqual(DBTaskResult.objects.ready().count(), 0) def test_unknown_backend(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--backend", "unknown"] ) self.assertIn("The connection 'unknown' doesn't exist.", output.getvalue()) def test_incorrect_backend(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--backend", "dummy"] ) self.assertIn("Backend 'dummy' is not a database backend", output.getvalue()) def test_negative_interval(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--interval", "-1"] ) self.assertIn("Must be greater than zero", output.getvalue()) def test_infinite_interval(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--interval", "inf"] ) self.assertIn("Must be a finite floating point value", output.getvalue()) def test_fractional_interval(self) -> None: with mock.patch( "django_tasks_db.management.commands.db_worker.Worker" ) as worker_class: execute_from_command_line( ["django-admin", "db_worker", "--interval", "0.1"] ) self.assertEqual(worker_class.mock_calls[0].kwargs["interval"], 0.1) def test_negative_max_tasks(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--max-tasks", "-1"] ) self.assertIn("Must be greater than zero", output.getvalue()) def test_too_long_worker_id(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--worker-id", "A" * 65] ) self.assertIn( "Worker ids must be shorter than 64 characters", output.getvalue() ) def test_empty_worker_id(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "db_worker", "--worker-id", ""] ) self.assertIn("Worker id must not be empty", output.getvalue()) def test_run_after(self) -> None: result = test_tasks.noop_task.using( run_after=timezone.now() + timedelta(hours=10) ).enqueue() self.assertEqual(DBTaskResult.objects.count(), 1) self.assertEqual(DBTaskResult.objects.ready().count(), 0) with self.assertNumQueries(3): self.run_worker() self.assertEqual(DBTaskResult.objects.count(), 1) self.assertEqual(DBTaskResult.objects.ready().count(), 0) self.assertEqual(DBTaskResult.objects.successful().count(), 0) DBTaskResult.objects.filter(id=result.id).update(run_after=timezone.now()) self.assertEqual(DBTaskResult.objects.ready().count(), 1) with self.assertNumQueries(9 if connection.vendor == "mysql" else 8): self.run_worker() self.assertEqual(DBTaskResult.objects.ready().count(), 0) self.assertEqual(DBTaskResult.objects.successful().count(), 1) def test_run_after_priority(self) -> None: old_task = test_tasks.noop_task.using( priority=20, run_after=timezone.now() - timedelta(hours=2) ).enqueue() very_old_task = test_tasks.noop_task.using( priority=20, run_after=timezone.now() - timedelta(hours=10) ).enqueue() far_future_result = test_tasks.noop_task.using( run_after=timezone.now() + timedelta(hours=10) ).enqueue() high_priority_far_future_result = test_tasks.noop_task.using( priority=10, run_after=timezone.now() + timedelta(hours=10) ).enqueue() future_result = test_tasks.noop_task.using( run_after=timezone.now() + timedelta(hours=2) ).enqueue() high_priority_result = test_tasks.noop_task.using(priority=10).enqueue() low_priority_result = test_tasks.noop_task.using(priority=2).enqueue() lower_priority_result = test_tasks.noop_task.using(priority=-2).enqueue() self.assertEqual( [dbt.task_result for dbt in DBTaskResult.objects.all()], [ very_old_task, old_task, high_priority_far_future_result, high_priority_result, low_priority_result, future_result, far_future_result, lower_priority_result, ], ) self.assertEqual( [dbt.task_result for dbt in DBTaskResult.objects.ready()], [ very_old_task, old_task, high_priority_result, low_priority_result, lower_priority_result, ], ) def test_verbose_logging(self) -> None: result = test_tasks.noop_task.enqueue() stdout = StringIO() self.run_worker(verbosity=3, stdout=stdout, stderr=stdout) self.assertEqual( stdout.getvalue().splitlines(), [ f"Starting worker worker_id={self.worker_id} queues=default", f"Task id={result.id} path=tests.tasks.noop_task state=RUNNING", f"Task id={result.id} path=tests.tasks.noop_task state=SUCCESSFUL", f"No more tasks to run for worker_id={self.worker_id} - exiting gracefully.", ], ) def test_invalid_task_path(self) -> None: db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [["exit", "1"]], "kwargs": {}}, task_path="subprocess.check_output", backend_name="default", ) self.run_worker() db_task_result.refresh_from_db() self.assertEqual(db_task_result.status, TaskResultStatus.FAILED) def test_missing_task_path(self) -> None: db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [], "kwargs": {}}, task_path="missing.func", backend_name="default", ) self.run_worker() db_task_result.refresh_from_db() self.assertEqual(db_task_result.status, TaskResultStatus.FAILED) def test_worker_doesnt_exit(self) -> None: result = test_tasks.exit_task.enqueue() self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) @skipIf(connection.vendor == "sqlite", "SQLite locks the entire database") def test_worker_with_locked_rows(self) -> None: result_1 = test_tasks.noop_task.enqueue() new_connection = connections.create_connection("default") with transaction.atomic(): locked_tasks_query = str(DBTaskResult.objects.select_for_update().query) try: # Start a transaction in the other connection with new_connection.cursor() as c: c.execute("BEGIN") # Lock the current rows in the table with new_connection.cursor() as c: c.execute(locked_tasks_query) results = list(c.fetchall()) self.assertEqual(len(results), 1) # Add another task which isn't locked result_2 = test_tasks.noop_task.enqueue() self.run_worker() finally: new_connection.close() result_1.refresh() result_2.refresh() self.assertEqual(result_1.status, TaskResultStatus.READY) self.assertEqual(result_2.status, TaskResultStatus.SUCCESSFUL) def test_max_tasks(self) -> None: results = [test_tasks.noop_task.enqueue() for _ in range(5)] stdout = StringIO() self.run_worker(max_tasks=2, stdout=stdout, verbosity=3) self.assertIn("Run maximum tasks (2)", stdout.getvalue()) for result in results: result.refresh() statuses = Counter(result.status for result in results) self.assertEqual(statuses[TaskResultStatus.SUCCESSFUL], 2) self.assertEqual(statuses[TaskResultStatus.READY], 3) def test_takes_context(self) -> None: result = test_tasks.get_task_id.enqueue() self.run_worker() result.refresh() self.assertEqual(result.return_value, result.id) def test_context(self) -> None: result = test_tasks.test_context.enqueue(1) self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) def test_task_import_string(self) -> None: db_task_result = DBTaskResult.objects.create( args_kwargs={"args": [], "kwargs": {}}, task_path="tests.tasks.some_test", backend_name="default", ) self.run_worker() db_task_result.refresh_from_db() self.assertEqual(db_task_result.status, TaskResultStatus.FAILED) self.assertIn("ImportError", db_task_result.traceback) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", }, } ) class DatabaseTaskResultTestCase(TransactionTestCase): def execute_in_new_connection(self, sql: str | QuerySet) -> Sequence: if isinstance(sql, QuerySet): sql = str(sql.query) new_connection = connections.create_connection("default") try: with new_connection.cursor() as c: c.execute(sql) return cast(list, c.fetchall()) finally: new_connection.close() def test_cross_connection(self) -> None: test_tasks.noop_task.enqueue() test_tasks.noop_task.enqueue() self.assertEqual(DBTaskResult.objects.count(), 2) self.assertEqual(DBTaskResult.objects.using("default").count(), 2) self.assertEqual( len(self.execute_in_new_connection(DBTaskResult.objects.all())), 2, ) @skipIf(connection.vendor == "sqlite", "SQLite handles locks differently") def test_locks_tasks(self) -> None: test_tasks.noop_task.enqueue() test_tasks.noop_task.enqueue() with transaction.atomic(): self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), 2, ) self.assertIsNotNone(DBTaskResult.objects.get_locked()) self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), # MySQL likes to lock all the rows 0 if connection.vendor == "mysql" else 1, ) DBTaskResult.objects.get_locked() with transaction.atomic(): # The original transaction has closed, so the result is unlocked self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), 2, ) @skipIf(connection.vendor != "sqlite", "SQLite handles locks differently") def test_locks_tasks_sqlite(self) -> None: result = test_tasks.noop_task.enqueue() with exclusive_transaction(): locked_result = DBTaskResult.objects.get_locked() self.assertEqual(result.id, str(locked_result.id)) # type:ignore[union-attr] with self.assertRaisesMessage(OperationalError, "is locked"): self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) # The original transaction has closed, so the database is unlocked self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) @skipIf(connection.vendor == "sqlite", "SQLite handles locks differently") def test_locks_tasks_filtered(self) -> None: result = test_tasks.noop_task.using(priority=10).enqueue() test_tasks.noop_task.enqueue() with transaction.atomic(): self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), 2, ) locked_result = DBTaskResult.objects.filter( priority=result.task.priority ).get_locked() self.assertEqual(str(locked_result.id), result.id) self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), 1, ) with transaction.atomic(): # The original transaction has closed, so the result is unlocked self.assertEqual( len( self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) ), 2, ) @skipIf(connection.vendor != "sqlite", "SQLite handles locks differently") def test_locks_tasks_filtered_sqlite(self) -> None: result = test_tasks.noop_task.using(priority=10).enqueue() test_tasks.noop_task.enqueue() with exclusive_transaction(): locked_result = DBTaskResult.objects.filter( priority=result.task.priority ).get_locked() self.assertEqual(result.id, str(locked_result.id)) with self.assertRaisesMessage(OperationalError, "is locked"): self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) # The original transaction has closed, so the database is unlocked self.execute_in_new_connection( DBTaskResult.objects.select_for_update(skip_locked=True) ) @exclusive_transaction() def test_lock_no_rows(self) -> None: self.assertEqual(DBTaskResult.objects.count(), 0) self.assertIsNone(DBTaskResult.objects.all().get_locked()) @skipIf(connection.vendor == "sqlite", "SQLite handles locks differently") def test_get_locked_with_locked_rows(self) -> None: result_1 = test_tasks.noop_task.enqueue() new_connection = connections.create_connection("default") with transaction.atomic(): locked_tasks_query = str(DBTaskResult.objects.select_for_update().query) try: # Start a transaction in the other connection with new_connection.cursor() as c: c.execute("BEGIN") # Lock the current rows in the table from the other connection with new_connection.cursor() as c: c.execute(locked_tasks_query) results = list(c.fetchall()) self.assertEqual(len(results), 1) self.assertEqual(normalize_uuid(results[0][0]), normalize_uuid(result_1.id)) with transaction.atomic(): # .count with skip_locked isn't supported self.assertEqual( len(DBTaskResult.objects.select_for_update(skip_locked=True)), 0 ) self.assertIsNone(DBTaskResult.objects.get_locked()) # Add another task which isn't locked result_2 = test_tasks.noop_task.enqueue() with transaction.atomic(): self.assertEqual( normalize_uuid( DBTaskResult.objects.select_for_update( skip_locked=True ).values_list("id", flat=True)[0] ), normalize_uuid(result_2.id), ) self.assertEqual( normalize_uuid(DBTaskResult.objects.get_locked().id), # type:ignore normalize_uuid(result_2.id), ) finally: new_connection.close() class ConnectionExclusiveTranscationTestCase(TransactionTestCase): def setUp(self) -> None: self.connection = connections.create_connection("default") def tearDown(self) -> None: self.connection.close() # connection.close() @skipIf(connection.vendor == "sqlite", "SQLite handled separately") def test_non_sqlite(self) -> None: self.assertFalse( connection_requires_manual_exclusive_transaction(self.connection) ) @skipIf( django.VERSION >= (5, 1), "Newer Django versions support custom transaction modes", ) @skipIf(connection.vendor != "sqlite", "SQLite only") def test_old_django_requires_manual_transaction(self) -> None: self.assertTrue( connection_requires_manual_exclusive_transaction(self.connection) ) @skipIf(django.VERSION < (5, 1), "Old Django versions require manual transactions") @skipIf(connection.vendor != "sqlite", "SQLite only") def test_explicit_transaction(self) -> None: # HACK: Set the attribute manually self.connection.transaction_mode = None # type:ignore[attr-defined] self.assertTrue( connection_requires_manual_exclusive_transaction(self.connection) ) self.connection.transaction_mode = "EXCLUSIVE" # type:ignore[attr-defined] self.assertFalse( connection_requires_manual_exclusive_transaction(self.connection) ) @skipIf(connection.vendor != "sqlite", "SQLite only") def test_exclusive_transaction(self) -> None: with self.assertNumQueries(2) as c: with exclusive_transaction(): pass self.assertEqual(c.captured_queries[0]["sql"], "BEGIN EXCLUSIVE") @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", "QUEUES": ["default", "queue-1"], }, "dummy": {"BACKEND": "django_tasks.backends.dummy.DummyBackend"}, } ) class DatabaseBackendPruneTaskResultsTestCase(TransactionTestCase): prune_task_results = staticmethod( partial(call_command, "prune_db_task_results", verbosity=0) ) def tearDown(self) -> None: # Reset the logger after every run, to ensure the correct `stdout` is used for handler in prune_db_tasks_logger.handlers: prune_db_tasks_logger.removeHandler(handler) def test_prunes_tasks(self) -> None: result = test_tasks.noop_task.enqueue() DBTaskResult.objects.all().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() ) self.assertEqual(DBTaskResult.objects.finished().count(), 1) stdout = StringIO() with self.assertNumQueries(3): self.prune_task_results(min_age_days=0, stdout=stdout, verbosity=3) self.assertEqual(DBTaskResult.objects.finished().count(), 0) with self.assertRaises(TaskResultDoesNotExist): result.refresh() self.assertEqual(stdout.getvalue().strip(), "Deleted 1 task result(s)") def test_doesnt_prune_new_tasks(self) -> None: result = test_tasks.noop_task.enqueue() self.assertEqual(DBTaskResult.objects.ready().count(), 1) stdout = StringIO() with self.assertNumQueries(3): self.prune_task_results(min_age_days=0, stdout=stdout, verbosity=3) self.assertEqual(DBTaskResult.objects.ready().count(), 1) result.refresh() self.assertEqual(stdout.getvalue().strip(), "Deleted 0 task result(s)") def test_doesnt_prune_running_tasks(self) -> None: result = test_tasks.noop_task.enqueue() DBTaskResult.objects.all().update(status=TaskResultStatus.RUNNING) self.assertEqual(DBTaskResult.objects.running().count(), 1) with self.assertNumQueries(3): self.prune_task_results(min_age_days=0) self.assertEqual(DBTaskResult.objects.running().count(), 1) result.refresh() def test_only_prunes_specified_queue(self) -> None: result = test_tasks.noop_task.enqueue() queue_1_result = test_tasks.noop_task.using(queue_name="queue-1").enqueue() DBTaskResult.objects.all().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() ) self.assertEqual(DBTaskResult.objects.successful().count(), 2) with self.assertNumQueries(3): self.prune_task_results(queue_name="queue-1", min_age_days=0) self.assertEqual(DBTaskResult.objects.successful().count(), 1) result.refresh() with self.assertRaises(TaskResultDoesNotExist): queue_1_result.refresh() def test_prune_all_queues(self) -> None: test_tasks.noop_task.enqueue() test_tasks.noop_task.using(queue_name="queue-1").enqueue() DBTaskResult.objects.all().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() ) self.assertEqual(DBTaskResult.objects.successful().count(), 2) with self.assertNumQueries(3): self.prune_task_results(queue_name="*", min_age_days=0) self.assertEqual(DBTaskResult.objects.successful().count(), 0) def test_min_age(self) -> None: one_day_result = test_tasks.noop_task.enqueue() DBTaskResult.objects.ready().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() - timedelta(days=1), ) three_day_result = test_tasks.noop_task.enqueue() DBTaskResult.objects.ready().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() - timedelta(days=3), ) self.assertEqual(DBTaskResult.objects.successful().count(), 2) with self.assertNumQueries(3): self.prune_task_results() self.assertEqual(DBTaskResult.objects.successful().count(), 2) with self.assertNumQueries(3): self.prune_task_results(min_age_days=3) self.assertEqual(DBTaskResult.objects.successful().count(), 1) one_day_result.refresh() with self.assertRaises(TaskResultDoesNotExist): three_day_result.refresh() with self.assertNumQueries(3): self.prune_task_results(min_age_days=1) self.assertEqual(DBTaskResult.objects.successful().count(), 0) def test_failed_min_age(self) -> None: successful_result = test_tasks.noop_task.enqueue() DBTaskResult.objects.ready().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() - timedelta(days=3), ) failed_result = test_tasks.noop_task.enqueue() DBTaskResult.objects.ready().update( status=TaskResultStatus.FAILED, finished_at=timezone.now() - timedelta(days=3), ) self.assertEqual(DBTaskResult.objects.finished().count(), 2) with self.assertNumQueries(3): self.prune_task_results() self.assertEqual(DBTaskResult.objects.finished().count(), 2) with self.assertNumQueries(3): self.prune_task_results(min_age_days=3, failed_min_age_days=5) self.assertEqual(DBTaskResult.objects.finished().count(), 1) failed_result.refresh() with self.assertRaises(TaskResultDoesNotExist): successful_result.refresh() with self.assertNumQueries(3): self.prune_task_results(min_age_days=3, failed_min_age_days=1) with self.assertRaises(TaskResultDoesNotExist): failed_result.refresh() def test_dry_run(self) -> None: test_tasks.noop_task.enqueue() DBTaskResult.objects.all().update( status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now() ) self.assertEqual(DBTaskResult.objects.count(), 1) stdout = StringIO() with self.assertNumQueries(1): self.prune_task_results( min_age_days=0, dry_run=True, stdout=stdout, verbosity=3 ) self.assertEqual(DBTaskResult.objects.count(), 1) self.assertEqual(stdout.getvalue().strip(), "Would delete 1 task result(s)") def test_unknown_backend(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "prune_db_task_results", "--backend", "unknown"] ) self.assertIn("The connection 'unknown' doesn't exist.", output.getvalue()) def test_incorrect_backend(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "prune_db_task_results", "--backend", "dummy"] ) self.assertIn("Backend 'dummy' is not a database backend", output.getvalue()) def test_negative_age(self) -> None: output = StringIO() with redirect_stderr(output): with self.assertRaises(SystemExit): execute_from_command_line( ["django-admin", "prune_db_task_results", "--min-age-days", "-1"] ) self.assertIn("Must be greater than zero", output.getvalue()) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_db.DatabaseBackend", }, } ) @skipIfInMemoryDB() class DatabaseWorkerProcessTestCase(TransactionTestCase): WORKER_STARTUP_TIME = 1 def setUp(self) -> None: self.processes: list[subprocess.Popen] = [] def tearDown(self) -> None: # Try n times to kill any remaining child processes for n in range(20): for process in self.processes: if process.poll() is None: if n >= 5: print("Still waiting for process", process.pid, process.args) # noqa: T201 process.kill() process.wait(1) def start_worker( self, args: list[str] | None = None, *, debug: bool = False, worker_id: str | None = None, ) -> subprocess.Popen: if args is None: args = [] if worker_id is None: worker_id = get_random_id() p = subprocess.Popen( [ sys.executable, "-m", "manage", "db_worker", "--verbosity", "3", "--no-startup-delay", "--worker-id", worker_id, *args, ], stdout=None if debug else subprocess.PIPE, stderr=None if debug else subprocess.STDOUT, env={ **os.environ, "DJANGO_SETTINGS_MODULE": "tests.db_worker_test_settings", "IN_TEST": "", }, text=True, ) self.processes.append(p) return p def test_run_subprocess(self) -> None: result = test_tasks.noop_task.enqueue() process = self.start_worker(["--batch"]) process.wait() self.assertEqual(process.returncode, 0) self.assertEqual(result.status, TaskResultStatus.READY) result.refresh() self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) @skipIf(sys.platform == "win32", "Terminate is always forceful on Windows") def test_interrupt_no_tasks(self) -> None: process = self.start_worker() time.sleep(self.WORKER_STARTUP_TIME) process.terminate() process.wait(timeout=0.5) self.assertEqual(process.returncode, 0) @skipIf(sys.platform == "win32", "Cannot emulate CTRL-C on Windows") def test_interrupt_signals(self) -> None: for sig in [ signal.SIGINT, # ctrl-c signal.SIGTERM, ]: with self.subTest(sig): result = test_tasks.sleep_for.enqueue(2) self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, []) self.assertGreater(result.args[0], self.WORKER_STARTUP_TIME) process = self.start_worker() # Make sure the task is running by now time.sleep(self.WORKER_STARTUP_TIME) result.refresh() self.assertEqual(result.status, TaskResultStatus.RUNNING) self.assertNotEqual( DBTaskResult.objects.get(id=result.id).worker_ids, [] ) process.send_signal(sig) process.wait(timeout=2) self.assertEqual(process.returncode, 0) result.refresh() self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) @skipIf(sys.platform == "win32", "Cannot emulate CTRL-C on Windows") def test_repeat_ctrl_c(self) -> None: result = test_tasks.hang.enqueue() self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, []) worker_id = get_random_id() process = self.start_worker(worker_id=worker_id) # Make sure the task is running by now time.sleep(self.WORKER_STARTUP_TIME) result.refresh() self.assertEqual(result.status, TaskResultStatus.RUNNING) self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [worker_id]) process.send_signal(signal.SIGINT) time.sleep(0.5) self.assertIsNone(process.poll()) result.refresh() self.assertEqual(result.status, TaskResultStatus.RUNNING) self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [worker_id]) process.send_signal(signal.SIGINT) process.wait(timeout=2) self.assertEqual(process.returncode, 0) result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertEqual(result.errors[0].exception_class, SystemExit) @skipIf(sys.platform == "win32", "Windows doesn't support SIGKILL") def test_kill(self) -> None: # Required to keep mypy happy assert hasattr(signal, "SIGKILL") result = test_tasks.hang.enqueue() process = self.start_worker() # Make sure the task is running by now time.sleep(self.WORKER_STARTUP_TIME) result.refresh() self.assertEqual(result.status, TaskResultStatus.RUNNING) process.kill() process.wait(timeout=2) self.assertEqual(process.returncode, -signal.SIGKILL) result.refresh() # TODO: https://github.com/RealOrangeOne/django-tasks-db/issues/46 self.assertEqual(result.status, TaskResultStatus.RUNNING) def test_system_exit_task(self) -> None: result = test_tasks.failing_task_system_exit.enqueue() process = self.start_worker(["--batch"]) process.wait(timeout=2) self.assertEqual(process.returncode, 0) result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertEqual(result.errors[0].exception_class, SystemExit) def test_keyboard_interrupt_task(self) -> None: result = test_tasks.failing_task_keyboard_interrupt.enqueue() process = self.start_worker(["--batch"]) process.wait(timeout=2) self.assertEqual(process.returncode, 0) result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertEqual(result.errors[0].exception_class, KeyboardInterrupt) def test_multiple_workers(self) -> None: results = [test_tasks.sleep_for.enqueue(0.1) for _ in range(10)] for _ in range(3): self.start_worker(["--batch"]) time.sleep(self.WORKER_STARTUP_TIME) for process in self.processes: process.wait(timeout=5) self.assertIsNotNone(process.returncode) for result in results: result.refresh() self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) all_output = "" for process in self.processes: stdout_text = process.stdout.read() # type:ignore[union-attr] all_output += stdout_text self.assertIn("gracefully", stdout_text) for result in results: # Running and successful self.assertEqual(all_output.count(result.id), 2) class CompatTestCase(SimpleTestCase): def test_compat_has_django_task(self) -> None: self.assertIn(Task, compat.TASK_CLASSES) if VERSION >= (6, 0): from django.tasks.base import Task as DjangoTask self.assertIn(DjangoTask, compat.TASK_CLASSES)