pax_global_header 0000666 0000000 0000000 00000000064 15244316143 0014515 g ustar 00root root 0000000 0000000 52 comment=59702622d316a311b0f398a30b410b4e4e863c12
RealOrangeOne-django-tasks-db-5970262/ 0000775 0000000 0000000 00000000000 15244316143 0017344 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/.github/ 0000775 0000000 0000000 00000000000 15244316143 0020704 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/.github/workflows/ 0000775 0000000 0000000 00000000000 15244316143 0022741 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/.github/workflows/ci.yml 0000664 0000000 0000000 00000011204 15244316143 0024055 0 ustar 00root root 0000000 0000000 name: CI
on:
pull_request:
push:
branches:
- master
tags:
- "*"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [windows-latest, macos-latest, ubuntu-latest]
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
django-version: ["5.2", "6.0"]
exclude:
- django-version: "6.0"
python-version: "3.10"
- django-version: "6.0"
python-version: "3.11"
steps:
- uses: actions/checkout@v6
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v5
with:
path: ~/.cache/pip
key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }}
- uses: taiki-e/install-action@just
- name: Install dependencies
run: |
pip install --upgrade pip
pip install -e . --group dev
pip install Django~=${{ matrix.django-version }}
- name: Install compat deps
run: pip install -e '.[compat]'
if: matrix.django-version != '6.0'
- name: Lint
run: just lint
- name: Run fast tests
if: ${{ !cancelled() }}
run: just test-fast -v2
- name: Run tests
if: ${{ !cancelled() }}
run: just test -v2 --keepdb
test-postgres:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
strategy:
fail-fast: false
matrix:
django-version: ["5.2", "6.0"]
steps:
- uses: actions/checkout@v6
- name: Set up Python 3.14
uses: actions/setup-python@v6
with:
python-version: "3.14"
- uses: actions/cache@v5
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
- uses: taiki-e/install-action@just
- name: Install dependencies
run: |
pip install --upgrade pip
pip install -e . --group dev --group postgres
pip install Django~=${{ matrix.django-version }}
- name: Install compat deps
run: pip install -e '.[compat]'
if: matrix.django-version != '6.0'
- name: Run tests
run: just test -v2
env:
DATABASE_URL: postgres://postgres:postgres@localhost/postgres
test-mysql:
runs-on: ubuntu-latest
services:
mysql:
image: ${{ matrix.engine }}
env:
MYSQL_ROOT_PASSWORD: django
MYSQL_DATABASE: django
ports:
- 3306:3306
strategy:
fail-fast: false
matrix:
django-version: ["5.2", "6.0"]
engine: ["mysql:lts", "mariadb:lts"]
steps:
- uses: actions/checkout@v6
- name: Set up Python 3.14
uses: actions/setup-python@v6
with:
python-version: "3.14"
- uses: actions/cache@v5
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
- uses: taiki-e/install-action@just
- name: Install dependencies
run: |
pip install --upgrade pip
pip install -e . --group dev --group mysql
pip install Django~=${{ matrix.django-version }}
- name: Install compat deps
run: pip install -e '.[compat]'
if: matrix.django-version != '6.0'
- name: Run tests
run: just test -v2
env:
DATABASE_URL: mysql://root:django@127.0.0.1/django
build:
permissions:
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
runs-on: ubuntu-latest
needs:
- test
- test-postgres
- test-mysql
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Install dependencies
run: |
python -m pip install --upgrade pip build
- name: Build package
run: python -m build
- name: Save built package
uses: actions/upload-artifact@v6
with:
name: package
path: dist
- name: Publish to PyPi
if: ${{ github.ref_type == 'tag' }}
uses: pypa/gh-action-pypi-publish@release/v1
with:
print-hash: true
RealOrangeOne-django-tasks-db-5970262/.gitignore 0000664 0000000 0000000 00000006703 15244316143 0021342 0 ustar 00root root 0000000 0000000 # Created by https://www.toptal.com/developers/gitignore/api/python
# Edit at https://www.toptal.com/developers/gitignore?templates=python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
*.sqlite3
*.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
### Python Patch ###
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
poetry.toml
# ruff
.ruff_cache/
# LSP config files
pyrightconfig.json
# End of https://www.toptal.com/developers/gitignore/api/python
# Editor config files
.vscode
RealOrangeOne-django-tasks-db-5970262/CONTRIBUTING.md 0000664 0000000 0000000 00000003106 15244316143 0021575 0 ustar 00root root 0000000 0000000 # Contributing
Found a bug? Want to fix an open issue? Got an idea for an improvement? Please contribute!
**All** contributions are welcome, from absolutely anyone. Just open a PR, Issue or Discussion (as relevant) - no need to ask beforehand. If you're going to work on an issue, it's a good idea to say so on the issue, to make sure work isn't duplicated.
## Development set up
Fork, then clone the repo:
```sh
git clone git@github.com:your-username/django-tasks-db.git
```
Set up a venv:
```sh
python -m venv .venv
source .venv/bin/activate
python -m pip install -e . --group dev
```
> [!TIP]
> To include support for a specific database, you can stack group flags (e.g. `--group dev --group postgres`). This is optional.
Then you can run the tests with the [just](https://just.systems/man/en/) command runner:
```sh
just test
```
If you don't have `just` installed, you can look in the `justfile` for the commands that are run.
To help with testing on different databases, there's a `docker-compose.yml` file to run PostgreSQL and MySQL in Docker, as well as some additional `just` commands for testing:
```sh
just start-dbs
just test-postgres
just test-mysql
just test-sqlite
# To run all of the above:
just test-dbs
```
Due to the database worker process tests, tests cannot run using an in-memory database, which makes them run quite slowly locally. If you’re not modifying the worker and want your tests to run faster, run:
```sh
just test-fast
```
## Opening the PR
Before opening a PR, ensure all tests and linters pass.
To check for linter issues, run:
```sh
just lint
```
RealOrangeOne-django-tasks-db-5970262/LICENSE 0000664 0000000 0000000 00000003017 15244316143 0020352 0 ustar 00root root 0000000 0000000 Copyright (c) Jake Howard and individual contributors.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
RealOrangeOne-django-tasks-db-5970262/README.md 0000664 0000000 0000000 00000005254 15244316143 0020631 0 ustar 00root root 0000000 0000000 # Django Tasks DB
[](https://github.com/RealOrangeOne/django-tasks-db/actions/workflows/ci.yml)
A [Django Tasks](https://docs.djangoproject.com/en/stable/topics/tasks/) backend which uses Django's ORM to store tasks in the database.
## Installation
```
python -m pip install django-tasks-db
```
First, add `django_tasks_db` to your `INSTALLED_APPS`:
```python
INSTALLED_APPS = [
# ...
"django_tasks_db",
]
```
Finally, add it to your `TASKS` configuration:
```python
TASKS = {
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["default"]
}
}
```
## Usage
### Worker
You can run the `db_worker` command to run tasks as they're created. Check the `--help` for more options.
```shell
./manage.py db_worker
```
In `DEBUG`, the worker will automatically reload when code is changed (or by using `--reload`). This is not recommended in production environments as tasks may not be stopped cleanly.
### Pruning old tasks
After a while, tasks may start to build up in your database. This can be managed using the `prune_db_task_results` management command, which deletes completed tasks according to the given retention policy. Check the `--help` for the available options.
### Customizing the task id
By default, the database worker uses `uuid.uuid4` to generate a task id. This can be customized using the `id_function` option:
```python
TASKS = {
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"OPTIONS": {
"id_function": "uuid.uuid7"
}
}
}
```
The `id_function` must return a UUID (either `uuid.UUID` or string representation). Additionally, the PostgreSQL-specific [`RandomUUID`](https://docs.djangoproject.com/en/stable/ref/contrib/postgres/functions/#django.contrib.postgres.functions.RandomUUID) or other database expressions are supported on Django 6.0+.
## Contributing
See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute.
Note: Prior to `0.12.0`, this backend was included in [`django-tasks`](https://github.com/RealOrangeOne/django-tasks/). Whilst the commit history was cleaned up, it's still quite messy. Don't look too closely.
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/ 0000775 0000000 0000000 00000000000 15244316143 0022460 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/__init__.py 0000664 0000000 0000000 00000000371 15244316143 0024572 0 ustar 00root root 0000000 0000000 # ruff: noqa: E402
import django_stubs_ext
django_stubs_ext.monkeypatch()
import importlib.metadata
__version__ = importlib.metadata.version(__name__)
from .backend import DatabaseBackend
__all__ = [
"DatabaseBackend",
"__version__",
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/admin.py 0000664 0000000 0000000 00000003700 15244316143 0024122 0 ustar 00root root 0000000 0000000 from datetime import datetime
from typing import Any
from django.contrib import admin
from django.http import HttpRequest
from django.utils.html import format_html
from .models import DBTaskResult, get_date_max
@admin.register(DBTaskResult)
class DBTaskResultAdmin(admin.ModelAdmin):
list_display = (
"id",
"task_name",
"status",
"enqueued_at",
"started_at",
"finished_at",
"priority",
"queue_name",
)
list_filter = ("status", "priority", "queue_name")
ordering = ["-enqueued_at"]
def has_add_permission(
self, request: HttpRequest, obj: DBTaskResult | None = None
) -> bool:
return False
def has_delete_permission(
self, request: HttpRequest, obj: DBTaskResult | None = None
) -> bool:
return False
def has_change_permission(
self, request: HttpRequest, obj: DBTaskResult | None = None
) -> bool:
return False
def get_readonly_fields(
self, request: HttpRequest, obj: DBTaskResult | None = None
) -> list[str]:
fields = [f.name for f in self.model._meta.fields]
fields[fields.index("traceback")] = "formatted_traceback"
fields[fields.index("run_after")] = "display_run_after"
return fields
def get_fields(
self, request: HttpRequest, obj: DBTaskResult | None = None
) -> list[Any]:
return self.get_readonly_fields(request, obj)
@admin.display(description="traceback")
def formatted_traceback(self, obj: DBTaskResult) -> str:
if not obj.traceback:
return "-"
return format_html(
"{}",
obj.traceback,
)
@admin.display(description="run after")
def display_run_after(self, obj: DBTaskResult) -> datetime | str:
if obj.run_after == get_date_max():
return self.get_empty_value_display()
return obj.run_after # type: ignore[no-any-return]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/apps.py 0000664 0000000 0000000 00000000376 15244316143 0024003 0 ustar 00root root 0000000 0000000 from django.apps import AppConfig
class TasksAppConfig(AppConfig):
name = "django_tasks_db"
label = "django_tasks_database"
verbose_name = "Tasks Database Backend"
def ready(self) -> None:
from . import signal_handlers # noqa
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/backend.py 0000664 0000000 0000000 00000012153 15244316143 0024423 0 ustar 00root root 0000000 0000000 from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, TypeVar
from django import VERSION
from django.apps import apps
from django.core import checks
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db.models import Expression
from django.utils.module_loading import import_string
from django.utils.version import PY311
from typing_extensions import ParamSpec
from .compat import (
BaseTaskBackend,
BaseTaskResult,
Task,
TaskResultDoesNotExist,
normalize_json,
task_enqueued,
)
if TYPE_CHECKING:
from .models import DBTaskResult
T = TypeVar("T")
P = ParamSpec("P")
@dataclass(frozen=True, slots=PY311, kw_only=True) # type: ignore[literal-required]
class TaskResult(BaseTaskResult[P, T]): # type: ignore[misc]
db_result: "DBTaskResult"
class DatabaseBackend(BaseTaskBackend):
supports_async_task = True
supports_get_result = True
supports_defer = True
supports_priority = True
def __init__(self, alias: str, params: dict) -> None:
super().__init__(alias, params)
if id_function := self.options.get("id_function"):
if callable(id_function):
self.id_function = id_function
else:
self.id_function = import_string(id_function)
else:
self.id_function = None
def _get_id(self) -> Any:
if self.id_function is None:
# Defer model import to avoid AppRegistryNotReady when the
# backend is instantiated before apps are fully loaded (e.g.
# by @task() decorators in third-party packages).
from .models import DBTaskResult
# Fall back to the default defined on the model
self.id_function = DBTaskResult._meta.pk.default
result_id = self.id_function()
if VERSION < (6, 0) and isinstance(result_id, Expression):
raise ImproperlyConfigured(
"id_function cannot be a database expression until Django 6.0"
)
return result_id
def _task_to_db_task(
self,
task: Task[P, T],
args: P.args, # type:ignore[valid-type]
kwargs: P.kwargs, # type:ignore[valid-type]
) -> "DBTaskResult":
from .models import DBTaskResult
return DBTaskResult.objects.create(
id=self._get_id(),
args_kwargs=normalize_json({"args": args, "kwargs": kwargs}),
priority=task.priority,
task_path=task.module_path,
queue_name=task.queue_name,
run_after=task.run_after, # type: ignore[misc]
backend_name=self.alias,
)
async def _atask_to_db_task(
self,
task: Task[P, T],
args: P.args, # type:ignore[valid-type]
kwargs: P.kwargs, # type:ignore[valid-type]
) -> "DBTaskResult":
from .models import DBTaskResult
return await DBTaskResult.objects.acreate(
id=self._get_id(),
args_kwargs=normalize_json({"args": args, "kwargs": kwargs}),
priority=task.priority,
task_path=task.module_path,
queue_name=task.queue_name,
run_after=task.run_after, # type: ignore[misc]
backend_name=self.alias,
)
def enqueue(
self,
task: Task[P, T],
args: P.args, # type:ignore[valid-type]
kwargs: P.kwargs, # type:ignore[valid-type]
) -> TaskResult[P, T]:
self.validate_task(task)
db_result = self._task_to_db_task(task, args, kwargs)
task_enqueued.send(type(self), task_result=db_result.task_result)
return db_result.task_result
async def aenqueue(
self,
task: Task[P, T],
args: P.args, # type:ignore[valid-type]
kwargs: P.kwargs, # type:ignore[valid-type]
) -> TaskResult[P, T]:
self.validate_task(task)
db_result = await self._atask_to_db_task(task, args, kwargs)
await task_enqueued.asend(type(self), task_result=db_result.task_result)
return db_result.task_result
def get_result(self, result_id: str) -> TaskResult:
from .models import DBTaskResult
try:
return DBTaskResult.objects.get(id=result_id).task_result
except (DBTaskResult.DoesNotExist, ValidationError) as e:
raise TaskResultDoesNotExist(result_id) from e
async def aget_result(self, result_id: str) -> TaskResult:
from .models import DBTaskResult
try:
return (await DBTaskResult.objects.aget(id=result_id)).task_result
except (DBTaskResult.DoesNotExist, ValidationError) as e:
raise TaskResultDoesNotExist(result_id) from e
def check(self, **kwargs: Any) -> list[checks.CheckMessage]:
if apps.is_installed("django_tasks_db"):
return super().check(**kwargs)
else:
backend_name = self.__class__.__name__
return [
checks.Error(
f"{backend_name} configured as django_tasks_db backend, but database app not installed",
"Insert 'django_tasks_db' in INSTALLED_APPS",
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/compat.py 0000664 0000000 0000000 00000006076 15244316143 0024326 0 ustar 00root root 0000000 0000000 from importlib.util import find_spec
from typing import TypeVar
from typing_extensions import ParamSpec
if find_spec("django.tasks"):
from django.tasks import task_backends
from django.tasks.backends.base import BaseTaskBackend
from django.tasks.base import (
DEFAULT_TASK_BACKEND_ALIAS,
DEFAULT_TASK_PRIORITY,
DEFAULT_TASK_QUEUE_NAME,
TASK_MAX_PRIORITY,
TASK_MIN_PRIORITY,
Task,
TaskContext,
TaskError,
TaskResultStatus,
)
from django.tasks.base import (
TaskResult as BaseTaskResult,
)
from django.tasks.exceptions import InvalidTaskBackend, TaskResultDoesNotExist
from django.tasks.signals import (
task_enqueued,
task_finished,
task_started,
)
from django.utils.json import normalize_json
TASKS_LOGGER = "django.tasks"
else:
try:
from typing import Generic
from django_tasks import ( # type: ignore[assignment,no-redef,unused-ignore]
task_backends,
)
from django_tasks.backends.base import ( # type: ignore[assignment,no-redef,unused-ignore]
BaseTaskBackend,
)
from django_tasks.base import ( # type: ignore[assignment,no-redef,unused-ignore]
DEFAULT_TASK_BACKEND_ALIAS,
DEFAULT_TASK_PRIORITY,
DEFAULT_TASK_QUEUE_NAME,
TASK_MAX_PRIORITY,
TASK_MIN_PRIORITY,
Task,
TaskContext,
TaskError,
TaskResultStatus,
)
from django_tasks.base import (
TaskResult as _TaskResultWithoutP,
)
from django_tasks.exceptions import ( # type: ignore[assignment,no-redef,unused-ignore]
InvalidTaskBackendError as InvalidTaskBackend,
)
from django_tasks.exceptions import ( # type: ignore[assignment,no-redef,unused-ignore]
TaskResultDoesNotExist,
)
from django_tasks.signals import ( # type: ignore[no-redef,unused-ignore]
task_enqueued,
task_finished,
task_started,
)
from django_tasks.utils import ( # type: ignore[no-redef,unused-ignore]
normalize_json,
)
TASKS_LOGGER = "django_tasks"
T = TypeVar("T")
P = ParamSpec("P")
class BaseTaskResult(_TaskResultWithoutP[T], Generic[P, T]): # type: ignore[misc,no-redef,unused-ignore]
pass
except (ImportError, ModuleNotFoundError) as e:
raise ValueError(
"Either use Django 6+ or include django-tasks with [compat] extra"
) from e
__all__ = [
"BaseTaskBackend",
"BaseTaskResult",
"BaseTaskResult",
"DEFAULT_TASK_BACKEND_ALIAS",
"DEFAULT_TASK_PRIORITY",
"DEFAULT_TASK_QUEUE_NAME",
"InvalidTaskBackend",
"TASK_MAX_PRIORITY",
"TASK_MIN_PRIORITY",
"TASKS_LOGGER",
"Task",
"TaskContext",
"TaskError",
"TaskResultDoesNotExist",
"TaskResultStatus",
"normalize_json",
"task_backends",
"task_enqueued",
"task_finished",
"task_started",
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/management/ 0000775 0000000 0000000 00000000000 15244316143 0024574 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/management/commands/ 0000775 0000000 0000000 00000000000 15244316143 0026375 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/management/commands/db_worker.py 0000664 0000000 0000000 00000031323 15244316143 0030727 0 ustar 00root root 0000000 0000000 import logging
import math
import os
import random
import signal
import sys
import time
from argparse import ArgumentParser, ArgumentTypeError, BooleanOptionalAction
from types import FrameType
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.core.management.base import BaseCommand, CommandError
from django.db import close_old_connections
from django.db.utils import OperationalError
from django.utils.autoreload import DJANGO_AUTORELOAD_ENV, run_with_reloader
from django.utils.crypto import get_random_string
from django_tasks_db.backend import DatabaseBackend
from django_tasks_db.compat import (
DEFAULT_TASK_BACKEND_ALIAS,
DEFAULT_TASK_QUEUE_NAME,
TASKS_LOGGER,
InvalidTaskBackend,
TaskContext,
task_backends,
task_finished,
task_started,
)
from django_tasks_db.models import DBTaskResult
from django_tasks_db.utils import exclusive_transaction, is_locked_database_exception
logger = logging.getLogger("django_tasks_db")
class Worker:
def __init__(
self,
*,
queue_names: list[str],
interval: float,
batch: bool,
backend_name: str,
startup_delay: bool,
max_tasks: int | None,
worker_id: str,
excluded_queue_names: list[str],
):
self.queue_names = queue_names
self.process_all_queues = "*" in queue_names
self.excluded_queue_names = excluded_queue_names
self.interval = interval
self.batch = batch
self.backend_name = backend_name
self.startup_delay = startup_delay
self.max_tasks = max_tasks
self.running = True
self.running_task = False
self._run_tasks = 0
self.worker_id = worker_id
def shutdown(self, signum: int, frame: FrameType | None) -> None:
if not self.running:
logger.warning(
"Received %s - terminating current task.", signal.strsignal(signum)
)
self.reset_signals()
sys.exit(1)
logger.warning(
"Received %s - shutting down gracefully... (press Ctrl+C again to force)",
signal.strsignal(signum),
)
self.running = False
if not self.running_task:
# If we're not currently running a task, exit immediately.
# This is useful if we're currently in a `sleep`.
sys.exit(0)
def configure_signals(self) -> None:
signal.signal(signal.SIGINT, self.shutdown)
signal.signal(signal.SIGTERM, self.shutdown)
if hasattr(signal, "SIGQUIT"):
signal.signal(signal.SIGQUIT, self.shutdown)
def reset_signals(self) -> None:
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
if hasattr(signal, "SIGQUIT"):
signal.signal(signal.SIGQUIT, signal.SIG_DFL)
def run(self) -> None:
logger.info(
"Starting worker worker_id=%s queues=%s",
self.worker_id,
",".join(self.queue_names),
)
if self.startup_delay and self.interval:
# Add a random small delay before starting to avoid a thundering herd
time.sleep(random.random()) # noqa: S311
while self.running:
# Check for dropped/expired connections right after waking up
close_old_connections()
tasks = DBTaskResult.objects.ready().filter(backend_name=self.backend_name)
if not self.process_all_queues:
tasks = tasks.filter(queue_name__in=self.queue_names)
if self.excluded_queue_names:
tasks = tasks.exclude(queue_name__in=self.excluded_queue_names)
with exclusive_transaction(tasks.db):
try:
task_result = tasks.get_locked()
retrieved_task_result = True
if task_result is not None:
# "claim" the task, so it isn't run by another worker process
task_result.claim(self.worker_id)
except OperationalError as e:
retrieved_task_result = False
# Ignore locked databases and keep trying.
# It should unlock eventually.
if is_locked_database_exception(e):
task_result = None
else:
raise
if task_result is not None:
self.run_task(task_result)
if self.batch and retrieved_task_result and task_result is None:
# If we're running in "batch" mode, terminate the loop (and thus the worker)
logger.info(
"No more tasks to run for worker_id=%s - exiting gracefully.",
self.worker_id,
)
return None
if self.max_tasks is not None and self._run_tasks >= self.max_tasks:
logger.info(
"Run maximum tasks (%d) on worker=%s - exiting gracefully.",
self._run_tasks,
self.worker_id,
)
return None
# Emulate Django's request behaviour and check for expired
# database connections periodically.
close_old_connections()
# If ctrl-c has just interrupted a task, self.running was cleared,
# and we should not sleep, but rather exit immediately.
if self.running and not task_result:
# Wait before checking for another task
time.sleep(self.interval)
def run_task(self, db_task_result: DBTaskResult) -> None:
"""
Run the given task, marking it as successful or failed.
"""
try:
self.running_task = True
task = db_task_result.task
task_result = db_task_result.task_result
backend_type = task.get_backend()
task_started.send(sender=backend_type, task_result=task_result)
if task.takes_context:
return_value = task.call(
TaskContext(task_result=task_result),
*task_result.args,
**task_result.kwargs,
)
else:
return_value = task.call(*task_result.args, **task_result.kwargs)
# Setting the return and success value inside the error handling,
# So errors setting it (eg JSON encode) can still be recorded
db_task_result.set_successful(return_value)
task_finished.send(
sender=backend_type, task_result=db_task_result.task_result
)
except BaseException as e:
db_task_result.set_failed(e)
try:
sender = type(db_task_result.task.get_backend())
task_result = db_task_result.task_result
except (ImportError, SuspiciousOperation):
logger.exception("Task id=%s failed unexpectedly", db_task_result.id)
else:
task_finished.send(
sender=sender,
task_result=task_result,
)
finally:
self.running_task = False
self._run_tasks += 1
def valid_backend_name(val: str) -> str:
try:
backend = task_backends[val]
except InvalidTaskBackend as e:
raise ArgumentTypeError(e.args[0]) from e
if not isinstance(backend, DatabaseBackend):
raise ArgumentTypeError(f"Backend '{val}' is not a database backend")
return val
def valid_interval(val: str) -> float:
num = float(val)
if not math.isfinite(num):
raise ArgumentTypeError("Must be a finite floating point value")
if num < 0:
raise ArgumentTypeError("Must be zero or greater")
return num
def valid_max_tasks(val: str) -> int:
num = int(val)
if num <= 0:
raise ArgumentTypeError("Must be greater than zero")
return num
def validate_worker_id(val: str) -> str:
if not val:
raise ArgumentTypeError("Worker id must not be empty")
if len(val) > 64:
raise ArgumentTypeError("Worker ids must be shorter than 64 characters")
return val
class Command(BaseCommand):
help = "Run a database background worker"
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument(
"--queue-name",
nargs="?",
default=DEFAULT_TASK_QUEUE_NAME,
type=str,
help="The queues to process. Separate multiple with a comma. To process all queues, use '*' (default: %(default)r)",
)
parser.add_argument(
"--exclude-queues",
nargs="?",
default="",
type=str,
help="Queues to exclude. Separate multiple with a comma.",
)
parser.add_argument(
"--interval",
nargs="?",
default=1,
type=valid_interval,
help="The interval (in seconds) to wait, when there are no tasks in the queue, before checking for tasks again (default: %(default)r)",
)
parser.add_argument(
"--batch",
action="store_true",
help="Process all outstanding tasks, then exit. Can be used in combination with --max-tasks.",
)
parser.add_argument(
"--reload",
action=BooleanOptionalAction,
default=settings.DEBUG,
help="Reload the worker on code changes. Not recommended for production as tasks may not be stopped cleanly (default: DEBUG)",
)
parser.add_argument(
"--backend",
nargs="?",
default=DEFAULT_TASK_BACKEND_ALIAS,
type=valid_backend_name,
dest="backend_name",
help="The backend to operate on (default: %(default)r)",
)
parser.add_argument(
"--no-startup-delay",
action="store_false",
dest="startup_delay",
help="Don't add a small delay at startup.",
)
parser.add_argument(
"--max-tasks",
nargs="?",
default=None,
type=valid_max_tasks,
help="If provided, the maximum number of tasks the worker will execute before exiting.",
)
parser.add_argument(
"--worker-id",
nargs="?",
type=validate_worker_id,
help="Worker id. MUST be unique across worker pool (default: auto-generate)",
default=get_random_string(32),
)
def configure_logging(self, verbosity: int) -> None:
tasks_logger = logging.getLogger(TASKS_LOGGER)
if verbosity == 0:
tasks_logger.setLevel(logging.CRITICAL)
logger.setLevel(logging.CRITICAL)
elif verbosity == 1:
tasks_logger.setLevel(logging.INFO)
logger.setLevel(logging.INFO)
else:
tasks_logger.setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
# If no handler is configured, the logs won't show,
# regardless of the set level.
if not tasks_logger.hasHandlers():
tasks_logger.addHandler(logging.StreamHandler(self.stdout))
if not logger.hasHandlers():
logger.addHandler(logging.StreamHandler(self.stdout))
def handle(
self,
*,
verbosity: int,
queue_name: str,
interval: float,
batch: bool,
backend_name: str,
startup_delay: bool,
reload: bool,
max_tasks: int | None,
worker_id: str,
exclude_queues: str,
**options: dict,
) -> None:
self.configure_logging(verbosity)
if reload and batch:
logger.warning(
"Warning: --reload and --batch cannot be specified together. Disabling autoreload."
)
reload = False
queue_names = queue_name.split(",")
excluded_queue_names = exclude_queues.split(",") if exclude_queues else []
if excluded_queue_names and "*" not in queue_names:
raise CommandError("--exclude-queues can only be used with --queue-name=*")
worker = Worker(
queue_names=queue_names,
interval=interval,
batch=batch,
backend_name=backend_name,
startup_delay=startup_delay,
max_tasks=max_tasks,
worker_id=worker_id,
excluded_queue_names=excluded_queue_names,
)
if reload:
if os.environ.get(DJANGO_AUTORELOAD_ENV) == "true":
# Only the child process should configure its signals
worker.configure_signals()
run_with_reloader(worker.run)
else:
worker.configure_signals()
worker.run()
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/management/commands/prune_db_task_results.py 0000664 0000000 0000000 00000010200 15244316143 0033341 0 ustar 00root root 0000000 0000000 import logging
from argparse import ArgumentParser, ArgumentTypeError
from datetime import timedelta
from django.core.management.base import BaseCommand
from django.db.models import Q
from django.utils import timezone
from django_tasks_db.backend import DatabaseBackend
from django_tasks_db.compat import (
DEFAULT_TASK_BACKEND_ALIAS,
DEFAULT_TASK_QUEUE_NAME,
InvalidTaskBackend,
TaskResultStatus,
task_backends,
)
from django_tasks_db.models import DBTaskResult
logger = logging.getLogger("django_tasks_db.prune_db_task_results")
def valid_backend_name(val: str) -> DatabaseBackend:
try:
backend = task_backends[val]
except InvalidTaskBackend as e:
raise ArgumentTypeError(e.args[0]) from e
if not isinstance(backend, DatabaseBackend):
raise ArgumentTypeError(f"Backend '{val}' is not a database backend")
return backend
def valid_positive_int(val: str) -> int:
num = int(val)
if num < 0:
raise ArgumentTypeError("Must be greater than zero")
return num
class Command(BaseCommand):
help = "Prune finished database task results"
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument(
"--backend",
nargs="?",
default=DEFAULT_TASK_BACKEND_ALIAS,
type=valid_backend_name,
dest="backend",
help="The backend to operate on (default: %(default)r)",
)
parser.add_argument(
"--queue-name",
nargs="?",
default=DEFAULT_TASK_QUEUE_NAME,
type=str,
help="The queues to process. Separate multiple with a comma. To process all queues, use '*' (default: %(default)r)",
)
parser.add_argument(
"--min-age-days",
nargs="?",
default=14,
type=valid_positive_int,
help="The minimum age (in days) of a finished task result to be pruned (default: %(default)r)",
)
parser.add_argument(
"--failed-min-age-days",
nargs="?",
default=None,
type=valid_positive_int,
help="The minimum age (in days) of a failed task result to be pruned (default: min-age-days)",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't delete the task results, just show how many would be deleted",
)
def configure_logging(self, verbosity: int) -> None:
if verbosity == 0:
logger.setLevel(logging.WARNING)
elif verbosity == 1:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.DEBUG)
# If no handler is configured, the logs won't show,
# regardless of the set level.
if not logger.hasHandlers():
logger.addHandler(logging.StreamHandler(self.stdout))
def handle(
self,
*,
verbosity: int,
backend: DatabaseBackend,
min_age_days: int,
failed_min_age_days: int | None,
queue_name: str,
dry_run: bool,
**options: dict,
) -> None:
self.configure_logging(verbosity)
min_age = timezone.now() - timedelta(days=min_age_days)
failed_min_age = (
(timezone.now() - timedelta(days=failed_min_age_days))
if failed_min_age_days is not None
else None
)
results = DBTaskResult.objects.finished().filter(backend_name=backend.alias)
queue_names = queue_name.split(",")
if "*" not in queue_names:
results = results.filter(queue_name__in=queue_names)
if failed_min_age is None:
results = results.filter(finished_at__lte=min_age)
else:
results = results.filter(
Q(status=TaskResultStatus.SUCCESSFUL, finished_at__lte=min_age)
| Q(status=TaskResultStatus.FAILED, finished_at__lte=failed_min_age)
)
if dry_run:
logger.info("Would delete %d task result(s)", results.count())
else:
deleted, _ = results.delete()
logger.info("Deleted %d task result(s)", deleted)
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/ 0000775 0000000 0000000 00000000000 15244316143 0024634 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0001_initial.py 0000664 0000000 0000000 00000003600 15244316143 0027276 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-06-08 09:10
import uuid
from django.db import migrations, models
import django_tasks_db.models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="DBTaskResult",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
(
"status",
models.CharField(
choices=[
("NEW", "New"),
("RUNNING", "Running"),
("FAILED", "Failed"),
("COMPLETE", "Complete"),
],
default="NEW",
max_length=8,
),
),
("args_kwargs", models.JSONField()),
("priority", models.PositiveSmallIntegerField(default=0)),
("task_path", models.TextField()),
("queue_name", models.TextField(default="default")),
("backend_name", models.TextField()),
("run_after", models.DateTimeField(null=True)),
("result", models.JSONField(default=None, null=True)),
],
options={
"ordering": [
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(
models.F("run_after"), descending=True, nulls_last=True
),
],
},
bases=(django_tasks_db.models.GenericBase, models.Model),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0002_alter_dbtaskresult_options.py 0000664 0000000 0000000 00000001320 15244316143 0033314 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-06-08 15:24
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0001_initial"),
]
operations = [
migrations.AlterModelOptions(
name="dbtaskresult",
options={
"ordering": [
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(
models.F("run_after"), descending=True, nulls_last=True
),
],
"verbose_name": "Task Result",
"verbose_name_plural": "Task Results",
},
),
]
0003_dbtaskresult_enqueued_at_dbtaskresult_finished_at.py 0000664 0000000 0000000 00000001314 15244316143 0040002 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 5.0.6 on 2024-06-13 10:24
import django.utils.timezone
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0002_alter_dbtaskresult_options"),
]
operations = [
migrations.AddField(
model_name="dbtaskresult",
name="enqueued_at",
field=models.DateTimeField(
auto_now_add=True, default=django.utils.timezone.now
),
preserve_default=False,
),
migrations.AddField(
model_name="dbtaskresult",
name="finished_at",
field=models.DateTimeField(null=True),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0004_dbtaskresult_started_at.py 0000664 0000000 0000000 00000000731 15244316143 0032573 0 ustar 00root root 0000000 0000000 # Generated by Django 5.0.6 on 2024-07-04 09:31
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"django_tasks_database",
"0003_dbtaskresult_enqueued_at_dbtaskresult_finished_at",
),
]
operations = [
migrations.AddField(
model_name="dbtaskresult",
name="started_at",
field=models.DateTimeField(null=True),
),
]
0005_alter_dbtaskresult_priority_and_more.py 0000664 0000000 0000000 00000001233 15244316143 0035275 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 4.2.13 on 2024-07-10 15:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0004_dbtaskresult_started_at"),
]
operations = [
migrations.AlterField(
model_name="dbtaskresult",
name="priority",
field=models.IntegerField(default=0),
),
migrations.AddConstraint(
model_name="dbtaskresult",
constraint=models.CheckConstraint(
condition=models.Q(("priority__range", (-100, 100))),
name="priority_range",
),
),
]
0006_alter_dbtaskresult_args_kwargs_and_more.py 0000664 0000000 0000000 00000005057 15244316143 0035737 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 4.2.14 on 2024-08-02 13:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0005_alter_dbtaskresult_priority_and_more"),
]
operations = [
migrations.AlterField(
model_name="dbtaskresult",
name="args_kwargs",
field=models.JSONField(verbose_name="args kwargs"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="backend_name",
field=models.TextField(verbose_name="backend name"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="enqueued_at",
field=models.DateTimeField(auto_now_add=True, verbose_name="enqueued at"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="finished_at",
field=models.DateTimeField(null=True, verbose_name="finished at"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="priority",
field=models.IntegerField(default=0, verbose_name="priority"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="queue_name",
field=models.TextField(default="default", verbose_name="queue name"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="result",
field=models.JSONField(default=None, null=True, verbose_name="result"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="run_after",
field=models.DateTimeField(null=True, verbose_name="run after"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="started_at",
field=models.DateTimeField(null=True, verbose_name="started at"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="status",
field=models.CharField(
choices=[
("NEW", "New"),
("RUNNING", "Running"),
("FAILED", "Failed"),
("COMPLETE", "Complete"),
],
default="NEW",
max_length=8,
verbose_name="status",
),
),
migrations.AlterField(
model_name="dbtaskresult",
name="task_path",
field=models.TextField(verbose_name="task path"),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0007_add_separate_results_fields.py0000664 0000000 0000000 00000001340 15244316143 0033375 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-08-23 14:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0006_alter_dbtaskresult_args_kwargs_and_more"),
]
operations = [
migrations.AddField(
model_name="dbtaskresult",
name="exception_data",
field=models.JSONField(
default=None, null=True, verbose_name="exception data"
),
),
migrations.AddField(
model_name="dbtaskresult",
name="return_value",
field=models.JSONField(
default=None, null=True, verbose_name="return value"
),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0008_separate_results_field.py 0000664 0000000 0000000 00000003122 15244316143 0032403 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-08-23 14:38
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import StateApps
def separate_results_field(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
# If a task succeeded, the result is its return value
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="SUCCEEDED"
).update(return_value=models.F("result"))
# If a task failed, the result is the exception data (or nothing)
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="FAILED"
).update(exception_data=models.F("result"))
def merge_results_field(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
# If a task succeeded, the result is its return value
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="SUCCEEDED"
).update(result=models.F("return_value"))
# If a task failed, the result is the exception data (or nothing)
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="FAILED"
).update(result=models.F("exception_data"))
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0007_add_separate_results_fields"),
]
operations = [migrations.RunPython(separate_results_field, merge_results_field)]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0009_remove_results_field.py 0000664 0000000 0000000 00000000540 15244316143 0032076 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-08-23 14:51
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0008_separate_results_field"),
]
operations = [
migrations.RemoveField(
model_name="dbtaskresult",
name="result",
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0010_alter_dbtaskresult_status.py 0000664 0000000 0000000 00000001323 15244316143 0033146 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-11-21 13:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0009_remove_results_field"),
]
operations = [
migrations.AlterField(
model_name="dbtaskresult",
name="status",
field=models.CharField(
choices=[
("NEW", "New"),
("RUNNING", "Running"),
("FAILED", "Failed"),
("SUCCEEDED", "Succeeded"),
],
default="NEW",
max_length=9,
verbose_name="status",
),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0011_rename_complete_status.py 0000664 0000000 0000000 00000002014 15244316143 0032406 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-08-23 14:38
from django.db import migrations
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import StateApps
def separate_results_field(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="COMPLETE"
).update(status="SUCCEEDED")
def merge_results_field(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="SUCCEEDED"
).update(status="COMPLETE")
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0010_alter_dbtaskresult_status"),
]
operations = [migrations.RunPython(separate_results_field, merge_results_field)]
0012_add_separate_exception_fields.py 0000664 0000000 0000000 00000001306 15244316143 0033611 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 5.1.1 on 2024-11-22 16:32
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0011_rename_complete_status"),
]
operations = [
migrations.AddField(
model_name="dbtaskresult",
name="exception_class_path",
field=models.TextField(default="", verbose_name="exception class path"),
preserve_default=False,
),
migrations.AddField(
model_name="dbtaskresult",
name="traceback",
field=models.TextField(default="", verbose_name="traceback"),
preserve_default=False,
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0013_separate_exception_fields.py 0000664 0000000 0000000 00000001754 15244316143 0033070 0 ustar 00root root 0000000 0000000 # Generated by Django 4.2.13 on 2024-08-23 14:38
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import StateApps
from django.db.models.functions import Coalesce
def separate_exception_fields(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.using(schema_editor.connection.alias).update(
exception_class_path=Coalesce(
models.F("exception_data__exc_type"), models.Value("", models.JSONField())
),
traceback=Coalesce(
models.F("exception_data__exc_traceback"),
models.Value("", models.JSONField()),
),
)
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0012_add_separate_exception_fields"),
]
operations = [migrations.RunPython(separate_exception_fields)]
0014_remove_dbtaskresult_exception_data.py 0000664 0000000 0000000 00000000552 15244316143 0034730 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 5.1.1 on 2024-11-22 16:32
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0013_separate_exception_fields"),
]
operations = [
migrations.RemoveField(
model_name="dbtaskresult",
name="exception_data",
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0015_correctly_order_run_after.py 0000664 0000000 0000000 00000001253 15244316143 0033122 0 ustar 00root root 0000000 0000000 # Generated by Django 5.1.5 on 2025-04-27 12:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0014_remove_dbtaskresult_exception_data"),
]
operations = [
migrations.AlterModelOptions(
name="dbtaskresult",
options={
"ordering": [
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after"), nulls_last=True),
],
"verbose_name": "Task Result",
"verbose_name_plural": "Task Results",
},
),
]
0016_remove_dbtaskresult_django_task_new_ordering_idx_and_more.py 0000664 0000000 0000000 00000007300 15244316143 0041477 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 5.2.2 on 2025-06-13 14:56
import datetime
from django.conf import settings
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import StateApps
def update_status(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.filter(status="NEW").update(status="READY")
def revert_status_rename(
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.filter(status="READY").update(status="NEW")
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0015_correctly_order_run_after"),
]
operations = [
migrations.AlterModelOptions(
name="dbtaskresult",
options={
"ordering": [
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after")),
],
"verbose_name": "Task Result",
"verbose_name_plural": "Task Results",
},
),
migrations.AlterField(
model_name="dbtaskresult",
name="backend_name",
field=models.CharField(max_length=32, verbose_name="backend name"),
),
migrations.AlterField(
model_name="dbtaskresult",
name="queue_name",
field=models.CharField(
default="default", max_length=32, verbose_name="queue name"
),
),
migrations.AlterField(
model_name="dbtaskresult",
name="run_after",
field=models.DateTimeField(
default=datetime.datetime(
9999,
1,
1,
0,
0,
tzinfo=datetime.timezone.utc if settings.USE_TZ else None,
),
verbose_name="run after",
),
preserve_default=False,
),
migrations.AlterField(
model_name="dbtaskresult",
name="status",
field=models.CharField(
choices=[
("READY", "Ready"),
("RUNNING", "Running"),
("FAILED", "Failed"),
("SUCCEEDED", "Succeeded"),
],
default="READY",
max_length=9,
verbose_name="status",
),
),
migrations.AddField(
model_name="dbtaskresult",
name="worker_ids",
field=models.JSONField(default=list, verbose_name="worker id"),
),
migrations.RunPython(update_status, revert_status_rename),
migrations.AddIndex(
model_name="dbtaskresult",
index=models.Index(
models.F("status"),
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after")),
condition=models.Q(("status", "READY")),
name="django_task_new_ordering_idx",
),
),
migrations.AddIndex(
model_name="dbtaskresult",
index=models.Index(
fields=["queue_name"], name="django_task_queue_n_99c321_idx"
),
),
migrations.AddIndex(
model_name="dbtaskresult",
index=models.Index(
fields=["backend_name"], name="django_task_backend_071754_idx"
),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0017_dbtaskresult_metadata.py 0000664 0000000 0000000 00000000770 15244316143 0032230 0 ustar 00root root 0000000 0000000 # Generated by Django 6.0rc1 on 2025-12-17 08:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"django_tasks_database",
"0016_remove_dbtaskresult_django_task_new_ordering_idx_and_more",
),
]
operations = [
migrations.AddField(
model_name="dbtaskresult",
name="metadata",
field=models.JSONField(default=dict, verbose_name="metadata"),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/0018_rename_succeeded_status.py 0000664 0000000 0000000 00000002677 15244316143 0032550 0 ustar 00root root 0000000 0000000 # Generated by Django 6.0 on 2026-01-09
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import StateApps
def forwards(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="SUCCEEDED"
).update(status="SUCCESSFUL")
def backwards(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
DBTaskResult = apps.get_model("django_tasks_database", "DBTaskResult")
DBTaskResult.objects.using(schema_editor.connection.alias).filter(
status="SUCCESSFUL"
).update(status="SUCCEEDED")
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0017_dbtaskresult_metadata"),
]
operations = [
migrations.AlterField(
model_name="dbtaskresult",
name="status",
field=models.CharField(
choices=[
("READY", "Ready"),
("RUNNING", "Running"),
("FAILED", "Failed"),
("SUCCESSFUL", "Successful"),
],
default="READY",
max_length=10,
verbose_name="status",
),
),
migrations.RunPython(forwards, backwards),
]
0019_rename_django_task_new_ordering_idx_tasks_db_new_ordering_idx_and_more.py 0000664 0000000 0000000 00000003735 15244316143 0044155 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 6.0.2 on 2026-02-06 16:30
from django.db import migrations
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.migrations.state import ProjectState
class RenameIndexIfExists(migrations.RenameIndex):
"""RenameIndex that silently skips if the source index does not exist."""
def database_forwards(
self,
app_label: str,
schema_editor: BaseDatabaseSchemaEditor,
from_state: ProjectState,
to_state: ProjectState,
) -> None:
from_model = from_state.apps.get_model(app_label, self.model_name)
with schema_editor.connection.cursor() as cursor:
existing = schema_editor.connection.introspection.get_constraints(
cursor, from_model._meta.db_table
)
if self.old_name not in existing:
return
super().database_forwards(app_label, schema_editor, from_state, to_state)
def database_backwards(
self,
app_label: str,
schema_editor: BaseDatabaseSchemaEditor,
from_state: ProjectState,
to_state: ProjectState,
) -> None:
to_model = to_state.apps.get_model(app_label, self.model_name)
with schema_editor.connection.cursor() as cursor:
existing = schema_editor.connection.introspection.get_constraints(
cursor, to_model._meta.db_table
)
if self.new_name not in existing:
return
super().database_backwards(app_label, schema_editor, from_state, to_state)
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0018_rename_succeeded_status"),
]
operations = [
RenameIndexIfExists(
model_name="dbtaskresult",
new_name="tasks_db_new_ordering_idx",
old_name="django_task_new_ordering_idx",
),
migrations.RemoveField(
model_name="dbtaskresult",
name="metadata",
),
]
0020_update_db_task_result_ordering.py 0000664 0000000 0000000 00000002542 15244316143 0034033 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 6.0.4 on 2026-04-20 14:30
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"django_tasks_database",
"0019_rename_django_task_new_ordering_idx_tasks_db_new_ordering_idx_and_more",
),
]
operations = [
migrations.AlterModelOptions(
name="dbtaskresult",
options={
"ordering": [
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after")),
models.OrderBy(models.F("enqueued_at")),
],
"verbose_name": "Task Result",
"verbose_name_plural": "Task Results",
},
),
migrations.RemoveIndex(
model_name="dbtaskresult",
name="tasks_db_new_ordering_idx",
),
migrations.AddIndex(
model_name="dbtaskresult",
index=models.Index(
models.F("status"),
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after")),
models.OrderBy(models.F("enqueued_at")),
condition=models.Q(("status", "READY")),
name="tasks_db_new_ordering_idx",
),
),
]
0021_conditional_partial_index_ordering.py 0000664 0000000 0000000 00000001607 15244316143 0034674 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations # Generated by Django 6.0.5 on 2026-08-21 13:40
from django.db import migrations, models
import django_tasks_db.models
class Migration(migrations.Migration):
dependencies = [
("django_tasks_database", "0020_update_db_task_result_ordering"),
]
operations = [
migrations.RemoveIndex(
model_name="dbtaskresult",
name="tasks_db_new_ordering_idx",
),
migrations.AddIndex(
model_name="dbtaskresult",
index=django_tasks_db.models.ConditionalPartialIndex(
models.F("status"),
models.OrderBy(models.F("priority"), descending=True),
models.OrderBy(models.F("run_after")),
models.OrderBy(models.F("enqueued_at")),
condition=models.Q(("status", "READY")),
name="tasks_db_new_ordering_idx",
),
),
]
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/migrations/__init__.py 0000664 0000000 0000000 00000000000 15244316143 0026733 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/models.py 0000664 0000000 0000000 00000022735 15244316143 0024326 0 ustar 00root root 0000000 0000000 import datetime
import logging
import uuid
from collections.abc import Sequence
from traceback import format_exception
from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar
from django.conf import settings
from django.core.exceptions import SuspiciousOperation
from django.db import models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.backends.ddl_references import Statement # add this
from django.db.models import F, Q
from django.db.models.constraints import CheckConstraint
from django.utils import timezone
from django.utils.module_loading import import_string
from django.utils.translation import gettext_lazy as _
from typing_extensions import ParamSpec
from .compat import (
DEFAULT_TASK_PRIORITY,
DEFAULT_TASK_QUEUE_NAME,
TASK_MAX_PRIORITY,
TASK_MIN_PRIORITY,
Task,
TaskError,
TaskResultStatus,
)
from .utils import normalize_uuid, retry
logger = logging.getLogger("django_tasks_db")
T = TypeVar("T")
P = ParamSpec("P")
if TYPE_CHECKING:
from .backend import TaskResult
class GenericBase(Generic[P, T]):
pass
else:
class GenericBase:
"""
https://code.djangoproject.com/ticket/33174
"""
def __class_getitem__(cls, _):
return cls
def get_date_max() -> datetime.datetime:
return datetime.datetime(
9999, 1, 1, tzinfo=datetime.timezone.utc if settings.USE_TZ else None
)
class ConditionalPartialIndex(models.Index):
"""
An index that applies its condition only on backends that support partial/filtered indexes.
On unsupported backends, the index is created without the condition,
rather than being skipped altogether.
"""
def __init__(
self,
*expressions: Any,
condition: Q | None = None,
**kwargs: Any,
) -> None:
# Capture condition before passing to constructor so it's unset and the index can always be applied.
# See add_index.
self._partial_condition = condition
super().__init__(*expressions, **kwargs)
@property
def contains_expressions(self) -> bool:
# Pretend there are no expressions.
# Some DB backends claim to not support expressions, but do support them if they're simple enough.
return False
def create_sql(
self,
model: type[models.Model],
schema_editor: BaseDatabaseSchemaEditor,
using: str = "",
**kwargs: Any,
) -> Statement:
if (
self._partial_condition is not None
and schema_editor.connection.features.supports_partial_indexes
):
# Add condition back just before executing SQL so the condition is included.
self.condition = self._partial_condition
try:
return super().create_sql(model, schema_editor, using=using, **kwargs)
finally:
self.condition = None
return super().create_sql(model, schema_editor, using=using, **kwargs)
def deconstruct(self) -> tuple[str, Sequence[Any], dict[str, Any]]:
path, args, kwargs = super().deconstruct()
if self._partial_condition is not None:
kwargs["condition"] = self._partial_condition
return path, args, kwargs
class DBTaskResultQuerySet(models.QuerySet["DBTaskResult"]):
def ready(self) -> "DBTaskResultQuerySet":
"""
Return tasks which are ready to be processed.
"""
return self.filter(
status=TaskResultStatus.READY,
).filter(
models.Q(run_after=get_date_max()) | models.Q(run_after__lte=timezone.now())
)
def successful(self) -> "DBTaskResultQuerySet":
return self.filter(status=TaskResultStatus.SUCCESSFUL)
def failed(self) -> "DBTaskResultQuerySet":
return self.filter(status=TaskResultStatus.FAILED)
def running(self) -> "DBTaskResultQuerySet":
return self.filter(status=TaskResultStatus.RUNNING)
def finished(self) -> "DBTaskResultQuerySet":
return self.failed() | self.successful()
def get_locked(self) -> Optional["DBTaskResult"]:
"""
Get a job, locking the row and accounting for deadlocks.
"""
return self.select_for_update(skip_locked=True).first()
class DBTaskResult(GenericBase[P, T], models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
status = models.CharField(
_("status"),
choices=TaskResultStatus.choices,
default=TaskResultStatus.READY,
max_length=max(len(value) for value in TaskResultStatus.values),
)
enqueued_at = models.DateTimeField(_("enqueued at"), auto_now_add=True)
started_at = models.DateTimeField(_("started at"), null=True)
finished_at = models.DateTimeField(_("finished at"), null=True)
args_kwargs = models.JSONField(_("args kwargs"))
priority = models.IntegerField(_("priority"), default=DEFAULT_TASK_PRIORITY)
task_path = models.TextField(_("task path"))
worker_ids = models.JSONField(_("worker id"), default=list)
queue_name = models.CharField(
_("queue name"), default=DEFAULT_TASK_QUEUE_NAME, max_length=32
)
backend_name = models.CharField(_("backend name"), max_length=32)
run_after = models.DateTimeField(_("run after"))
return_value = models.JSONField(_("return value"), default=None, null=True)
exception_class_path = models.TextField(_("exception class path"))
traceback = models.TextField(_("traceback"))
objects = DBTaskResultQuerySet.as_manager()
class Meta:
ordering = [F("priority").desc(), F("run_after").asc(), F("enqueued_at").asc()]
verbose_name = _("Task Result")
verbose_name_plural = _("Task Results")
indexes = [
ConditionalPartialIndex(
"status",
*ordering,
name="tasks_db_new_ordering_idx",
condition=Q(status=TaskResultStatus.READY),
),
models.Index(fields=["queue_name"]),
models.Index(fields=["backend_name"]),
]
constraints = [
CheckConstraint(
condition=Q(priority__range=(TASK_MIN_PRIORITY, TASK_MAX_PRIORITY)),
name="priority_range",
)
]
@property
def task(self) -> Task[P, T]:
task = import_string(self.task_path)
if not isinstance(task, Task):
raise SuspiciousOperation(
f"Task {self.id} does not point to a Task ({self.task_path})"
)
return task.using(
priority=self.priority,
queue_name=self.queue_name,
run_after=None if self.run_after == get_date_max() else self.run_after,
backend=self.backend_name,
)
@property
def task_result(self) -> "TaskResult[P, T]":
from .backend import TaskResult
task_result = TaskResult(
db_result=self,
task=self.task,
id=normalize_uuid(self.id),
status=TaskResultStatus[self.status],
enqueued_at=self.enqueued_at,
started_at=self.started_at,
last_attempted_at=self.started_at,
finished_at=self.finished_at,
args=self.args_kwargs["args"],
kwargs=self.args_kwargs["kwargs"],
backend=self.backend_name,
errors=[],
worker_ids=self.worker_ids,
)
if self.status == TaskResultStatus.FAILED:
task_result.errors.append(
TaskError(
exception_class_path=self.exception_class_path,
traceback=self.traceback,
)
)
object.__setattr__(task_result, "_return_value", self.return_value)
return task_result
@property
def task_name(self) -> str:
# If the function for an existing task is no longer available, it'll either raise an
# ImportError or ModuleNotFoundError (a subclass of ImportError).
try:
return self.task.name
except ImportError:
pass
try:
return self.task_path.rsplit(".", 1)[1]
except IndexError:
return self.task_path
def claim(self, worker_id: str) -> None:
"""
Mark as job as being run
"""
self.status = TaskResultStatus.RUNNING
self.started_at = timezone.now()
self.worker_ids = [*self.worker_ids, worker_id]
self.save(update_fields=["status", "started_at", "worker_ids"])
@retry()
def set_successful(self, return_value: Any) -> None:
self.status = TaskResultStatus.SUCCESSFUL
self.finished_at = timezone.now()
self.return_value = return_value
self.exception_class_path = ""
self.traceback = ""
self.save(
update_fields=[
"status",
"return_value",
"finished_at",
"exception_class_path",
"traceback",
]
)
@retry()
def set_failed(self, exc: BaseException) -> None:
self.status = TaskResultStatus.FAILED
self.finished_at = timezone.now()
self.exception_class_path = f"{type(exc).__module__}.{type(exc).__qualname__}"
self.traceback = "".join(format_exception(exc))
self.return_value = None
self.save(
update_fields=[
"status",
"return_value",
"finished_at",
"exception_class_path",
"traceback",
]
)
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/py.typed 0000664 0000000 0000000 00000000000 15244316143 0024145 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/django_tasks_db/signal_handlers.py 0000664 0000000 0000000 00000000544 15244316143 0026172 0 ustar 00root root 0000000 0000000 from typing import Any
from django.db.models.signals import pre_save
from django.dispatch import receiver
from .models import DBTaskResult, get_date_max
@receiver(pre_save, sender=DBTaskResult)
def set_run_after(sender: Any, instance: DBTaskResult, **kwargs: Any) -> None:
if instance.run_after is None:
instance.run_after = get_date_max()
RealOrangeOne-django-tasks-db-5970262/django_tasks_db/utils.py 0000664 0000000 0000000 00000005340 15244316143 0024174 0 ustar 00root root 0000000 0000000 import time
from collections.abc import Callable, Generator
from contextlib import contextmanager
from functools import wraps
from typing import Any, TypeVar
from uuid import UUID
from django.db import OperationalError, transaction
from django.db.backends.base.base import BaseDatabaseWrapper
from typing_extensions import ParamSpec
T = TypeVar("T")
P = ParamSpec("P")
@contextmanager
def exclusive_transaction(using: str | None = None) -> Generator[Any, Any, Any]:
"""
Wrapper around `transaction.atomic` which ensures transactions on SQLite are exclusive.
"""
connection: BaseDatabaseWrapper = transaction.get_connection(using)
if connection.vendor == "sqlite":
if not hasattr(connection, "transaction_mode"):
# Manually called to set `transaction_mode`
connection.get_connection_params()
old_transaction_mode = connection.transaction_mode # type: ignore[attr-defined]
try:
connection.transaction_mode = "EXCLUSIVE" # type: ignore[attr-defined]
with transaction.atomic(using=using):
yield
finally:
connection.transaction_mode = old_transaction_mode # type: ignore[attr-defined]
else:
with transaction.atomic(using=using):
yield
def normalize_uuid(val: str | UUID) -> str:
"""
Normalize a UUID into its dashed representation.
This works around engines like MySQL which don't store values in a uuid field,
and thus drops the dashes.
"""
if isinstance(val, str):
val = UUID(val)
return str(val)
def retry(*, retries: int = 3, backoff_delay: float = 0.1) -> Callable:
"""
Retry the given code `retries` times, raising the final error.
`backoff_delay` can be used to add a delay between attempts.
"""
def wrapper(f: Callable[P, T]) -> Callable[P, T]:
@wraps(f)
def inner_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: # type:ignore[return]
for attempt in range(1, retries + 1):
try:
return f(*args, **kwargs)
except KeyboardInterrupt:
# Let the user ctrl-C out of the program without a retry
raise
except BaseException:
if attempt == retries:
raise
time.sleep(backoff_delay)
return inner_wrapper
return wrapper
def is_locked_database_exception(e: OperationalError) -> bool:
if isinstance(e.args[0], str) and "is locked" in e.args[0].lower():
return True
# MySQL has an error code in the first argument, and message in the second
elif isinstance(e.args[0], int) and e.args[0] == 1205:
return True
return False
RealOrangeOne-django-tasks-db-5970262/docker-compose.yml 0000664 0000000 0000000 00000000700 15244316143 0022776 0 ustar 00root root 0000000 0000000 services:
postgresql:
image: postgres:alpine
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 15432:5432
mysql:
image: mysql:lts
environment:
MYSQL_ROOT_PASSWORD: django
MYSQL_DATABASE: django
ports:
- 13306:3306
mariadb:
image: mariadb:lts
environment:
MYSQL_ROOT_PASSWORD: django
MYSQL_DATABASE: django
ports:
- 13307:3306
RealOrangeOne-django-tasks-db-5970262/justfile 0000664 0000000 0000000 00000002360 15244316143 0021115 0 ustar 00root root 0000000 0000000 # Recipes
@default:
just --list
test *ARGS:
python -m manage check
python -m manage makemigrations --dry-run --check --noinput
python -m coverage run --source=django_tasks_db -m manage test --shuffle --noinput {{ ARGS }}
python -m coverage report
python -m coverage html
test-fast *ARGS:
python -m manage test --shuffle --noinput --settings tests.settings_fast {{ ARGS }}
format:
python -m ruff check django_tasks_db tests --fix
python -m ruff format django_tasks_db tests
lint:
python -m ruff check django_tasks_db tests
python -m ruff format django_tasks_db tests --check
python -m mypy django_tasks_db tests
start-dbs:
docker-compose up -d
test-sqlite *ARGS:
python -m manage test --shuffle --noinput {{ ARGS }}
test-postgres *ARGS:
DATABASE_URL=postgres://postgres:postgres@localhost:15432/postgres python -m manage test --shuffle --noinput {{ ARGS }}
test-mysql *ARGS:
DATABASE_URL=mysql://root:django@127.0.0.1:13306/django python -m manage test --shuffle --noinput {{ ARGS }}
test-mariadb *ARGS:
DATABASE_URL=mysql://root:django@127.0.0.1:13307/django python -m manage test --shuffle --noinput {{ ARGS }}
test-dbs *ARGS: start-dbs test-postgres test-mysql test-mariadb test-sqlite
RealOrangeOne-django-tasks-db-5970262/manage.py 0000775 0000000 0000000 00000000370 15244316143 0021151 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
RealOrangeOne-django-tasks-db-5970262/pyproject.toml 0000664 0000000 0000000 00000005430 15244316143 0022262 0 ustar 00root root 0000000 0000000 [build-system]
build-backend = "setuptools.build_meta"
requires = [
"setuptools",
]
[project]
name = "django-tasks-db"
description = "An ORM-based backend for Django Tasks"
authors = [
{name = "Jake Howard"},
]
version = "0.13.0"
readme = "README.md"
license = "BSD-3-Clause"
license-files = ["LICENSE"]
classifiers = [
"Development Status :: 4 - Beta",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Programming Language :: Python :: Implementation :: CPython",
"Environment :: Web Environment",
"Framework :: Django",
"Framework :: Django",
"Framework :: Django :: 5.2",
"Framework :: Django :: 6.0",
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Natural Language :: English",
"Topic :: Internet :: WWW/HTTP",
"Typing :: Typed"
]
requires-python = ">=3.10"
dependencies = [
"django>=5.2",
"typing_extensions",
"django-stubs-ext",
]
[project.optional-dependencies]
compat = [
"django>=5.2,<6.0",
"django-tasks>=0.12.0",
]
[project.urls]
Source = "https://github.com/RealOrangeOne/django-tasks-db"
Issues = "https://github.com/RealOrangeOne/django-tasks-db/issues"
Changelog = "https://github.com/RealOrangeOne/django-tasks-db/releases"
[dependency-groups]
dev = [
"ruff==0.16.4",
"coverage==7.15.4",
"django-stubs[compatible-mypy]",
"mypy==2.3.1",
"dj-database-url",
]
mysql = [
"mysqlclient"
]
postgres = [
"psycopg[binary]",
]
[tool.ruff.lint]
select = ["E", "F", "I", "W", "N", "B", "A", "C4", "T20", "DJ", "S", "UP"]
ignore = ["E501", "DJ008"]
[tool.ruff.lint.per-file-ignores]
"tests/db_worker_test_settings.py" = ["F403", "F405"]
"tests/settings_fast.py" = ["F403", "F405"]
"tests/**.py" = ["S101", "S603", "S105"]
[tool.mypy]
plugins = ["mypy_django_plugin.main"]
warn_unused_ignores = true
warn_return_any = true
show_error_codes = true
strict_optional = true
implicit_optional = true
disallow_subclassing_any = true
disallow_untyped_calls = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
disallow_untyped_decorators = true
check_untyped_defs = true
ignore_missing_imports = true
[tool.django-stubs]
django_settings_module = "tests.settings"
[tool.coverage.run]
branch = true
[tool.coverage.report]
show_missing = true
ignore_errors = true
exclude_also = [
# Don't complain if tests don't hit defensive assertion code:
"raise AssertionError",
"raise NotImplementedError",
# Don't complain about abstract methods, they aren't run:
"@(abc.)?abstractmethod",
# Nor complain about type checking
"if TYPE_CHECKING:",
]
RealOrangeOne-django-tasks-db-5970262/tests/ 0000775 0000000 0000000 00000000000 15244316143 0020506 5 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/tests/__init__.py 0000664 0000000 0000000 00000000000 15244316143 0022605 0 ustar 00root root 0000000 0000000 RealOrangeOne-django-tasks-db-5970262/tests/db_worker_test_settings.py 0000664 0000000 0000000 00000000504 15244316143 0026014 0 ustar 00root root 0000000 0000000 from .settings import *
TASKS = {"default": {"BACKEND": "django_tasks_db.DatabaseBackend"}}
# Force the test DB to be used
if "sqlite" in DATABASES["default"]["ENGINE"]:
DATABASES["default"]["NAME"] = DATABASES["default"]["TEST"]["NAME"]
else:
DATABASES["default"]["NAME"] = "test_" + DATABASES["default"]["NAME"]
RealOrangeOne-django-tasks-db-5970262/tests/settings.py 0000664 0000000 0000000 00000003775 15244316143 0022734 0 ustar 00root root 0000000 0000000 import os
import sys
import dj_database_url
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
IN_TEST = "IN_TEST" in os.environ or (len(sys.argv) > 1 and sys.argv[1] == "test")
ALLOWED_HOSTS = ["*"]
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django_tasks_db",
"tests",
]
MIDDLEWARE = [
"django.contrib.sessions.middleware.SessionMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
]
SECRET_KEY = "abcde12345"
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"handlers": {
"console": {
"class": "logging.StreamHandler",
},
},
"loggers": {
"django.tasks": {
"handlers": ["console"],
"level": "INFO",
"propagate": False,
},
},
}
DATABASES = {
"default": dj_database_url.config(
default="sqlite:///" + os.path.join(BASE_DIR, "db.sqlite3")
)
}
if "sqlite" in DATABASES["default"]["ENGINE"]:
DATABASES["default"]["OPTIONS"] = {"timeout": 5}
DATABASES["default"]["TEST"] = {
"NAME": os.path.join(BASE_DIR, "db-test.sqlite3"),
"OPTIONS": {
"timeout": 5,
},
}
USE_TZ = True
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
TASKS = {
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["default", "queue-1"],
}
}
if not IN_TEST:
DEBUG = True
RealOrangeOne-django-tasks-db-5970262/tests/settings_fast.py 0000664 0000000 0000000 00000000236 15244316143 0023736 0 ustar 00root root 0000000 0000000 from .settings import *
# Unset custom test settings to use in-memory DB
if "sqlite" in DATABASES["default"]["ENGINE"]:
del DATABASES["default"]["TEST"]
RealOrangeOne-django-tasks-db-5970262/tests/tasks.py 0000664 0000000 0000000 00000003163 15244316143 0022210 0 ustar 00root root 0000000 0000000 import time
from typing import Any
try:
from django.tasks import TaskContext, task
except (ImportError, ModuleNotFoundError):
from django_tasks import ( # type: ignore[assignment,no-redef,unused-ignore]
TaskContext,
task,
)
@task()
def noop_task(*args: Any, **kwargs: Any) -> None:
return None
@task
def noop_task_from_bare_decorator(*args: Any, **kwargs: Any) -> None:
return None
@task()
async def noop_task_async(*args: Any, **kwargs: Any) -> None:
return None
@task()
def calculate_meaning_of_life() -> int:
return 42
@task()
def failing_task_value_error() -> None:
raise ValueError("This task failed due to ValueError")
@task()
def failing_task_system_exit() -> None:
raise SystemExit("This task failed due to SystemExit")
@task()
def failing_task_keyboard_interrupt() -> None:
raise KeyboardInterrupt("This task failed due to KeyboardInterrupt")
@task()
def complex_exception() -> None:
raise ValueError(ValueError("This task failed"))
@task()
def complex_return_value() -> Any:
# Return something which isn't JSON serializable nor picklable
return lambda: True
@task()
def exit_task() -> None:
exit(1)
@task()
def hang() -> None:
"""
Do nothing for 5 minutes
"""
time.sleep(300)
@task()
def sleep_for(seconds: float) -> None:
time.sleep(seconds)
@task(takes_context=True)
def get_task_id(context: TaskContext) -> str:
return context.task_result.id
@task(takes_context=True)
def test_context(context: TaskContext, attempt: int) -> None:
assert isinstance(context, TaskContext)
assert context.attempt == attempt
RealOrangeOne-django-tasks-db-5970262/tests/tests.py 0000664 0000000 0000000 00000203625 15244316143 0022232 0 ustar 00root root 0000000 0000000 import contextlib
import copy
import logging
import os
import signal
import subprocess
import sys
import time
import uuid
import warnings
from collections import Counter
from collections.abc import Iterator, Sequence
from contextlib import redirect_stderr
from datetime import datetime, timedelta
from functools import partial
from io import StringIO
from typing import Any, cast
from unittest import mock, skipIf
from django import VERSION
from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.core.management import CommandError, call_command, execute_from_command_line
from django.db import connection, connections, transaction
from django.db.models import QuerySet
from django.db.utils import IntegrityError, OperationalError
from django.test import SimpleTestCase, TestCase, TransactionTestCase, override_settings
from django.test.testcases import _deferredSkip # type:ignore[attr-defined]
from django.utils import timezone
from django.utils.crypto import get_random_string
try:
from django.tasks import TaskResultStatus, default_task_backend, task_backends
from django.tasks.base import Task
from django.tasks.exceptions import InvalidTask, TaskResultDoesNotExist
from django.tasks.signals import task_enqueued
DUMMY_BACKEND = "django.tasks.backends.dummy.DummyBackend"
LOGGER = "django.tasks"
except (ImportError, ModuleNotFoundError):
from django_tasks import ( # type: ignore[assignment,no-redef,unused-ignore]
TaskResultStatus,
default_task_backend,
task_backends,
)
from django_tasks.base import ( # type: ignore[assignment,no-redef,unused-ignore]
Task,
)
from django_tasks.exceptions import ( # type: ignore[assignment,no-redef,unused-ignore]
InvalidTaskError as InvalidTask,
)
from django_tasks.exceptions import ( # type: ignore[assignment,no-redef,unused-ignore]
TaskResultDoesNotExist,
)
from django_tasks.signals import ( # type: ignore[assignment,no-redef,unused-ignore]
task_enqueued,
)
DUMMY_BACKEND = "django_tasks.backends.dummy.DummyBackend"
LOGGER = "django_tasks"
from django_tasks_db import DatabaseBackend, compat
from django_tasks_db.admin import DBTaskResultAdmin
from django_tasks_db.management.commands.prune_db_task_results import (
logger as prune_db_tasks_logger,
)
from django_tasks_db.models import DBTaskResult, get_date_max
from django_tasks_db.utils import (
exclusive_transaction,
normalize_uuid,
)
from tests import tasks as test_tasks
def skipIfInMemoryDB() -> Any: # noqa:N802
return _deferredSkip(
lambda: connection.vendor == "sqlite" and connection.is_in_memory_db(), # type:ignore[attr-defined]
"Tests cannot run on in-memory DB",
"skipIfInMemoryDB",
)
class DatabaseBackendTestCase(TransactionTestCase):
@contextlib.contextmanager
def _capture_task_enqueued_signal(
self,
) -> Iterator[list[tuple[type[DatabaseBackend], Any]]]:
received: list[tuple[type[DatabaseBackend], Any]] = []
def capture_signal(
sender: type[DatabaseBackend], task_result: Any, **kwargs: Any
) -> None:
received.append((sender, task_result))
task_enqueued.connect(
capture_signal, dispatch_uid="db_enqueue_signal", weak=False
)
try:
yield received
finally:
task_enqueued.disconnect(capture_signal, dispatch_uid="db_enqueue_signal")
def get_task_count_in_new_connection(self) -> int:
"""
See what other connections see
"""
new_connection = connections.create_connection("default")
try:
with new_connection.cursor() as c:
c.execute(str(DBTaskResult.objects.values("id").query))
return len(c.fetchall())
finally:
new_connection.close()
def test_using_correct_backend(self) -> None:
self.assertEqual(default_task_backend, task_backends["default"])
self.assertIsInstance(task_backends["default"], DatabaseBackend)
self.assertEqual(default_task_backend.alias, "default")
self.assertEqual(default_task_backend.options, {})
def test_enqueue_task(self) -> None:
for task in [test_tasks.noop_task, test_tasks.noop_task_async]:
with self.subTest(task), self.assertNumQueries(1):
with self._capture_task_enqueued_signal() as received:
result = cast(Task, task).enqueue(1, two=3)
self.assertEqual(uuid.UUID(result.id).version, 4)
self.assertEqual(result.status, TaskResultStatus.READY)
self.assertFalse(result.is_finished)
self.assertIsNone(result.started_at)
self.assertIsNone(result.last_attempted_at)
self.assertIsNone(result.finished_at)
with self.assertRaisesMessage(ValueError, "Task has not finished yet"):
result.return_value # noqa:B018
self.assertEqual(result.task, task)
self.assertEqual(result.args, [1])
self.assertEqual(result.kwargs, {"two": 3})
self.assertEqual(result.attempts, 0)
self.assertEqual(received, [(DatabaseBackend, result)])
async def test_enqueue_task_async(self) -> None:
for task in [test_tasks.noop_task, test_tasks.noop_task_async]:
with self.subTest(task):
with self._capture_task_enqueued_signal() as received:
result = await cast(Task, task).aenqueue()
self.assertEqual(uuid.UUID(result.id).version, 4)
self.assertEqual(result.status, TaskResultStatus.READY)
self.assertFalse(result.is_finished)
self.assertIsNone(result.started_at)
self.assertIsNone(result.last_attempted_at)
self.assertIsNone(result.finished_at)
with self.assertRaisesMessage(ValueError, "Task has not finished yet"):
result.return_value # noqa:B018
self.assertEqual(result.task, task)
self.assertEqual(result.args, [])
self.assertEqual(result.kwargs, {})
self.assertEqual(result.attempts, 0)
self.assertEqual(received, [(DatabaseBackend, result)])
def test_get_result(self) -> None:
with self.assertNumQueries(1):
result = default_task_backend.enqueue(test_tasks.noop_task, [], {})
with self.assertNumQueries(1):
new_result = default_task_backend.get_result(result.id)
self.assertEqual(result, new_result)
async def test_get_result_async(self) -> None:
result = await default_task_backend.aenqueue(test_tasks.noop_task, [], {})
new_result = await default_task_backend.aget_result(result.id)
self.assertEqual(result, new_result)
def test_refresh_result(self) -> None:
result = default_task_backend.enqueue(
test_tasks.calculate_meaning_of_life, [], {}
)
DBTaskResult.objects.all().update(
status=TaskResultStatus.SUCCESSFUL,
started_at=timezone.now(),
finished_at=timezone.now(),
return_value=42,
worker_ids=[get_random_string(32)],
)
self.assertEqual(result.status, TaskResultStatus.READY)
self.assertFalse(result.is_finished)
self.assertIsNone(result.started_at)
self.assertIsNone(result.last_attempted_at)
self.assertIsNone(result.finished_at)
self.assertEqual(result.attempts, 0)
with self.assertNumQueries(1):
result.refresh()
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
self.assertTrue(result.is_finished)
self.assertEqual(result.return_value, 42)
self.assertEqual(result.attempts, 1)
async def test_refresh_result_async(self) -> None:
result = await default_task_backend.aenqueue(
test_tasks.calculate_meaning_of_life, [], {}
)
await DBTaskResult.objects.all().aupdate(
status=TaskResultStatus.SUCCESSFUL,
started_at=timezone.now(),
finished_at=timezone.now(),
return_value=42,
worker_ids=[get_random_string(32)],
)
self.assertEqual(result.status, TaskResultStatus.READY)
self.assertFalse(result.is_finished)
self.assertIsNone(result.started_at)
self.assertIsNone(result.last_attempted_at)
self.assertIsNone(result.finished_at)
self.assertEqual(result.attempts, 0)
await result.arefresh()
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
self.assertTrue(result.is_finished)
self.assertEqual(result.return_value, 42)
self.assertEqual(result.attempts, 1)
def test_get_missing_result(self) -> None:
with self.assertRaises(TaskResultDoesNotExist):
default_task_backend.get_result(str(uuid.uuid4()))
async def test_async_get_missing_result(self) -> None:
with self.assertRaises(TaskResultDoesNotExist):
await default_task_backend.aget_result(str(uuid.uuid4()))
def test_invalid_uuid(self) -> None:
with self.assertRaises(TaskResultDoesNotExist):
default_task_backend.get_result("123")
async def test_async_invalid_uuid(self) -> None:
with self.assertRaises(TaskResultDoesNotExist):
await default_task_backend.aget_result("123")
def test_invalid_task_path(self) -> None:
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [["exit", "1"]], "kwargs": {}},
task_path="subprocess.check_output",
backend_name="default",
)
with self.assertRaisesMessage(
SuspiciousOperation,
f"Task {db_task_result.id} does not point to a Task ({db_task_result.task_path})",
):
_ = db_task_result.task
def test_missing_task_path(self) -> None:
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
task_path="missing.func",
backend_name="default",
)
with self.assertRaises(ImportError):
_ = db_task_result.task
def test_task_name(self) -> None:
for task_path, expected_task_name in [
("tests.tasks.noop_task", "noop_task"),
("tests.tasks.task_not_found", "task_not_found"),
("tests.tasks.module_not_found.module_not_found", "module_not_found"),
("unexpected_function", "unexpected_function"),
]:
with self.subTest(task_path):
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
task_path=task_path,
backend_name="default",
)
self.assertEqual(db_task_result.task_name, expected_task_name)
def test_check(self) -> None:
errors = list(default_task_backend.check())
self.assertEqual(len(errors), 0, errors)
@override_settings(INSTALLED_APPS=[])
def test_database_backend_app_missing(self) -> None:
errors = list(default_task_backend.check())
self.assertEqual(len(errors), 1)
self.assertIn("django_tasks_db", errors[0].hint)
def test_priority_range_check(self) -> None:
with self.assertRaises(IntegrityError):
DBTaskResult.objects.create(
task_path="", backend_name="default", priority=-101, args_kwargs={}
)
with self.assertRaises(IntegrityError):
DBTaskResult.objects.create(
task_path="", backend_name="default", priority=101, args_kwargs={}
)
# Django accepts the float, but only stores an int
result = DBTaskResult.objects.create(
task_path="", backend_name="default", priority=3.1, args_kwargs={}
)
result.refresh_from_db()
self.assertEqual(result.priority, 3)
DBTaskResult.objects.create(
task_path="", backend_name="default", priority=100, args_kwargs={}
)
DBTaskResult.objects.create(
task_path="", backend_name="default", priority=-100, args_kwargs={}
)
DBTaskResult.objects.create(
task_path="", backend_name="default", priority=0, args_kwargs={}
)
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
}
}
)
def test_doesnt_wait_until_transaction_commit(self) -> None:
with transaction.atomic():
result = test_tasks.noop_task.enqueue()
self.assertIsNotNone(result.enqueued_at)
self.assertEqual(DBTaskResult.objects.count(), 1)
# SQLite locks the table during this transaction
if connection.vendor != "sqlite":
self.assertEqual(self.get_task_count_in_new_connection(), 0)
if connection.vendor != "sqlite":
self.assertEqual(self.get_task_count_in_new_connection(), 1)
def test_enqueue_logs(self) -> None:
with self.assertLogs(LOGGER, level="DEBUG") as captured_logs:
result = test_tasks.noop_task.enqueue()
self.assertEqual(len(captured_logs.output), 1)
self.assertIn("enqueued", captured_logs.output[0])
self.assertIn(result.id, captured_logs.output[0])
def test_index_scan_for_ready(self) -> None:
test_tasks.noop_task.enqueue()
# Quickly duplicate tasks
db_task = DBTaskResult.objects.get()
db_task.id = None
DBTaskResult.objects.bulk_create([copy.copy(db_task) for _ in range(5000)])
# Update query plan for certain databases
if connection.vendor == "postgresql":
with connection.cursor() as c:
c.execute(f"ANALYZE {DBTaskResult._meta.db_table};")
elif connection.vendor == "mysql":
with connection.cursor() as c:
c.execute(f"ANALYZE TABLE {DBTaskResult._meta.db_table};")
plan = DBTaskResult.objects.ready().explain()
if connection.vendor == "postgresql":
self.assertIn("tasks_db_new_ordering_idx", plan)
elif connection.vendor == "sqlite":
self.assertIn("USING INDEX tasks_db_new_ordering_idx", plan)
elif connection.vendor == "mysql":
if getattr(connection, "mysql_is_mariadb", False):
# MariaDB may discover the index, but the query planner choose not to use it
self.assertIn("tasks_db_new_ordering_idx", plan)
else:
self.assertIn("Index lookup", plan)
self.assertIn("using tasks_db_new_ordering_idx", plan)
else:
self.fail("Unknown database engine")
def test_run_after_tz(self) -> None:
for use_tz in [True, False]:
with self.subTest(use_tz=use_tz):
with override_settings(USE_TZ=use_tz):
result = test_tasks.noop_task.enqueue()
self.assertIsNone(
DBTaskResult.objects.get(id=result.id).task.run_after
)
def test_run_after_null_0016_migration(self) -> None:
from datetime import timezone
for use_tz in [True, False]:
with self.subTest(use_tz=use_tz):
with override_settings(USE_TZ=use_tz):
result = test_tasks.noop_task.enqueue()
db_result = DBTaskResult.objects.get(id=result.id)
# Literal taken from migration
db_result.run_after = datetime(
9999,
1,
1,
tzinfo=timezone.utc if use_tz else None,
)
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore", module="django.db", category=RuntimeWarning
)
db_result.save()
self.assertIsNone(
DBTaskResult.objects.get(id=result.id).task.run_after
)
def test_validate_on_enqueue(self) -> None:
with override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["unknown_queue"],
}
}
):
task_with_custom_queue_name = test_tasks.noop_task.using(
queue_name="unknown_queue"
)
with self.assertRaisesMessage(
InvalidTask, "Queue 'unknown_queue' is not valid for backend"
):
task_with_custom_queue_name.enqueue()
async def test_validate_on_aenqueue(self) -> None:
with override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["unknown_queue"],
}
}
):
task_with_custom_queue_name = test_tasks.noop_task.using(
queue_name="unknown_queue"
)
with self.assertRaisesMessage(
InvalidTask, "Queue 'unknown_queue' is not valid for backend"
):
await task_with_custom_queue_name.aenqueue()
def test_custom_id_function(self) -> None:
for id_function in ["uuid.uuid1", uuid.uuid1]:
with self.subTest(id_function):
with override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"OPTIONS": {"id_function": id_function},
}
}
):
result = test_tasks.noop_task.enqueue()
self.assertEqual(uuid.UUID(result.id).version, 1)
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"OPTIONS": {"id_function": "missing.function"},
}
}
)
def test_unknown_id_function(self) -> None:
with self.assertRaises(ImportError):
test_tasks.noop_task.enqueue()
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"OPTIONS": {
"id_function": "django.contrib.postgres.functions.RandomUUID"
},
}
}
)
@skipIf(connection.vendor != "postgresql", "RandomUUID only works on postgres")
@skipIf(VERSION < (6, 0), "DB expressions are only supported on 6.0+")
def test_postgres_db_id_function(self) -> None:
with self.assertNumQueries(1) as c:
result = test_tasks.noop_task.enqueue()
self.assertIn("GEN_RANDOM_UUID", c.captured_queries[0]["sql"])
self.assertEqual(uuid.UUID(result.id).version, 4)
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"OPTIONS": {"id_function": "django.db.models.functions.Now"},
}
}
)
@skipIf(VERSION >= (6, 0), "DB expressions are supported on 6.0+")
def test_postgres_id_function_expression(self) -> None:
with self.assertRaisesMessage(
ImproperlyConfigured,
"id_function cannot be a database expression until Django 6.0",
):
test_tasks.noop_task.enqueue()
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["default", "queue-1"],
},
"dummy": {"BACKEND": DUMMY_BACKEND},
}
)
class DatabaseBackendWorkerTestCase(TransactionTestCase):
worker_id = get_random_string(32)
run_worker = staticmethod(
partial(
call_command,
"db_worker",
verbosity=0,
batch=True,
interval=0,
startup_delay=False,
worker_id=worker_id,
)
)
def tearDown(self) -> None:
logger = logging.getLogger("django_tasks_db")
tasks_logger = logging.getLogger(LOGGER)
# Reset the logger after every run, to ensure the correct `stdout` is used
for handler in logger.handlers:
logger.removeHandler(handler)
for handler in tasks_logger.handlers:
tasks_logger.removeHandler(handler)
def test_run_enqueued_task(self) -> None:
for task in [
test_tasks.noop_task,
# test_tasks.noop_task_async,
]:
with self.subTest(task):
result = cast(Task, task).enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
self.assertEqual(result.status, TaskResultStatus.READY)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker()
self.assertEqual(result.status, TaskResultStatus.READY)
self.assertEqual(result.attempts, 0)
result.refresh()
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc]
self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc]
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
self.assertEqual(result.attempts, 1)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_batch_processes_all_tasks(self) -> None:
for _ in range(3):
test_tasks.noop_task.enqueue()
test_tasks.failing_task_value_error.enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 4)
with self.assertNumQueries(28 if connection.vendor == "mysql" else 23):
self.run_worker()
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
self.assertEqual(DBTaskResult.objects.successful().count(), 3)
self.assertEqual(DBTaskResult.objects.failed().count(), 1)
def test_no_tasks(self) -> None:
with self.assertNumQueries(4 if connection.vendor == "mysql" else 3):
self.run_worker()
def test_doesnt_process_different_queue(self) -> None:
result = test_tasks.noop_task.using(queue_name="queue-1").enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(4 if connection.vendor == "mysql" else 3):
self.run_worker()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker(queue_name=result.task.queue_name)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_process_all_queues(self) -> None:
test_tasks.noop_task.using(queue_name="queue-1").enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(4 if connection.vendor == "mysql" else 3):
self.run_worker()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker(queue_name="*")
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_process_all_queues_excluding_one_queue(self) -> None:
test_tasks.noop_task.enqueue()
excluded_result = test_tasks.noop_task.using(queue_name="queue-1").enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 2)
self.run_worker(queue_name="*", exclude_queues="queue-1")
excluded_result.refresh()
self.assertEqual(excluded_result.status, TaskResultStatus.READY)
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
self.assertEqual(DBTaskResult.objects.successful().count(), 1)
def test_exclude_queues_requires_processing_all_queues(self) -> None:
test_tasks.noop_task.enqueue()
test_tasks.noop_task.using(queue_name="queue-1").enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 2)
with self.assertRaisesMessage(
CommandError, "--exclude-queues can only be used with --queue-name=*"
):
self.run_worker(queue_name="default,queue-1", exclude_queues="queue-1")
self.assertEqual(DBTaskResult.objects.ready().count(), 2)
self.assertEqual(DBTaskResult.objects.successful().count(), 0)
def test_excluding_unknown_queue_processes_selected_queues(self) -> None:
test_tasks.noop_task.using(queue_name="queue-1").enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
self.run_worker(queue_name="*", exclude_queues="missing")
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
self.assertEqual(DBTaskResult.objects.successful().count(), 1)
def test_failing_task(self) -> None:
result = test_tasks.failing_task_value_error.enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker()
self.assertEqual(result.status, TaskResultStatus.READY)
result.refresh()
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore
self.assertGreaterEqual(result.finished_at, result.started_at) # type: ignore
self.assertEqual(result.status, TaskResultStatus.FAILED)
with self.assertRaisesMessage(ValueError, "Task failed"):
result.return_value # noqa: B018
self.assertEqual(result.errors[0].exception_class, ValueError)
traceback = result.errors[0].traceback
self.assertTrue(
traceback
and traceback.endswith("ValueError: This task failed due to ValueError\n"),
traceback,
)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_complex_exception(self) -> None:
result = test_tasks.complex_exception.enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker()
self.assertEqual(result.status, TaskResultStatus.READY)
result.refresh()
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertGreaterEqual(result.started_at, result.enqueued_at) # type: ignore
self.assertGreaterEqual(result.finished_at, result.started_at) # type: ignore
self.assertEqual(result.status, TaskResultStatus.FAILED)
with self.assertRaisesMessage(ValueError, "Task failed"):
result.return_value # noqa: B018
self.assertEqual(result.errors[0].exception_class, ValueError)
self.assertIn(
'ValueError(ValueError("This task failed"))', result.errors[0].traceback
)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_complex_return_value(self) -> None:
result = test_tasks.complex_return_value.enqueue()
self.run_worker()
result.refresh()
self.assertEqual(result.status, TaskResultStatus.FAILED)
self.assertIsNotNone(result.started_at)
self.assertIsNotNone(result.last_attempted_at)
self.assertIsNotNone(result.finished_at)
self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc]
self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc]
self.assertIsNone(result._return_value) # type: ignore[attr-defined]
self.assertEqual(result.errors[0].exception_class, TypeError)
self.assertIn("is not JSON serializable", result.errors[0].traceback)
def test_doesnt_process_different_backend(self) -> None:
result = test_tasks.failing_task_value_error.enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(4 if connection.vendor == "mysql" else 3):
self.run_worker(backend_name="dummy")
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker(backend_name=result.backend)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
def test_unknown_backend(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--backend", "unknown"]
)
self.assertIn("The connection 'unknown' doesn't exist.", output.getvalue())
def test_incorrect_backend(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--backend", "dummy"]
)
self.assertIn("Backend 'dummy' is not a database backend", output.getvalue())
def test_negative_interval(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--interval", "-1"]
)
self.assertIn("Must be zero or greater", output.getvalue())
def test_infinite_interval(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--interval", "inf"]
)
self.assertIn("Must be a finite floating point value", output.getvalue())
def test_fractional_interval(self) -> None:
with mock.patch(
"django_tasks_db.management.commands.db_worker.Worker"
) as worker_class:
execute_from_command_line(
["django-admin", "db_worker", "--interval", "0.1"]
)
self.assertEqual(worker_class.mock_calls[0].kwargs["interval"], 0.1)
def test_zero_interval(self) -> None:
with mock.patch(
"django_tasks_db.management.commands.db_worker.Worker"
) as worker_class:
execute_from_command_line(["django-admin", "db_worker", "--interval", "0"])
self.assertEqual(worker_class.mock_calls[0].kwargs["interval"], 0)
def test_negative_max_tasks(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--max-tasks", "-1"]
)
self.assertIn("Must be greater than zero", output.getvalue())
def test_zero_max_tasks(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--max-tasks", "0"]
)
self.assertIn("Must be greater than zero", output.getvalue())
def test_too_long_worker_id(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--worker-id", "A" * 65]
)
self.assertIn(
"Worker ids must be shorter than 64 characters", output.getvalue()
)
def test_empty_worker_id(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "db_worker", "--worker-id", ""]
)
self.assertIn("Worker id must not be empty", output.getvalue())
def test_run_after(self) -> None:
result = test_tasks.noop_task.using(
run_after=timezone.now() + timedelta(hours=10)
).enqueue()
self.assertEqual(DBTaskResult.objects.count(), 1)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
with self.assertNumQueries(4 if connection.vendor == "mysql" else 3):
self.run_worker()
self.assertEqual(DBTaskResult.objects.count(), 1)
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
self.assertEqual(DBTaskResult.objects.successful().count(), 0)
DBTaskResult.objects.filter(id=result.id).update(run_after=timezone.now())
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
with self.assertNumQueries(10 if connection.vendor == "mysql" else 8):
self.run_worker()
self.assertEqual(DBTaskResult.objects.ready().count(), 0)
self.assertEqual(DBTaskResult.objects.successful().count(), 1)
def test_run_after_priority(self) -> None:
old_task = test_tasks.noop_task.using(
priority=20, run_after=timezone.now() - timedelta(hours=2)
).enqueue()
very_old_task = test_tasks.noop_task.using(
priority=20, run_after=timezone.now() - timedelta(hours=10)
).enqueue()
far_future_result = test_tasks.noop_task.using(
run_after=timezone.now() + timedelta(hours=10)
).enqueue()
high_priority_far_future_result = test_tasks.noop_task.using(
priority=10, run_after=timezone.now() + timedelta(hours=10)
).enqueue()
future_result = test_tasks.noop_task.using(
run_after=timezone.now() + timedelta(hours=2)
).enqueue()
high_priority_result = test_tasks.noop_task.using(priority=10).enqueue()
low_priority_result = test_tasks.noop_task.using(priority=2).enqueue()
lower_priority_result = test_tasks.noop_task.using(priority=-2).enqueue()
self.assertEqual(
[dbt.task_result for dbt in DBTaskResult.objects.all()],
[
very_old_task,
old_task,
high_priority_far_future_result,
high_priority_result,
low_priority_result,
future_result,
far_future_result,
lower_priority_result,
],
)
self.assertEqual(
[dbt.task_result for dbt in DBTaskResult.objects.ready()],
[
very_old_task,
old_task,
high_priority_result,
low_priority_result,
lower_priority_result,
],
)
def test_verbose_logging(self) -> None:
result = test_tasks.noop_task.enqueue()
stdout = StringIO()
self.run_worker(verbosity=3, stdout=stdout, stderr=stdout)
self.assertEqual(
stdout.getvalue().splitlines(),
[
f"Starting worker worker_id={self.worker_id} queues=default",
f"Task id={result.id} path=tests.tasks.noop_task state=RUNNING",
f"Task id={result.id} path=tests.tasks.noop_task state=SUCCESSFUL",
f"No more tasks to run for worker_id={self.worker_id} - exiting gracefully.",
],
)
def test_invalid_task_path(self) -> None:
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [["exit", "1"]], "kwargs": {}},
task_path="subprocess.check_output",
backend_name="default",
)
self.run_worker()
db_task_result.refresh_from_db()
self.assertEqual(db_task_result.status, TaskResultStatus.FAILED)
def test_missing_task_path(self) -> None:
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
task_path="missing.func",
backend_name="default",
)
self.run_worker()
db_task_result.refresh_from_db()
self.assertEqual(db_task_result.status, TaskResultStatus.FAILED)
def test_worker_doesnt_exit_on_failure(self) -> None:
result = test_tasks.exit_task.enqueue()
self.run_worker()
result.refresh()
self.assertEqual(result.status, TaskResultStatus.FAILED)
@skipIf(connection.vendor == "sqlite", "SQLite locks the entire database")
def test_worker_with_locked_rows(self) -> None:
result_1 = test_tasks.noop_task.enqueue()
new_connection = connections.create_connection("default")
with transaction.atomic():
locked_tasks_query = str(DBTaskResult.objects.select_for_update().query)
try:
# Start a transaction in the other connection
with new_connection.cursor() as c:
c.execute("BEGIN")
# Lock the current rows in the table
with new_connection.cursor() as c:
c.execute(locked_tasks_query)
results = list(c.fetchall())
self.assertEqual(len(results), 1)
# Add another task which isn't locked
result_2 = test_tasks.noop_task.enqueue()
self.run_worker()
finally:
new_connection.close()
result_1.refresh()
result_2.refresh()
self.assertEqual(result_1.status, TaskResultStatus.READY)
self.assertEqual(result_2.status, TaskResultStatus.SUCCESSFUL)
def test_max_tasks(self) -> None:
results = [test_tasks.noop_task.enqueue() for _ in range(5)]
stdout = StringIO()
self.run_worker(max_tasks=2, stdout=stdout, verbosity=3)
self.assertIn("Run maximum tasks (2)", stdout.getvalue())
for result in results:
result.refresh()
statuses = Counter(result.status for result in results)
self.assertEqual(statuses[TaskResultStatus.SUCCESSFUL], 2)
self.assertEqual(statuses[TaskResultStatus.READY], 3)
def test_takes_context(self) -> None:
result = test_tasks.get_task_id.enqueue()
self.run_worker()
result.refresh()
self.assertEqual(result.return_value, result.id)
def test_context(self) -> None:
result = test_tasks.test_context.enqueue(1)
self.run_worker()
result.refresh()
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
def test_task_import_string(self) -> None:
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
task_path="tests.tasks.some_test",
backend_name="default",
)
self.run_worker()
db_task_result.refresh_from_db()
self.assertEqual(db_task_result.status, TaskResultStatus.FAILED)
self.assertIn("ImportError", db_task_result.traceback)
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
},
}
)
class DatabaseTaskResultTestCase(TransactionTestCase):
def execute_in_new_connection(self, sql: str | QuerySet) -> Sequence:
if isinstance(sql, QuerySet):
sql = str(sql.query)
new_connection = connections.create_connection("default")
try:
with new_connection.cursor() as c:
c.execute(sql)
return cast(list, c.fetchall())
finally:
new_connection.close()
def test_cross_connection(self) -> None:
test_tasks.noop_task.enqueue()
test_tasks.noop_task.enqueue()
self.assertEqual(DBTaskResult.objects.count(), 2)
self.assertEqual(DBTaskResult.objects.using("default").count(), 2)
self.assertEqual(
len(self.execute_in_new_connection(DBTaskResult.objects.all())),
2,
)
@skipIf(connection.vendor == "sqlite", "SQLite handles locks differently")
def test_locks_tasks(self) -> None:
test_tasks.noop_task.enqueue()
test_tasks.noop_task.enqueue()
with transaction.atomic():
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
2,
)
self.assertIsNotNone(DBTaskResult.objects.get_locked())
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
# MySQL likes to lock all the rows
0 if connection.vendor == "mysql" else 1,
)
DBTaskResult.objects.get_locked()
with transaction.atomic():
# The original transaction has closed, so the result is unlocked
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
2,
)
@skipIf(connection.vendor != "sqlite", "SQLite handles locks differently")
def test_locks_tasks_sqlite(self) -> None:
result = test_tasks.noop_task.enqueue()
with exclusive_transaction():
locked_result = DBTaskResult.objects.get_locked()
self.assertEqual(result.id, str(locked_result.id)) # type: ignore[union-attr]
with self.assertRaisesMessage(OperationalError, "is locked"):
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
# The original transaction has closed, so the database is unlocked
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
@skipIf(connection.vendor == "sqlite", "SQLite handles locks differently")
def test_locks_tasks_filtered(self) -> None:
result = test_tasks.noop_task.using(priority=10).enqueue()
test_tasks.noop_task.enqueue()
with transaction.atomic():
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
2,
)
locked_result = DBTaskResult.objects.filter(
priority=result.task.priority
).get_locked()
self.assertEqual(str(locked_result.id), result.id) # type: ignore[union-attr]
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
1,
)
with transaction.atomic():
# The original transaction has closed, so the result is unlocked
self.assertEqual(
len(
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
),
2,
)
@skipIf(connection.vendor != "sqlite", "SQLite handles locks differently")
def test_locks_tasks_filtered_sqlite(self) -> None:
result = test_tasks.noop_task.using(priority=10).enqueue()
test_tasks.noop_task.enqueue()
with exclusive_transaction():
locked_result = DBTaskResult.objects.filter(
priority=result.task.priority
).get_locked()
self.assertEqual(result.id, str(locked_result.id)) # type: ignore[union-attr]
with self.assertRaisesMessage(OperationalError, "is locked"):
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
# The original transaction has closed, so the database is unlocked
self.execute_in_new_connection(
DBTaskResult.objects.select_for_update(skip_locked=True)
)
@exclusive_transaction()
def test_lock_no_rows(self) -> None:
self.assertEqual(DBTaskResult.objects.count(), 0)
self.assertIsNone(DBTaskResult.objects.all().get_locked())
@skipIf(connection.vendor == "sqlite", "SQLite handles locks differently")
def test_get_locked_with_locked_rows(self) -> None:
result_1 = test_tasks.noop_task.enqueue()
new_connection = connections.create_connection("default")
with transaction.atomic():
locked_tasks_query = str(DBTaskResult.objects.select_for_update().query)
try:
# Start a transaction in the other connection
with new_connection.cursor() as c:
c.execute("BEGIN")
# Lock the current rows in the table from the other connection
with new_connection.cursor() as c:
c.execute(locked_tasks_query)
results = list(c.fetchall())
self.assertEqual(len(results), 1)
self.assertEqual(normalize_uuid(results[0][0]), normalize_uuid(result_1.id))
with transaction.atomic():
# .count with skip_locked isn't supported
self.assertEqual(
len(DBTaskResult.objects.select_for_update(skip_locked=True)), 0
)
self.assertIsNone(DBTaskResult.objects.get_locked())
# Add another task which isn't locked
result_2 = test_tasks.noop_task.enqueue()
with transaction.atomic():
self.assertEqual(
normalize_uuid(
DBTaskResult.objects.select_for_update(
skip_locked=True
).values_list("id", flat=True)[0]
),
normalize_uuid(result_2.id),
)
self.assertEqual(
normalize_uuid(DBTaskResult.objects.get_locked().id), # type: ignore[union-attr]
normalize_uuid(result_2.id),
)
finally:
new_connection.close()
class ConnectionExclusiveTranscationTestCase(TransactionTestCase):
def setUp(self) -> None:
self.connection = connections.create_connection("default")
def tearDown(self) -> None:
self.connection.close()
# connection.close()
@skipIf(connection.vendor != "sqlite", "SQLite only")
def test_exclusive_transaction(self) -> None:
with self.assertNumQueries(2) as c:
with exclusive_transaction():
pass
self.assertEqual(c.captured_queries[0]["sql"], "BEGIN EXCLUSIVE")
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
"QUEUES": ["default", "queue-1"],
},
"dummy": {"BACKEND": DUMMY_BACKEND},
}
)
class DatabaseBackendPruneTaskResultsTestCase(TransactionTestCase):
prune_task_results = staticmethod(
partial(call_command, "prune_db_task_results", verbosity=0)
)
def tearDown(self) -> None:
# Reset the logger after every run, to ensure the correct `stdout` is used
for handler in prune_db_tasks_logger.handlers:
prune_db_tasks_logger.removeHandler(handler)
def test_prunes_tasks(self) -> None:
result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.all().update(
status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now()
)
self.assertEqual(DBTaskResult.objects.finished().count(), 1)
stdout = StringIO()
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=0, stdout=stdout, verbosity=3)
self.assertEqual(DBTaskResult.objects.finished().count(), 0)
with self.assertRaises(TaskResultDoesNotExist):
result.refresh()
self.assertEqual(stdout.getvalue().strip(), "Deleted 1 task result(s)")
def test_doesnt_prune_new_tasks(self) -> None:
result = test_tasks.noop_task.enqueue()
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
stdout = StringIO()
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=0, stdout=stdout, verbosity=3)
self.assertEqual(DBTaskResult.objects.ready().count(), 1)
result.refresh()
self.assertEqual(stdout.getvalue().strip(), "Deleted 0 task result(s)")
def test_doesnt_prune_running_tasks(self) -> None:
result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.all().update(status=TaskResultStatus.RUNNING)
self.assertEqual(DBTaskResult.objects.running().count(), 1)
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=0)
self.assertEqual(DBTaskResult.objects.running().count(), 1)
result.refresh()
def test_only_prunes_specified_queue(self) -> None:
result = test_tasks.noop_task.enqueue()
queue_1_result = test_tasks.noop_task.using(queue_name="queue-1").enqueue()
DBTaskResult.objects.all().update(
status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now()
)
self.assertEqual(DBTaskResult.objects.successful().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results(queue_name="queue-1", min_age_days=0)
self.assertEqual(DBTaskResult.objects.successful().count(), 1)
result.refresh()
with self.assertRaises(TaskResultDoesNotExist):
queue_1_result.refresh()
def test_prune_all_queues(self) -> None:
test_tasks.noop_task.enqueue()
test_tasks.noop_task.using(queue_name="queue-1").enqueue()
DBTaskResult.objects.all().update(
status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now()
)
self.assertEqual(DBTaskResult.objects.successful().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results(queue_name="*", min_age_days=0)
self.assertEqual(DBTaskResult.objects.successful().count(), 0)
def test_min_age(self) -> None:
one_day_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.SUCCESSFUL,
finished_at=timezone.now() - timedelta(days=1),
)
three_day_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.SUCCESSFUL,
finished_at=timezone.now() - timedelta(days=3),
)
self.assertEqual(DBTaskResult.objects.successful().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results()
self.assertEqual(DBTaskResult.objects.successful().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=3)
self.assertEqual(DBTaskResult.objects.successful().count(), 1)
one_day_result.refresh()
with self.assertRaises(TaskResultDoesNotExist):
three_day_result.refresh()
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=1)
self.assertEqual(DBTaskResult.objects.successful().count(), 0)
def test_failed_min_age(self) -> None:
successful_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.SUCCESSFUL,
finished_at=timezone.now() - timedelta(days=3),
)
failed_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.FAILED,
finished_at=timezone.now() - timedelta(days=3),
)
self.assertEqual(DBTaskResult.objects.finished().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results()
self.assertEqual(DBTaskResult.objects.finished().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=3, failed_min_age_days=5)
self.assertEqual(DBTaskResult.objects.finished().count(), 1)
failed_result.refresh()
with self.assertRaises(TaskResultDoesNotExist):
successful_result.refresh()
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=3, failed_min_age_days=1)
with self.assertRaises(TaskResultDoesNotExist):
failed_result.refresh()
def test_failed_min_age_zero(self) -> None:
successful_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.SUCCESSFUL,
finished_at=timezone.now() - timedelta(days=1),
)
failed_result = test_tasks.noop_task.enqueue()
DBTaskResult.objects.ready().update(
status=TaskResultStatus.FAILED,
finished_at=timezone.now() - timedelta(days=1),
)
self.assertEqual(DBTaskResult.objects.finished().count(), 2)
with self.assertNumQueries(3):
self.prune_task_results(min_age_days=14, failed_min_age_days=0)
self.assertEqual(DBTaskResult.objects.finished().count(), 1)
successful_result.refresh()
with self.assertRaises(TaskResultDoesNotExist):
failed_result.refresh()
def test_dry_run(self) -> None:
test_tasks.noop_task.enqueue()
DBTaskResult.objects.all().update(
status=TaskResultStatus.SUCCESSFUL, finished_at=timezone.now()
)
self.assertEqual(DBTaskResult.objects.count(), 1)
stdout = StringIO()
with self.assertNumQueries(1):
self.prune_task_results(
min_age_days=0, dry_run=True, stdout=stdout, verbosity=3
)
self.assertEqual(DBTaskResult.objects.count(), 1)
self.assertEqual(stdout.getvalue().strip(), "Would delete 1 task result(s)")
def test_unknown_backend(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "prune_db_task_results", "--backend", "unknown"]
)
self.assertIn("The connection 'unknown' doesn't exist.", output.getvalue())
def test_incorrect_backend(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "prune_db_task_results", "--backend", "dummy"]
)
self.assertIn("Backend 'dummy' is not a database backend", output.getvalue())
def test_negative_age(self) -> None:
output = StringIO()
with redirect_stderr(output):
with self.assertRaises(SystemExit):
execute_from_command_line(
["django-admin", "prune_db_task_results", "--min-age-days", "-1"]
)
self.assertIn("Must be greater than zero", output.getvalue())
@override_settings(
TASKS={
"default": {
"BACKEND": "django_tasks_db.DatabaseBackend",
},
}
)
@skipIfInMemoryDB()
class DatabaseWorkerProcessTestCase(TransactionTestCase):
WORKER_STARTUP_TIME = 1
def setUp(self) -> None:
self.processes: list[subprocess.Popen] = []
def tearDown(self) -> None:
# Try n times to kill any remaining child processes
for n in range(20):
for process in self.processes:
if process.poll() is None:
if n >= 5:
print("Still waiting for process", process.pid, process.args) # noqa: T201
process.kill()
process.wait(1)
def start_worker(
self,
args: list[str] | None = None,
*,
debug: bool = False,
worker_id: str | None = None,
) -> subprocess.Popen:
if args is None:
args = []
if worker_id is None:
worker_id = get_random_string(32)
p = subprocess.Popen(
[
sys.executable,
"-m",
"manage",
"db_worker",
"--verbosity",
"3",
"--no-startup-delay",
"--worker-id",
worker_id,
*args,
],
stdout=None if debug else subprocess.PIPE,
stderr=None if debug else subprocess.STDOUT,
env={
**os.environ,
"DJANGO_SETTINGS_MODULE": "tests.db_worker_test_settings",
"IN_TEST": "",
},
text=True,
)
self.processes.append(p)
return p
def test_run_subprocess(self) -> None:
result = test_tasks.noop_task.enqueue()
process = self.start_worker(["--batch"])
process.wait()
self.assertEqual(process.returncode, 0)
self.assertEqual(result.status, TaskResultStatus.READY)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
@skipIf(sys.platform == "win32", "Terminate is always forceful on Windows")
def test_interrupt_no_tasks(self) -> None:
process = self.start_worker()
time.sleep(self.WORKER_STARTUP_TIME)
process.terminate()
process.wait(timeout=0.5)
self.assertEqual(process.returncode, 0)
@skipIf(sys.platform == "win32", "Cannot emulate CTRL-C on Windows")
def test_interrupt_signals(self) -> None:
for sig in [
signal.SIGINT, # ctrl-c
signal.SIGTERM,
]:
with self.subTest(sig):
result = test_tasks.sleep_for.enqueue(2)
self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [])
self.assertGreater(result.args[0], self.WORKER_STARTUP_TIME)
process = self.start_worker()
# Make sure the task is running by now
time.sleep(self.WORKER_STARTUP_TIME)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.RUNNING)
self.assertNotEqual(
DBTaskResult.objects.get(id=result.id).worker_ids, []
)
process.send_signal(sig)
process.wait(timeout=2)
self.assertEqual(process.returncode, 0)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
@skipIf(sys.platform == "win32", "Cannot emulate CTRL-C on Windows")
def test_repeat_ctrl_c(self) -> None:
result = test_tasks.hang.enqueue()
self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [])
worker_id = get_random_string(32)
process = self.start_worker(worker_id=worker_id)
# Make sure the task is running by now
time.sleep(self.WORKER_STARTUP_TIME)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.RUNNING)
self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [worker_id])
process.send_signal(signal.SIGINT)
time.sleep(0.5)
self.assertIsNone(process.poll())
result.refresh()
self.assertEqual(result.status, TaskResultStatus.RUNNING)
self.assertEqual(DBTaskResult.objects.get(id=result.id).worker_ids, [worker_id])
process.send_signal(signal.SIGINT)
process.wait(timeout=2)
self.assertEqual(process.returncode, 0)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.FAILED)
self.assertEqual(result.errors[0].exception_class, SystemExit)
@skipIf(sys.platform == "win32", "Windows doesn't support SIGKILL")
def test_kill(self) -> None:
# Required to keep mypy happy
assert hasattr(signal, "SIGKILL")
result = test_tasks.hang.enqueue()
process = self.start_worker()
# Make sure the task is running by now
time.sleep(self.WORKER_STARTUP_TIME)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.RUNNING)
process.kill()
process.wait(timeout=2)
self.assertEqual(process.returncode, -signal.SIGKILL)
result.refresh()
# TODO: https://github.com/RealOrangeOne/django-tasks-db/issues/46
self.assertEqual(result.status, TaskResultStatus.RUNNING)
def test_system_exit_task(self) -> None:
result = test_tasks.failing_task_system_exit.enqueue()
process = self.start_worker(["--batch"])
process.wait(timeout=2)
self.assertEqual(process.returncode, 0)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.FAILED)
self.assertEqual(result.errors[0].exception_class, SystemExit)
def test_keyboard_interrupt_task(self) -> None:
result = test_tasks.failing_task_keyboard_interrupt.enqueue()
process = self.start_worker(["--batch"])
process.wait(timeout=2)
self.assertEqual(process.returncode, 0)
result.refresh()
self.assertEqual(result.status, TaskResultStatus.FAILED)
self.assertEqual(result.errors[0].exception_class, KeyboardInterrupt)
def test_multiple_workers(self) -> None:
results = [test_tasks.sleep_for.enqueue(0.1) for _ in range(10)]
for _ in range(3):
self.start_worker(["--batch"])
time.sleep(self.WORKER_STARTUP_TIME)
for process in self.processes:
process.wait(timeout=5)
self.assertIsNotNone(process.returncode)
for result in results:
result.refresh()
self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL)
all_output = ""
for process in self.processes:
stdout_text = process.stdout.read() # type:ignore[union-attr]
all_output += stdout_text
self.assertIn("gracefully", stdout_text)
for result in results:
# Running and successful
self.assertEqual(all_output.count(result.id), 2)
class CompatTestCase(SimpleTestCase):
def test_compat_has_django_task(self) -> None:
if VERSION >= (6, 0):
from django.tasks.base import Task as DjangoTask
self.assertEqual(DjangoTask, compat.Task)
else:
self.assertEqual(Task, compat.Task)
class DBTaskResultOrderingTestCase(TestCase):
def _create_task(
self,
priority: int = 0,
run_after: "datetime | None" = None,
enqueued_at: "datetime | None" = None,
) -> DBTaskResult:
if run_after is None:
run_after = timezone.now()
task = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
run_after=run_after,
priority=priority,
enqueued_at=enqueued_at or timezone.now(),
)
return task
def test_higher_priority_tasks_come_first(self) -> None:
now = timezone.now()
low = self._create_task(priority=-1, run_after=now)
normal = self._create_task(priority=0, run_after=now)
high = self._create_task(priority=1, run_after=now)
self.assertEqual(
list(DBTaskResult.objects.values_list("id", flat=True)),
[high.id, normal.id, low.id],
)
def test_earlier_run_after_comes_first_given_same_priority(self) -> None:
now = timezone.now()
early = self._create_task(run_after=now - timedelta(hours=1))
late = self._create_task(run_after=now + timedelta(hours=1))
on_time = self._create_task(run_after=now)
self.assertEqual(
list(DBTaskResult.objects.values_list("id", flat=True)),
[early.id, on_time.id, late.id],
)
def test_earlier_enqueued_at_comes_first_given_same_priority_and_run_after(
self,
) -> None:
now = timezone.now()
run_after = now + timedelta(hours=1)
first = self._create_task(
run_after=run_after, enqueued_at=now - timedelta(minutes=5)
)
second = self._create_task(run_after=run_after, enqueued_at=now)
third = self._create_task(
run_after=run_after, enqueued_at=now + timedelta(minutes=5)
)
self.assertEqual(
list(DBTaskResult.objects.values_list("id", flat=True)),
[first.id, second.id, third.id],
)
class AdminTestCase(TestCase):
def setUp(self) -> None:
self.admin = DBTaskResultAdmin(DBTaskResult, admin.AdminSite) # type:ignore[arg-type]
def test_display_run_after_returns_empty_for_date_max_or_none(self) -> None:
for run_after_value in [get_date_max(), None]:
with self.subTest(run_after_value):
db_task_result = DBTaskResult.objects.create( # type:ignore
args_kwargs={"args": [], "kwargs": {}},
run_after=run_after_value,
)
result = self.admin.display_run_after(db_task_result)
self.assertEqual(result, self.admin.get_empty_value_display())
def test_display_run_after_returns_db_value(self) -> None:
expected_run_after = timezone.make_aware(
datetime(2025, 4, 3),
timezone.get_current_timezone(),
)
db_task_result = DBTaskResult.objects.create(
args_kwargs={"args": [], "kwargs": {}},
run_after=expected_run_after,
)
result = self.admin.display_run_after(db_task_result)
self.assertEqual(result, expected_run_after)