pax_global_header00006660000000000000000000000064151414151040014506gustar00rootroot0000000000000052 comment=fefcb02697c47c68bcd9d81ffae395abcf8bd4d2 django-tasks-rq-0.12.0/000077500000000000000000000000001514141510400145735ustar00rootroot00000000000000django-tasks-rq-0.12.0/.github/000077500000000000000000000000001514141510400161335ustar00rootroot00000000000000django-tasks-rq-0.12.0/.github/workflows/000077500000000000000000000000001514141510400201705ustar00rootroot00000000000000django-tasks-rq-0.12.0/.github/workflows/ci.yml000066400000000000000000000042051514141510400213070ustar00rootroot00000000000000name: CI on: pull_request: push: branches: - master tags: - "*" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [windows-latest, macos-latest, ubuntu-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] django-version: ["4.2", "5.2", "6.0"] exclude: - django-version: "6.0" python-version: "3.10" - django-version: "6.0" python-version: "3.11" - django-version: "4.2" python-version: "3.14" steps: - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v5 with: path: ~/.cache/pip key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }} - uses: taiki-e/install-action@just - name: Install dependencies run: | pip install --upgrade pip pip install -e . --group dev pip install Django~=${{ matrix.django-version }} - name: Lint run: just lint - name: Run tests if: ${{ !cancelled() }} run: just test -v2 build: permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing runs-on: ubuntu-latest needs: - test steps: - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.14" - name: Install dependencies run: | python -m pip install --upgrade pip build - name: Build package run: python -m build - name: Save built package uses: actions/upload-artifact@v6 with: name: package path: dist - name: Publish to PyPi if: ${{ github.ref_type == 'tag' }} uses: pypa/gh-action-pypi-publish@release/v1 with: print-hash: true django-tasks-rq-0.12.0/.gitignore000066400000000000000000000066441514141510400165750ustar00rootroot00000000000000# Created by https://www.toptal.com/developers/gitignore/api/python # Edit at https://www.toptal.com/developers/gitignore?templates=python ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py *.sqlite3 *.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: # .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # poetry # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. # This is especially recommended for binary packages to ensure reproducibility, and is more # commonly ignored for libraries. # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control #poetry.lock # pdm # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. # https://pdm.fming.dev/#use-with-ide .pdm.toml # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ ### Python Patch ### # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration poetry.toml # ruff .ruff_cache/ # LSP config files pyrightconfig.json # End of https://www.toptal.com/developers/gitignore/api/python django-tasks-rq-0.12.0/CONTRIBUTING.md000066400000000000000000000014421514141510400170250ustar00rootroot00000000000000# Contributing Found a bug? Want to fix an open issue? Got an idea for an improvement? Please contribute! **All** contributions are welcome, from absolutely anyone. Just open a PR, Issue or Discussion (as relevant) - no need to ask beforehand. If you're going to work on an issue, it's a good idea to say so on the issue, to make sure work isn't duplicated. ## Development set up Fork, then clone the repo: ```sh git clone git@github.com:your-username/django-tasks-rq.git ``` Set up a venv: ```sh python -m venv .venv source .venv/bin/activate python -m pip install -e --group dev ``` Then you can run the tests with the [just](https://just.systems/man/en/) command runner: ```sh just test ``` If you don't have `just` installed, you can look in the `justfile` for the commands that are run. django-tasks-rq-0.12.0/LICENSE000066400000000000000000000030171514141510400156010ustar00rootroot00000000000000Copyright (c) Jake Howard and individual contributors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. django-tasks-rq-0.12.0/README.md000066400000000000000000000043721514141510400160600ustar00rootroot00000000000000# Django Tasks RQ [![CI](https://github.com/RealOrangeOne/django-tasks-rq/actions/workflows/ci.yml/badge.svg)](https://github.com/RealOrangeOne/django-tasks-rq/actions/workflows/ci.yml) ![PyPI](https://img.shields.io/pypi/v/django-tasks-rq.svg) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/django-tasks-rq.svg) ![PyPI - Status](https://img.shields.io/pypi/status/django-tasks-rq.svg) ![PyPI - License](https://img.shields.io/pypi/l/django-tasks-rq.svg) A [Django Tasks](https://docs.djangoproject.com/en/stable/topics/tasks/) backend which uses RQ as its underlying queue. ## Installation ``` python -m pip install django-tasks ``` First, add `django_tasks_rq` to your `INSTALLED_APPS`: ```python INSTALLED_APPS = [ # ... "django_tasks_rq", ] ``` Finally, add it to your `TASKS` configuration: ```python TASKS = { "default": { "BACKEND": "django_tasks_rq.RQBackend", "QUEUES": ["default"] } } ``` ## Usage The RQ-based backend acts as an interface between [Django's tasks interface](https://docs.djangoproject.com/en/stable/topics/tasks/) and `RQ`, allowing tasks to be defined and enqueued using `django.tasks`, but stored in Redis and executed using RQ's workers. Any queues defined in `QUEUES` must also be defined in `django-rq`'s `RQ_QUEUES` setting. ### Job class To use `rq` with `django-tasks-rq`, a custom `Job` class must be used. This can be passed to the worker using `--job-class`: ```shell ./manage.py rqworker --job-class django_tasks_rq.Job ``` ### Priorities `rq` has no native concept of priorities - instead relying on workers to define which queues they should pop tasks from in order. Therefore, `task.priority` has little effect on execution priority. If a task has a priority of `100`, it is enqueued at the top of the queue, and will be the next task executed by a worker. All other priorities will enqueue the task to the back of the queue. The queue value is not stored, and will always be `0`. ## Contributing See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute. Note: Prior to `0.12.0`, this backend was included in [`django-tasks`](https://github.com/RealOrangeOne/django-tasks/). Whilst the commit history was cleaned up, it's still quite messy. Don't look too closely. django-tasks-rq-0.12.0/django_tasks_rq/000077500000000000000000000000001514141510400177445ustar00rootroot00000000000000django-tasks-rq-0.12.0/django_tasks_rq/__init__.py000066400000000000000000000003371514141510400220600ustar00rootroot00000000000000# ruff: noqa: E402 import django_stubs_ext django_stubs_ext.monkeypatch() import importlib.metadata __version__ = importlib.metadata.version(__name__) from .backend import Job, RQBackend __all__ = ["RQBackend", "Job"] django-tasks-rq-0.12.0/django_tasks_rq/backend.py000066400000000000000000000230761514141510400217150ustar00rootroot00000000000000from collections.abc import Iterable from importlib.metadata import version from types import TracebackType from typing import Any, TypeVar, cast import django_rq from django.apps import apps from django.core.checks import messages from django.core.exceptions import SuspiciousOperation from django.utils.functional import cached_property from django_tasks.backends.base import BaseTaskBackend from django_tasks.base import ( DEFAULT_TASK_PRIORITY, TASK_MAX_PRIORITY, Task, TaskContext, TaskError, TaskResult, TaskResultStatus, ) from django_tasks.exceptions import TaskResultDoesNotExist from django_tasks.signals import task_enqueued, task_finished, task_started from django_tasks.utils import get_module_path, get_random_id from redis.client import Redis from rq.defaults import UNSERIALIZABLE_RETURN_VALUE_PAYLOAD from rq.exceptions import NoSuchJobError from rq.job import Callback, JobStatus from rq.job import Job as BaseJob from rq.registry import ScheduledJobRegistry from rq.results import Result from typing_extensions import ParamSpec from .compat import TASK_CLASSES T = TypeVar("T") P = ParamSpec("P") RQ_STATUS_TO_RESULT_STATUS = { JobStatus.QUEUED: TaskResultStatus.READY, JobStatus.FINISHED: TaskResultStatus.SUCCESSFUL, JobStatus.FAILED: TaskResultStatus.FAILED, JobStatus.STARTED: TaskResultStatus.RUNNING, JobStatus.DEFERRED: TaskResultStatus.READY, JobStatus.SCHEDULED: TaskResultStatus.READY, JobStatus.STOPPED: TaskResultStatus.FAILED, JobStatus.CANCELED: TaskResultStatus.FAILED, None: TaskResultStatus.READY, } class Job(BaseJob): def perform(self) -> Any: task_started.send( type(self.task_result.task.get_backend()), task_result=self.task_result ) return super().perform() def _execute(self) -> Any: """ Shim RQ's `Job` to call the underlying `Task` function. """ try: if self.func.takes_context: return self.func.call( TaskContext(task_result=self.task_result), *self.args, **self.kwargs ) return self.func.call(*self.args, **self.kwargs) finally: # Clear the task result cache, as it's changed now self.__dict__.pop("task_result", None) pass @property def func(self) -> Task: func = super().func if not isinstance(func, TASK_CLASSES): raise SuspiciousOperation( f"Task {self.id} does not point to a Task ({self.func_name})" ) return func # type: ignore[no-any-return] @cached_property def task_result(self) -> TaskResult: task: Task = self.func scheduled_job_registry = ScheduledJobRegistry( # type: ignore[no-untyped-call] queue=django_rq.get_queue(self.origin) ) if self.is_scheduled: run_after = scheduled_job_registry.get_scheduled_time(self) else: run_after = None task_result: TaskResult = TaskResult( task=task.using( priority=DEFAULT_TASK_PRIORITY, queue_name=self.origin, run_after=run_after, backend=self.meta["backend_name"], ), id=self.id, status=RQ_STATUS_TO_RESULT_STATUS[self.get_status()], enqueued_at=self.enqueued_at, started_at=self.started_at, last_attempted_at=self.started_at, finished_at=self.ended_at, args=list(self.args), kwargs=self.kwargs, backend=self.meta["backend_name"], errors=[], worker_ids=[], ) exception_classes = self.meta.get("_django_tasks_rq_exceptions", []).copy() rq_results = self.results() for rq_result in rq_results: task_result.worker_ids.append(rq_result.worker_name) if rq_result.type == Result.Type.FAILED: task_result.errors.append( TaskError( exception_class_path=( exception_classes.pop() if len(exception_classes) > 0 else get_module_path(Exception) ), traceback=rq_result.exc_string, # type: ignore[arg-type] ) ) if self.worker_name and task_result.status == TaskResultStatus.RUNNING: task_result.worker_ids.append(self.worker_name) if rq_results: object.__setattr__( task_result, "_return_value", rq_results[-1].return_value ) object.__setattr__( task_result, "last_attempted_at", rq_results[-1].created_at ) # If the return value couldn't be serialized, a specific string is saved instead. if task_result._return_value == UNSERIALIZABLE_RETURN_VALUE_PAYLOAD: # In these cases, the task should be marked as failed instead object.__setattr__(task_result, "status", TaskResultStatus.FAILED) task_result.errors.append( TaskError( exception_class_path=get_module_path(Exception), traceback=task_result._return_value, # The traceback isn't stored, so this is the best we can do ) ) object.__setattr__(task_result, "_return_value", None) return task_result def failed_callback( job: Job, connection: Redis | None, exception_class: type[Exception], exception_value: Exception, traceback: TracebackType, ) -> None: # Smuggle the exception class through meta job.meta.setdefault("_django_tasks_rq_exceptions", []).append( get_module_path(exception_class) ) job.save_meta() # type: ignore[no-untyped-call] task_result = job.task_result object.__setattr__(task_result, "status", TaskResultStatus.FAILED) task_finished.send(type(task_result.task.get_backend()), task_result=task_result) def success_callback(job: Job, connection: Redis | None, result: Any) -> None: task_result = job.task_result object.__setattr__(task_result, "status", TaskResultStatus.SUCCESSFUL) task_finished.send(type(task_result.task.get_backend()), task_result=task_result) class RQBackend(BaseTaskBackend): supports_async_task = True supports_get_result = True supports_defer = True def __init__(self, alias: str, params: dict) -> None: super().__init__(alias, params) if not self.queues: self.queues = set(django_rq.settings.QUEUES.keys()) def enqueue( self, task: Task[P, T], args: P.args, # type:ignore[valid-type] kwargs: P.kwargs, # type:ignore[valid-type] ) -> TaskResult[T]: self.validate_task(task) task_result: TaskResult[T] = TaskResult( task=task, id=get_random_id(), status=TaskResultStatus.READY, enqueued_at=None, started_at=None, last_attempted_at=None, finished_at=None, args=args, kwargs=kwargs, backend=self.alias, errors=[], worker_ids=[], ) queue = django_rq.get_queue(task.queue_name, job_class=Job) job = queue.create_job( task.module_path, args=args, kwargs=kwargs, job_id=task_result.id, status=JobStatus.SCHEDULED if task.run_after else JobStatus.QUEUED, on_failure=Callback(failed_callback), on_success=Callback(success_callback), meta={"backend_name": self.alias}, ) if task.run_after is None: job = queue.enqueue_job(job, at_front=task.priority == TASK_MAX_PRIORITY) else: job = queue.schedule_job(job, task.run_after) object.__setattr__(task_result, "enqueued_at", job.enqueued_at) task_enqueued.send(type(self), task_result=task_result) return task_result def _get_queues(self) -> list[django_rq.queues.DjangoRQ]: return django_rq.queues.get_queues(*self.queues, job_class=Job) # type: ignore[no-any-return,no-untyped-call] def _get_job(self, result_id: str) -> Job | None: try: return cast( Job, Job.fetch(result_id, connection=django_rq.get_connection()) ) except NoSuchJobError: return None def get_result(self, result_id: str) -> TaskResult: job = self._get_job(result_id) if job is None: raise TaskResultDoesNotExist(result_id) return job.task_result def check(self, **kwargs: Any) -> Iterable[messages.CheckMessage]: yield from super().check(**kwargs) backend_name = self.__class__.__name__ if not apps.is_installed("django_rq"): yield messages.Error( f"{backend_name} configured as django_tasks_rq backend, but django_rq app not installed", "Insert 'django_rq' in INSTALLED_APPS", ) for queue_name in self.queues: try: django_rq.get_queue(queue_name) except KeyError: yield messages.Error( f"{queue_name!r} is not configured for django-rq", f"Add {queue_name!r} to RQ_QUEUES", ) if tuple(map(int, (version("rq").split(".")))) < (2, 5, 0): yield messages.Error( "Only rq >= 2.5.0 is supported, found " + version("rq"), "Install a newer version of rq", ) django-tasks-rq-0.12.0/django_tasks_rq/compat.py000066400000000000000000000003561514141510400216050ustar00rootroot00000000000000try: from django.tasks.base import Task as DjangoTask except ImportError: DjangoTask = None from django_tasks.base import Task __all__ = ["TASK_CLASSES"] TASK_CLASSES = (Task, DjangoTask) if DjangoTask is not None else (Task,) django-tasks-rq-0.12.0/django_tasks_rq/py.typed000066400000000000000000000000001514141510400214310ustar00rootroot00000000000000django-tasks-rq-0.12.0/justfile000066400000000000000000000010531514141510400163420ustar00rootroot00000000000000# Recipes @default: just --list test *ARGS: python -m manage check python -m manage makemigrations --dry-run --check --noinput python -m coverage run --source=django_tasks_rq -m manage test --shuffle --noinput {{ ARGS }} python -m coverage report python -m coverage html format: python -m ruff check django_tasks_rq tests --fix python -m ruff format django_tasks_rq tests lint: python -m ruff check django_tasks_rq tests python -m ruff format django_tasks_rq tests --check python -m mypy django_tasks_rq tests django-tasks-rq-0.12.0/manage.py000066400000000000000000000003701514141510400163750ustar00rootroot00000000000000#!/usr/bin/env python import os import sys if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.settings") from django.core.management import execute_from_command_line execute_from_command_line(sys.argv) django-tasks-rq-0.12.0/pyproject.toml000066400000000000000000000052631514141510400175150ustar00rootroot00000000000000[build-system] build-backend = "setuptools.build_meta" requires = [ "setuptools", ] [project] name = "django-tasks-rq" description = "A Django Tasks backend which uses RQ as its underlying queue" authors = [ {name = "Jake Howard"}, ] version = "0.12.0" readme = "README.md" license = "BSD-3-Clause" license-files = ["LICENSE"] classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Environment :: Web Environment", "Framework :: Django", "Framework :: Django", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", "Framework :: Django :: 5.1", "Framework :: Django :: 5.2", "Framework :: Django :: 6.0", "Intended Audience :: Developers", "Operating System :: OS Independent", "Natural Language :: English", "Topic :: Internet :: WWW/HTTP", "Typing :: Typed" ] requires-python = ">=3.10" dependencies = [ "Django>=4.2", "typing_extensions", "django-stubs-ext", "django-rq", "rq>=2.5", "rq_scheduler", "django-tasks>=0.12.0", ] [project.urls] Source = "https://github.com/RealOrangeOne/django-tasks-rq" Issues = "https://github.com/RealOrangeOne/django-tasks-rq/issues" Changelog = "https://github.com/RealOrangeOne/django-tasks-rq/releases" [dependency-groups] dev = [ "ruff", "coverage", "django-stubs[compatible-mypy]", "fakeredis", ] [tool.ruff.lint] select = ["E", "F", "I", "W", "N", "B", "A", "C4", "T20", "DJ", "S", "UP"] ignore = ["E501", "DJ008"] [tool.ruff.lint.per-file-ignores] "tests/**.py" = ["S101", "S603", "S105"] [tool.mypy] plugins = ["mypy_django_plugin.main"] warn_unused_ignores = true warn_return_any = true show_error_codes = true strict_optional = true implicit_optional = true disallow_subclassing_any = true disallow_untyped_calls = true disallow_untyped_defs = true disallow_incomplete_defs = true disallow_untyped_decorators = true check_untyped_defs = true ignore_missing_imports = true [tool.django-stubs] django_settings_module = "tests.settings" [tool.coverage.run] branch = true [tool.coverage.report] show_missing = true ignore_errors = true exclude_also = [ # Don't complain if tests don't hit defensive assertion code: "raise AssertionError", "raise NotImplementedError", # Don't complain about abstract methods, they aren't run: "@(abc.)?abstractmethod", # Nor complain about type checking "if TYPE_CHECKING:", ] django-tasks-rq-0.12.0/tests/000077500000000000000000000000001514141510400157355ustar00rootroot00000000000000django-tasks-rq-0.12.0/tests/__init__.py000066400000000000000000000000001514141510400200340ustar00rootroot00000000000000django-tasks-rq-0.12.0/tests/settings.py000066400000000000000000000012041514141510400201440ustar00rootroot00000000000000import os import sys BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) IN_TEST = "IN_TEST" in os.environ or (len(sys.argv) > 1 and sys.argv[1] == "test") ALLOWED_HOSTS = ["*"] INSTALLED_APPS = [ "django_tasks_rq", "django_rq", "tests", ] SECRET_KEY = "abcde12345" USE_TZ = True if not IN_TEST: DEBUG = True TASKS = { "default": { "BACKEND": "django_tasks_rq.RQBackend", "QUEUES": ["default", "queue-1"], } } RQ_QUEUES = { "default": { "HOST": "localhost", "PORT": 6379, }, "queue-1": { "HOST": "localhost", "PORT": 6379, }, } django-tasks-rq-0.12.0/tests/tasks.py000066400000000000000000000027041514141510400174370ustar00rootroot00000000000000import time from typing import Any from django_tasks import TaskContext, task @task() def noop_task(*args: Any, **kwargs: Any) -> None: return None @task def noop_task_from_bare_decorator(*args: Any, **kwargs: Any) -> None: return None @task() async def noop_task_async(*args: Any, **kwargs: Any) -> None: return None @task() def calculate_meaning_of_life() -> int: return 42 @task() def failing_task_value_error() -> None: raise ValueError("This task failed due to ValueError") @task() def failing_task_system_exit() -> None: raise SystemExit("This task failed due to SystemExit") @task() def failing_task_keyboard_interrupt() -> None: raise KeyboardInterrupt("This task failed due to KeyboardInterrupt") @task() def complex_exception() -> None: raise ValueError(ValueError("This task failed")) @task() def complex_return_value() -> Any: # Return something which isn't JSON serializable nor picklable return lambda: True @task() def exit_task() -> None: exit(1) @task() def hang() -> None: """ Do nothing for 5 minutes """ time.sleep(300) @task() def sleep_for(seconds: float) -> None: time.sleep(seconds) @task(takes_context=True) def get_task_id(context: TaskContext) -> str: return context.task_result.id @task(takes_context=True) def test_context(context: TaskContext, attempt: int) -> None: assert isinstance(context, TaskContext) assert context.attempt == attempt django-tasks-rq-0.12.0/tests/tests.py000066400000000000000000000447531514141510400174660ustar00rootroot00000000000000import os import uuid from typing import cast from unittest.mock import patch import django_rq from asgiref.sync import async_to_sync from django import VERSION from django.core.exceptions import SuspiciousOperation from django.test import SimpleTestCase, override_settings from django_tasks import TaskResultStatus, default_task_backend, task_backends from django_tasks.base import Task from django_tasks.exceptions import InvalidTaskError, TaskResultDoesNotExist from fakeredis import FakeRedis, FakeStrictRedis from rq.defaults import UNSERIALIZABLE_RETURN_VALUE_PAYLOAD from rq.timeouts import TimerDeathPenalty from django_tasks_rq import compat from django_tasks_rq.backend import Job, RQBackend from tests import tasks as test_tasks # RQ # Configuration to pretend there is a Redis service available. # Set up the connection before RQ Django reads the settings. # The connection must be the same because in fakeredis connections # do not share the state. Therefore, we define a singleton object to reuse it. def get_fake_connection(config: dict, strict: bool) -> FakeRedis | FakeStrictRedis: redis_cls = FakeStrictRedis if strict else FakeRedis if "URL" in config: return redis_cls.from_url( config["URL"], db=config.get("DB"), ) return redis_cls( host=config["HOST"], port=config["PORT"], db=config.get("DB", 0), username=config.get("USERNAME", None), password=config.get("PASSWORD"), ) class RQBackendTestCase(SimpleTestCase): def setUp(self) -> None: super().setUp() try: from django_rq.connection_utils import get_redis_connection # noqa: F401 except ImportError: # django-rq < 3.2 fake_connection_patcher = patch( "django_rq.queues.get_redis_connection", get_fake_connection ) else: fake_connection_patcher = patch( "django_rq.connection_utils.get_redis_connection", get_fake_connection ) fake_connection_patcher.start() self.addCleanup(fake_connection_patcher.stop) django_rq.get_connection().flushall() def run_worker(self) -> None: from rq import SimpleWorker for queue in default_task_backend._get_queues(): # type: ignore[attr-defined] worker = SimpleWorker([queue], prepare_for_work=False, job_class=Job) # Use timer death penalty to support Windows worker.death_penalty_class = TimerDeathPenalty # HACK: Work around fakeredis not supporting `CLIENT LIST` worker.hostname = "example-hostname" worker.pid = os.getpid() with self.assertLogs("rq.worker"): worker.work(burst=True) def test_using_correct_backend(self) -> None: self.assertEqual(default_task_backend, task_backends["default"]) self.assertIsInstance(task_backends["default"], RQBackend) self.assertEqual(default_task_backend.alias, "default") self.assertEqual(default_task_backend.options, {}) def test_enqueue_task(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: with self.subTest(task): result = cast(Task, task).enqueue(1, two=3) self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 self.assertEqual(result.task, task) self.assertEqual(result.args, [1]) self.assertEqual(result.kwargs, {"two": 3}) self.assertEqual(result.attempts, 0) async def test_enqueue_task_async(self) -> None: for task in [test_tasks.noop_task, test_tasks.noop_task_async]: with self.subTest(task): result = await cast(Task, task).aenqueue() self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) with self.assertRaisesMessage(ValueError, "Task has not finished yet"): result.return_value # noqa:B018 self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) self.assertEqual(result.attempts, 0) def test_catches_exception(self) -> None: test_data = [ ( test_tasks.failing_task_value_error, # task function ValueError, # expected exception "This task failed due to ValueError", # expected message ), ( test_tasks.failing_task_system_exit, SystemExit, "This task failed due to SystemExit", ), ] for task, exception, message in test_data: with ( self.subTest(task), ): result = task.enqueue() with self.assertLogs("django_tasks", "DEBUG"): self.run_worker() result.refresh() # assert result self.assertEqual(result.status, TaskResultStatus.FAILED) with self.assertRaisesMessage(ValueError, "Task failed"): result.return_value # noqa: B018 self.assertTrue(result.is_finished) self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type, misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type, misc] self.assertEqual(result.errors[0].exception_class, exception) traceback = result.errors[0].traceback self.assertTrue( traceback and traceback.endswith(f"{exception.__name__}: {message}\n"), traceback, ) self.assertEqual(result.task, task) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) self.assertEqual(result.attempts, 1) def test_complex_exception(self) -> None: result = test_tasks.complex_exception.enqueue() with self.assertLogs("django_tasks", "DEBUG"): self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertIsNone(result._return_value) self.assertEqual(result.errors[0].exception_class, ValueError) self.assertIn( 'ValueError(ValueError("This task failed"))', result.errors[0].traceback ) self.assertEqual(result.task, test_tasks.complex_exception) self.assertEqual(result.args, []) self.assertEqual(result.kwargs, {}) self.assertEqual(result.attempts, 1) def test_complex_return_value(self) -> None: result = test_tasks.complex_return_value.enqueue() with self.assertLogs("django_tasks", "DEBUG"): self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.FAILED) self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertGreaterEqual(result.started_at, result.enqueued_at) # type:ignore[arg-type,misc] self.assertGreaterEqual(result.finished_at, result.started_at) # type:ignore[arg-type,misc] self.assertIsNone(result._return_value) self.assertEqual(result.errors[0].exception_class, Exception) self.assertIn(UNSERIALIZABLE_RETURN_VALUE_PAYLOAD, result.errors[0].traceback) def test_get_result(self) -> None: result = default_task_backend.enqueue(test_tasks.noop_task, [], {}) new_result = default_task_backend.get_result(result.id) self.assertEqual(result, new_result) async def test_get_result_async(self) -> None: result = await default_task_backend.aenqueue(test_tasks.noop_task, [], {}) new_result = await default_task_backend.aget_result(result.id) self.assertEqual(result, new_result) def test_refresh_result(self) -> None: result = default_task_backend.enqueue( test_tasks.calculate_meaning_of_life, (), {} ) self.run_worker() self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) result.refresh() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) self.assertEqual(result.attempts, 1) def test_refresh_result_async(self) -> None: result = async_to_sync(default_task_backend.aenqueue)( test_tasks.calculate_meaning_of_life, (), {} ) self.run_worker() self.assertEqual(result.status, TaskResultStatus.READY) self.assertFalse(result.is_finished) self.assertIsNone(result.started_at) self.assertIsNone(result.last_attempted_at) self.assertIsNone(result.finished_at) self.assertEqual(result.attempts, 0) async_to_sync(result.arefresh)() self.assertIsNotNone(result.started_at) self.assertIsNotNone(result.last_attempted_at) self.assertIsNotNone(result.finished_at) self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) self.assertTrue(result.is_finished) self.assertEqual(result.return_value, 42) self.assertEqual(result.attempts, 1) def test_get_missing_result(self) -> None: with self.assertRaises(TaskResultDoesNotExist): default_task_backend.get_result(str(uuid.uuid4())) async def test_async_get_missing_result(self) -> None: with self.assertRaises(TaskResultDoesNotExist): await default_task_backend.aget_result(str(uuid.uuid4())) def test_invalid_uuid(self) -> None: with self.assertRaises(TaskResultDoesNotExist): default_task_backend.get_result("123") async def test_async_invalid_uuid(self) -> None: with self.assertRaises(TaskResultDoesNotExist): await default_task_backend.aget_result("123") def test_invalid_task_path(self) -> None: job = django_rq.get_queue("default", job_class=Job).enqueue_call( # type: ignore[no-untyped-call] "subprocess.check_output", args=["exit", "1"] ) with self.assertRaisesMessage( SuspiciousOperation, f"Task {job.id} does not point to a Task (subprocess.check_output)", ): default_task_backend.get_result(job.id) def test_check(self) -> None: errors = list(default_task_backend.check()) self.assertEqual(len(errors), 0, errors) @override_settings(INSTALLED_APPS=[]) def test_rq_app_missing(self) -> None: errors = list(default_task_backend.check()) self.assertEqual(len(errors), 1) self.assertIn("django_rq", errors[0].hint) # type:ignore[arg-type] def test_enqueue_logs(self) -> None: with self.assertLogs("django_tasks", level="DEBUG") as captured_logs: result = test_tasks.noop_task.enqueue() self.assertEqual(len(captured_logs.output), 1) self.assertIn("enqueued", captured_logs.output[0]) self.assertIn(result.id, captured_logs.output[0]) def test_started_finished_logs(self) -> None: result = test_tasks.noop_task.enqueue() with self.assertLogs("django_tasks", level="DEBUG") as captured_logs: self.run_worker() self.assertEqual(len(captured_logs.output), 2) self.assertIn("state=RUNNING", captured_logs.output[0]) self.assertIn(result.id, captured_logs.output[0]) self.assertIn("state=SUCCESSFUL", captured_logs.output[1]) self.assertIn(result.id, captured_logs.output[1]) def test_failed_logs(self) -> None: result = test_tasks.failing_task_value_error.enqueue() with self.assertLogs("django_tasks", level="DEBUG") as captured_logs: self.run_worker() self.assertEqual(len(captured_logs.output), 2) self.assertIn("state=RUNNING", captured_logs.output[0]) self.assertIn(result.id, captured_logs.output[0]) self.assertIn("state=FAILED", captured_logs.output[1]) self.assertIn(result.id, captured_logs.output[1]) def test_queue_isolation(self) -> None: default_task = test_tasks.noop_task.enqueue() other_task = test_tasks.noop_task.using(queue_name="queue-1").enqueue() default_task_backend.get_result(default_task.id) default_task_backend.get_result(other_task.id) self.assertEqual(django_rq.get_queue("default").job_ids, [default_task.id]) self.assertEqual(django_rq.get_queue("queue-1").job_ids, [other_task.id]) @override_settings( TASKS={"default": {"BACKEND": "django_tasks_rq.RQBackend", "QUEUES": []}} ) def test_uses_rq_queues_for_queue_names(self) -> None: self.assertEqual(default_task_backend.queues, {"default", "queue-1"}) @override_settings( TASKS={ "default": { "BACKEND": "django_tasks_rq.RQBackend", "QUEUES": ["queue-2"], } } ) def test_unknown_queue_name(self) -> None: errors = list(default_task_backend.check()) self.assertEqual(len(errors), 1) self.assertIn("Add 'queue-2' to RQ_QUEUES", errors[0].hint) # type:ignore[arg-type] def test_takes_context(self) -> None: result = test_tasks.get_task_id.enqueue() self.run_worker() result.refresh() self.assertEqual(result.return_value, result.id) def test_context(self) -> None: result = test_tasks.test_context.enqueue(1) self.run_worker() result.refresh() self.assertEqual(result.status, TaskResultStatus.SUCCESSFUL) def test_exception_classes_pop_empty_list_bug(self) -> None: """Test for IndexError: pop from empty list bug in task_result property This test creates a scenario where there are failed results but no exception classes in the job meta, which should be handled gracefully but currently causes an IndexError. When the bug is present, this test will fail with IndexError: pop from empty list. """ # Create and run a failing task normally first result = test_tasks.failing_task_value_error.enqueue() with self.assertLogs("django_tasks", level="DEBUG"): self.run_worker() # Get the underlying RQ job job = cast(RQBackend, default_task_backend)._get_job(result.id) self.assertIsNotNone(job) assert job is not None # At this point, the job should have failed and have: # # - 1 failed result in job.results() # - 1 exception class in job.meta["_django_tasks_rq_exceptions"] # # Now simulate the bug scenario by removing the exceptions key from meta. # This creates a scenario where there are failed results but no # exception classes. job.meta.pop("_django_tasks_rq_exceptions", None) job.save_meta() # type: ignore[no-untyped-call] # Clear the cached task_result to force re-computation job.__dict__.pop("task_result", None) # This should work without throwing an IndexError # When the bug is present, this will fail with IndexError: pop from empty list task_result = job.task_result # Verify the task_result is properly constructed despite missing exception classes self.assertEqual(task_result.status, TaskResultStatus.FAILED) self.assertEqual(task_result.task, test_tasks.failing_task_value_error) self.assertIsInstance(task_result.errors, list) self.assertEqual( task_result.errors[0].exception_class_path, "builtins.Exception" ) def test_validate_on_enqueue(self) -> None: with override_settings( TASKS={ "default": { "BACKEND": "django_tasks_rq.RQBackend", "QUEUES": ["unknown_queue"], } } ): task_with_custom_queue_name = test_tasks.noop_task.using( queue_name="unknown_queue" ) with self.assertRaisesMessage( InvalidTaskError, "Queue 'unknown_queue' is not valid for backend" ): task_with_custom_queue_name.enqueue() async def test_validate_on_aenqueue(self) -> None: with override_settings( TASKS={ "default": { "BACKEND": "django_tasks_rq.RQBackend", "QUEUES": ["unknown_queue"], } } ): task_with_custom_queue_name = test_tasks.noop_task.using( queue_name="unknown_queue" ) with self.assertRaisesMessage( InvalidTaskError, "Queue 'unknown_queue' is not valid for backend" ): await task_with_custom_queue_name.aenqueue() class CompatTestCase(SimpleTestCase): def test_compat_has_django_task(self) -> None: self.assertIn(Task, compat.TASK_CLASSES) if VERSION >= (6, 0): from django.tasks.base import Task as DjangoTask self.assertIn(DjangoTask, compat.TASK_CLASSES)