first commit

.venv/lib/python3.12/site-packages/django/tasks/__init__.py

@@ -0,0 +1,45 @@
+from django.utils.connection import BaseConnectionHandler, ConnectionProxy
+from django.utils.module_loading import import_string
+
+from . import checks, signals  # NOQA
+from .base import (
+    DEFAULT_TASK_BACKEND_ALIAS,
+    DEFAULT_TASK_QUEUE_NAME,
+    Task,
+    TaskContext,
+    TaskResult,
+    TaskResultStatus,
+    task,
+)
+from .exceptions import InvalidTaskBackend
+
+__all__ = [
+    "DEFAULT_TASK_BACKEND_ALIAS",
+    "DEFAULT_TASK_QUEUE_NAME",
+    "default_task_backend",
+    "task",
+    "task_backends",
+    "Task",
+    "TaskContext",
+    "TaskResult",
+    "TaskResultStatus",
+]
+
+
+class TaskBackendHandler(BaseConnectionHandler):
+    settings_name = "TASKS"
+    exception_class = InvalidTaskBackend
+
+    def create_connection(self, alias):
+        params = self.settings[alias]
+        backend = params["BACKEND"]
+        try:
+            backend_cls = import_string(backend)
+        except ImportError as e:
+            raise InvalidTaskBackend(f"Could not find backend '{backend}': {e}") from e
+        return backend_cls(alias=alias, params=params)
+
+
+task_backends = TaskBackendHandler()
+
+default_task_backend = ConnectionProxy(task_backends, DEFAULT_TASK_BACKEND_ALIAS)

.venv/lib/python3.12/site-packages/django/tasks/backends/base.py

@@ -0,0 +1,112 @@
+from abc import ABCMeta, abstractmethod
+from inspect import iscoroutinefunction
+
+from asgiref.sync import sync_to_async
+
+from django.conf import settings
+from django.tasks import DEFAULT_TASK_QUEUE_NAME
+from django.tasks.base import (
+    DEFAULT_TASK_PRIORITY,
+    TASK_MAX_PRIORITY,
+    TASK_MIN_PRIORITY,
+    Task,
+)
+from django.tasks.exceptions import InvalidTask
+from django.utils import timezone
+from django.utils.inspect import get_func_args, is_module_level_function
+
+
+class BaseTaskBackend(metaclass=ABCMeta):
+    task_class = Task
+
+    # Does the backend support Tasks to be enqueued with the run_after
+    # attribute?
+    supports_defer = False
+
+    # Does the backend support coroutines to be enqueued?
+    supports_async_task = False
+
+    # Does the backend support results being retrieved (from any
+    # thread/process)?
+    supports_get_result = False
+
+    # Does the backend support executing Tasks in a given
+    # priority order?
+    supports_priority = False
+
+    def __init__(self, alias, params):
+        self.alias = alias
+        self.queues = set(params.get("QUEUES", [DEFAULT_TASK_QUEUE_NAME]))
+        self.options = params.get("OPTIONS", {})
+
+    def validate_task(self, task):
+        """
+        Determine whether the provided Task can be executed by the backend.
+        """
+        if not is_module_level_function(task.func):
+            raise InvalidTask("Task function must be defined at a module level.")
+
+        if not self.supports_async_task and iscoroutinefunction(task.func):
+            raise InvalidTask("Backend does not support async Tasks.")
+
+        task_func_args = get_func_args(task.func)
+        if task.takes_context and (
+            not task_func_args or task_func_args[0] != "context"
+        ):
+            raise InvalidTask(
+                "Task takes context but does not have a first argument of 'context'."
+            )
+
+        if not self.supports_priority and task.priority != DEFAULT_TASK_PRIORITY:
+            raise InvalidTask("Backend does not support setting priority of tasks.")
+        if (
+            task.priority < TASK_MIN_PRIORITY
+            or task.priority > TASK_MAX_PRIORITY
+            or int(task.priority) != task.priority
+        ):
+            raise InvalidTask(
+                f"priority must be a whole number between {TASK_MIN_PRIORITY} and "
+                f"{TASK_MAX_PRIORITY}."
+            )
+
+        if not self.supports_defer and task.run_after is not None:
+            raise InvalidTask("Backend does not support run_after.")
+
+        if (
+            settings.USE_TZ
+            and task.run_after is not None
+            and not timezone.is_aware(task.run_after)
+        ):
+            raise InvalidTask("run_after must be an aware datetime.")
+
+        if self.queues and task.queue_name not in self.queues:
+            raise InvalidTask(f"Queue '{task.queue_name}' is not valid for backend.")
+
+    @abstractmethod
+    def enqueue(self, task, args, kwargs):
+        """Queue up a task to be executed."""
+
+    async def aenqueue(self, task, args, kwargs):
+        """Queue up a task function (or coroutine) to be executed."""
+        return await sync_to_async(self.enqueue, thread_sensitive=True)(
+            task=task, args=args, kwargs=kwargs
+        )
+
+    def get_result(self, result_id):
+        """
+        Retrieve a task result by id.
+
+        Raise TaskResultDoesNotExist if such result does not exist.
+        """
+        raise NotImplementedError(
+            "This backend does not support retrieving or refreshing results."
+        )
+
+    async def aget_result(self, result_id):
+        """See get_result()."""
+        return await sync_to_async(self.get_result, thread_sensitive=True)(
+            result_id=result_id
+        )
+
+    def check(self, **kwargs):
+        return []

.venv/lib/python3.12/site-packages/django/tasks/backends/dummy.py

@@ -0,0 +1,64 @@
+from copy import deepcopy
+
+from django.tasks.base import TaskResult, TaskResultStatus
+from django.tasks.exceptions import TaskResultDoesNotExist
+from django.tasks.signals import task_enqueued
+from django.utils import timezone
+from django.utils.crypto import get_random_string
+
+from .base import BaseTaskBackend
+
+
+class DummyBackend(BaseTaskBackend):
+    supports_defer = True
+    supports_async_task = True
+    supports_priority = True
+
+    def __init__(self, alias, params):
+        super().__init__(alias, params)
+        self.results = []
+
+    def _store_result(self, result):
+        object.__setattr__(result, "enqueued_at", timezone.now())
+        self.results.append(result)
+        task_enqueued.send(type(self), task_result=result)
+
+    def enqueue(self, task, args, kwargs):
+        self.validate_task(task)
+
+        result = TaskResult(
+            task=task,
+            id=get_random_string(32),
+            status=TaskResultStatus.READY,
+            enqueued_at=None,
+            started_at=None,
+            last_attempted_at=None,
+            finished_at=None,
+            args=args,
+            kwargs=kwargs,
+            backend=self.alias,
+            errors=[],
+            worker_ids=[],
+        )
+
+        self._store_result(result)
+
+        # Copy the task to prevent mutation issues.
+        return deepcopy(result)
+
+    def get_result(self, result_id):
+        # Results are only scoped to the current thread, hence
+        # supports_get_result is False.
+        try:
+            return next(result for result in self.results if result.id == result_id)
+        except StopIteration:
+            raise TaskResultDoesNotExist(result_id) from None
+
+    async def aget_result(self, result_id):
+        try:
+            return next(result for result in self.results if result.id == result_id)
+        except StopIteration:
+            raise TaskResultDoesNotExist(result_id) from None
+
+    def clear(self):
+        self.results.clear()

.venv/lib/python3.12/site-packages/django/tasks/backends/immediate.py

@@ -0,0 +1,95 @@
+import logging
+from traceback import format_exception
+
+from django.tasks.base import TaskContext, TaskError, TaskResult, TaskResultStatus
+from django.tasks.signals import task_enqueued, task_finished, task_started
+from django.utils import timezone
+from django.utils.crypto import get_random_string
+from django.utils.json import normalize_json
+
+from .base import BaseTaskBackend
+
+logger = logging.getLogger(__name__)
+
+
+class ImmediateBackend(BaseTaskBackend):
+    supports_async_task = True
+    supports_priority = True
+
+    def __init__(self, alias, params):
+        super().__init__(alias, params)
+        self.worker_id = get_random_string(32)
+
+    def _execute_task(self, task_result):
+        """
+        Execute the Task for the given TaskResult, mutating it with the
+        outcome.
+        """
+        object.__setattr__(task_result, "enqueued_at", timezone.now())
+        task_enqueued.send(type(self), task_result=task_result)
+
+        task = task_result.task
+        task_start_time = timezone.now()
+        object.__setattr__(task_result, "status", TaskResultStatus.RUNNING)
+        object.__setattr__(task_result, "started_at", task_start_time)
+        object.__setattr__(task_result, "last_attempted_at", task_start_time)
+        task_result.worker_ids.append(self.worker_id)
+        task_started.send(sender=type(self), task_result=task_result)
+
+        try:
+            if task.takes_context:
+                raw_return_value = task.call(
+                    TaskContext(task_result=task_result),
+                    *task_result.args,
+                    **task_result.kwargs,
+                )
+            else:
+                raw_return_value = task.call(*task_result.args, **task_result.kwargs)
+
+            object.__setattr__(
+                task_result,
+                "_return_value",
+                normalize_json(raw_return_value),
+            )
+        except KeyboardInterrupt:
+            # If the user tried to terminate, let them
+            raise
+        except BaseException as e:
+            object.__setattr__(task_result, "finished_at", timezone.now())
+            exception_type = type(e)
+            task_result.errors.append(
+                TaskError(
+                    exception_class_path=(
+                        f"{exception_type.__module__}.{exception_type.__qualname__}"
+                    ),
+                    traceback="".join(format_exception(e)),
+                )
+            )
+            object.__setattr__(task_result, "status", TaskResultStatus.FAILED)
+            task_finished.send(type(self), task_result=task_result)
+        else:
+            object.__setattr__(task_result, "finished_at", timezone.now())
+            object.__setattr__(task_result, "status", TaskResultStatus.SUCCESSFUL)
+            task_finished.send(type(self), task_result=task_result)
+
+    def enqueue(self, task, args, kwargs):
+        self.validate_task(task)
+
+        task_result = TaskResult(
+            task=task,
+            id=get_random_string(32),
+            status=TaskResultStatus.READY,
+            enqueued_at=None,
+            started_at=None,
+            last_attempted_at=None,
+            finished_at=None,
+            args=args,
+            kwargs=kwargs,
+            backend=self.alias,
+            errors=[],
+            worker_ids=[],
+        )
+
+        self._execute_task(task_result)
+
+        return task_result

.venv/lib/python3.12/site-packages/django/tasks/base.py

@@ -0,0 +1,247 @@
+from dataclasses import dataclass, field, replace
+from datetime import datetime
+from inspect import isclass, iscoroutinefunction
+from typing import Any, Callable, Dict, Optional
+
+from asgiref.sync import async_to_sync, sync_to_async
+
+from django.db.models.enums import TextChoices
+from django.utils.json import normalize_json
+from django.utils.module_loading import import_string
+from django.utils.translation import pgettext_lazy
+
+from .exceptions import TaskResultMismatch
+
+DEFAULT_TASK_BACKEND_ALIAS = "default"
+DEFAULT_TASK_PRIORITY = 0
+DEFAULT_TASK_QUEUE_NAME = "default"
+TASK_MAX_PRIORITY = 100
+TASK_MIN_PRIORITY = -100
+TASK_REFRESH_ATTRS = {
+    "errors",
+    "_return_value",
+    "finished_at",
+    "started_at",
+    "last_attempted_at",
+    "status",
+    "enqueued_at",
+    "worker_ids",
+}
+
+
+class TaskResultStatus(TextChoices):
+    # The Task has just been enqueued, or is ready to be executed again.
+    READY = ("READY", pgettext_lazy("Task", "Ready"))
+    # The Task is currently running.
+    RUNNING = ("RUNNING", pgettext_lazy("Task", "Running"))
+    # The Task raised an exception during execution, or was unable to start.
+    FAILED = ("FAILED", pgettext_lazy("Task", "Failed"))
+    # The Task has finished running successfully.
+    SUCCESSFUL = ("SUCCESSFUL", pgettext_lazy("Task", "Successful"))
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Task:
+    priority: int
+    func: Callable  # The Task function.
+    backend: str
+    queue_name: str
+    run_after: Optional[datetime]  # The earliest this Task will run.
+
+    # Whether the Task receives the Task context when executed.
+    takes_context: bool = False
+
+    def __post_init__(self):
+        self.get_backend().validate_task(self)
+
+    @property
+    def name(self):
+        return self.func.__name__
+
+    def using(
+        self,
+        *,
+        priority=None,
+        queue_name=None,
+        run_after=None,
+        backend=None,
+    ):
+        """Create a new Task with modified defaults."""
+
+        changes = {}
+        if priority is not None:
+            changes["priority"] = priority
+        if queue_name is not None:
+            changes["queue_name"] = queue_name
+        if run_after is not None:
+            changes["run_after"] = run_after
+        if backend is not None:
+            changes["backend"] = backend
+        return replace(self, **changes)
+
+    def enqueue(self, *args, **kwargs):
+        """Queue up the Task to be executed."""
+        return self.get_backend().enqueue(self, args, kwargs)
+
+    async def aenqueue(self, *args, **kwargs):
+        """Queue up the Task to be executed."""
+        return await self.get_backend().aenqueue(self, args, kwargs)
+
+    def get_result(self, result_id):
+        """
+        Retrieve a task result by id.
+
+        Raise TaskResultDoesNotExist if such result does not exist, or raise
+        TaskResultMismatch if the result exists but belongs to another Task.
+        """
+        result = self.get_backend().get_result(result_id)
+        if result.task.func != self.func:
+            raise TaskResultMismatch(
+                f"Task does not match (received {result.task.module_path!r})"
+            )
+        return result
+
+    async def aget_result(self, result_id):
+        """See get_result()."""
+        result = await self.get_backend().aget_result(result_id)
+        if result.task.func != self.func:
+            raise TaskResultMismatch(
+                f"Task does not match (received {result.task.module_path!r})"
+            )
+        return result
+
+    def call(self, *args, **kwargs):
+        if iscoroutinefunction(self.func):
+            return async_to_sync(self.func)(*args, **kwargs)
+        return self.func(*args, **kwargs)
+
+    async def acall(self, *args, **kwargs):
+        if iscoroutinefunction(self.func):
+            return await self.func(*args, **kwargs)
+        return await sync_to_async(self.func)(*args, **kwargs)
+
+    def get_backend(self):
+        from . import task_backends
+
+        return task_backends[self.backend]
+
+    @property
+    def module_path(self):
+        return f"{self.func.__module__}.{self.func.__qualname__}"
+
+
+def task(
+    function=None,
+    *,
+    priority=DEFAULT_TASK_PRIORITY,
+    queue_name=DEFAULT_TASK_QUEUE_NAME,
+    backend=DEFAULT_TASK_BACKEND_ALIAS,
+    takes_context=False,
+):
+    from . import task_backends
+
+    def wrapper(f):
+        return task_backends[backend].task_class(
+            priority=priority,
+            func=f,
+            queue_name=queue_name,
+            backend=backend,
+            takes_context=takes_context,
+            run_after=None,
+        )
+
+    if function:
+        return wrapper(function)
+    return wrapper
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TaskError:
+    exception_class_path: str
+    traceback: str
+
+    @property
+    def exception_class(self):
+        # Lazy resolve the exception class.
+        exception_class = import_string(self.exception_class_path)
+
+        if not isclass(exception_class) or not issubclass(
+            exception_class, BaseException
+        ):
+            raise ValueError(
+                f"{self.exception_class_path!r} does not reference a valid exception."
+            )
+        return exception_class
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TaskResult:
+    task: Task
+
+    id: str  # Unique identifier for the task result.
+    status: TaskResultStatus
+    enqueued_at: Optional[datetime]  # Time the task was enqueued.
+    started_at: Optional[datetime]  # Time the task was started.
+    finished_at: Optional[datetime]  # Time the task was finished.
+
+    # Time the task was last attempted to be run.
+    last_attempted_at: Optional[datetime]
+
+    args: list  # Arguments to pass to the task function.
+    kwargs: Dict[str, Any]  # Keyword arguments to pass to the task function.
+    backend: str
+    errors: list[TaskError]  # Errors raised when running the task.
+    worker_ids: list[str]  # Workers which have processed the task.
+
+    _return_value: Optional[Any] = field(init=False, default=None)
+
+    def __post_init__(self):
+        object.__setattr__(self, "args", normalize_json(self.args))
+        object.__setattr__(self, "kwargs", normalize_json(self.kwargs))
+
+    @property
+    def return_value(self):
+        """
+        The return value of the task.
+
+        If the task didn't succeed, an exception is raised.
+        This is to distinguish against the task returning None.
+        """
+        if self.status == TaskResultStatus.SUCCESSFUL:
+            return self._return_value
+        elif self.status == TaskResultStatus.FAILED:
+            raise ValueError("Task failed")
+        else:
+            raise ValueError("Task has not finished yet")
+
+    @property
+    def is_finished(self):
+        return self.status in {TaskResultStatus.FAILED, TaskResultStatus.SUCCESSFUL}
+
+    @property
+    def attempts(self):
+        return len(self.worker_ids)
+
+    def refresh(self):
+        """Reload the cached task data from the task store."""
+        refreshed_task = self.task.get_backend().get_result(self.id)
+
+        for attr in TASK_REFRESH_ATTRS:
+            object.__setattr__(self, attr, getattr(refreshed_task, attr))
+
+    async def arefresh(self):
+        """
+        Reload the cached task data from the task store
+        """
+        refreshed_task = await self.task.get_backend().aget_result(self.id)
+        for attr in TASK_REFRESH_ATTRS:
+            object.__setattr__(self, attr, getattr(refreshed_task, attr))
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TaskContext:
+    task_result: TaskResult
+
+    @property
+    def attempt(self):
+        return self.task_result.attempts

.venv/lib/python3.12/site-packages/django/tasks/checks.py

@@ -0,0 +1,11 @@
+from django.core import checks
+
+
+@checks.register
+def check_tasks(app_configs=None, **kwargs):
+    """Checks all registered Task backends."""
+
+    from . import task_backends
+
+    for backend in task_backends.all():
+        yield from backend.check()

.venv/lib/python3.12/site-packages/django/tasks/exceptions.py

@@ -0,0 +1,21 @@
+from django.core.exceptions import ImproperlyConfigured
+
+
+class TaskException(Exception):
+    """Base class for task-related exceptions. Do not raise directly."""
+
+
+class InvalidTask(TaskException):
+    """The provided Task is invalid."""
+
+
+class InvalidTaskBackend(ImproperlyConfigured):
+    """The provided Task backend is invalid."""
+
+
+class TaskResultDoesNotExist(TaskException):
+    """The requested TaskResult does not exist."""
+
+
+class TaskResultMismatch(TaskException):
+    """The requested TaskResult is invalid."""

.venv/lib/python3.12/site-packages/django/tasks/signals.py

@@ -0,0 +1,64 @@
+import logging
+import sys
+
+from asgiref.local import Local
+
+from django.core.signals import setting_changed
+from django.dispatch import Signal, receiver
+
+from .base import TaskResultStatus
+
+logger = logging.getLogger("django.tasks")
+
+task_enqueued = Signal()
+task_finished = Signal()
+task_started = Signal()
+
+
+@receiver(setting_changed)
+def clear_tasks_handlers(*, setting, **kwargs):
+    """Reset the connection handler whenever the settings change."""
+    if setting == "TASKS":
+        from . import task_backends
+
+        task_backends._settings = task_backends.settings = (
+            task_backends.configure_settings(None)
+        )
+        task_backends._connections = Local()
+
+
+@receiver(task_enqueued)
+def log_task_enqueued(sender, task_result, **kwargs):
+    logger.debug(
+        "Task id=%s path=%s enqueued backend=%s",
+        task_result.id,
+        task_result.task.module_path,
+        task_result.backend,
+    )
+
+
+@receiver(task_started)
+def log_task_started(sender, task_result, **kwargs):
+    logger.info(
+        "Task id=%s path=%s state=%s",
+        task_result.id,
+        task_result.task.module_path,
+        task_result.status,
+    )
+
+
+@receiver(task_finished)
+def log_task_finished(sender, task_result, **kwargs):
+    logger.log(
+        (
+            logging.ERROR
+            if task_result.status == TaskResultStatus.FAILED
+            else logging.INFO
+        ),
+        "Task id=%s path=%s state=%s",
+        task_result.id,
+        task_result.task.module_path,
+        task_result.status,
+        # Signal is sent inside exception handlers, so exc_info() is available.
+        exc_info=sys.exc_info(),
+    )