Coverage for src/robot_dispatch/services/dispatcher.py: 99%
105 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-07 17:18 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-07 17:18 +0000
1from __future__ import annotations
3import uuid
4from dataclasses import dataclass
5from datetime import datetime, timedelta
7from robot_dispatch.config import Settings
8from robot_dispatch.db.models import Robot, Task, utcnow
9from robot_dispatch.domain.enums import RobotStatus, TaskEventKind, TaskStatus
10from robot_dispatch.domain.policies import RetryPolicy, should_retry
11from robot_dispatch.domain.state_machine import ensure_transition
12from robot_dispatch.logging_setup import get_logger
13from robot_dispatch.messaging.messages import (
14 COMMANDS_EXCHANGE,
15 CommandMessage,
16 command_routing_key,
17)
18from robot_dispatch.repositories.protocols import UnitOfWork, UnitOfWorkFactory
20logger = get_logger("dispatcher")
23@dataclass(frozen=True)
24class DispatchOutcome:
25 dispatched: bool
26 reason: str
27 task_id: uuid.UUID | None = None
28 robot_id: uuid.UUID | None = None
29 attempt: int | None = None
32class DispatcherService:
33 """Assigns queued tasks to idle robots.
35 Reliability rules:
36 * candidates are locked with `FOR UPDATE SKIP LOCKED`, so N replicas never
37 grab the same task;
38 * the assignment, the robot state change and the outgoing command (outbox)
39 are committed in one transaction: a crash cannot leave a task assigned to
40 a robot that never receives a command;
41 * every attempt gets a fresh `command_id` and a lease, so a silent robot is
42 detected by the reaper instead of stalling the task forever.
43 """
45 def __init__(
46 self,
47 uow_factory: UnitOfWorkFactory,
48 settings: Settings,
49 *,
50 clock=utcnow,
51 ) -> None:
52 self._uow_factory = uow_factory
53 self._settings = settings
54 self._clock = clock
56 async def dispatch_once(self) -> DispatchOutcome:
57 now = self._clock()
58 async with self._uow_factory() as uow:
59 candidates = await uow.tasks.lock_ready(now=now, limit=self._settings.dispatcher_batch_size)
60 if not candidates:
61 return DispatchOutcome(dispatched=False, reason="no_ready_tasks")
63 for task in candidates:
64 robot = await uow.robots.find_idle_for(
65 task_type=task.type,
66 now=now,
67 offline_after=self._settings.robot_offline_after_seconds,
68 )
69 if robot is None:
70 continue
71 outcome = await self._assign(uow, task, robot, now)
72 await uow.commit()
73 logger.info(
74 "task dispatched",
75 task_id=str(task.id),
76 robot=robot.name,
77 attempt=task.attempts,
78 )
79 return outcome
81 return DispatchOutcome(dispatched=False, reason="no_idle_robot")
83 async def _assign(
84 self, uow: UnitOfWork, task: Task, robot: Robot, now: datetime
85 ) -> DispatchOutcome:
86 previous = TaskStatus(task.status)
87 attempt = task.attempts + 1
88 command = CommandMessage(
89 command_id=uuid.uuid4(),
90 task_id=task.id,
91 task_type=task.type,
92 robot_id=robot.id,
93 namespace=robot.namespace,
94 attempt=attempt,
95 timeout_seconds=task.timeout_seconds,
96 payload=task.payload or {},
97 issued_at=now,
98 )
100 task.status = ensure_transition(previous, TaskStatus.ASSIGNED).value
101 task.attempts = attempt
102 task.robot_id = robot.id
103 task.command_id = command.command_id
104 task.lease_expires_at = now + timedelta(seconds=task.timeout_seconds)
105 task.available_at = now
106 task.started_at = task.started_at or now
107 task.updated_at = now
108 task.version += 1
110 robot.status = RobotStatus.BUSY.value
111 robot.current_task_id = task.id
112 robot.updated_at = now
114 await uow.events.add(
115 _event(
116 task,
117 TaskEventKind.DISPATCHED,
118 from_status=previous.value,
119 to_status=task.status,
120 robot_id=robot.id,
121 message=f"assigned to {robot.name}, attempt {attempt}/{task.max_attempts}",
122 payload={"command_id": str(command.command_id), "attempt": attempt},
123 )
124 )
125 await uow.outbox.add(
126 _command_outbox_message(command, timeout_seconds=task.timeout_seconds)
127 )
128 return DispatchOutcome(
129 dispatched=True,
130 reason="assigned",
131 task_id=task.id,
132 robot_id=robot.id,
133 attempt=attempt,
134 )
137class LeaseReaper:
138 """Owns every "robot went silent" case.
140 A task whose lease expired is re-queued with exponential backoff until
141 `max_attempts`, then failed. The robot is always released, so a dead robot
142 never blocks a capability slot.
143 """
145 def __init__(
146 self,
147 uow_factory: UnitOfWorkFactory,
148 settings: Settings,
149 *,
150 retry_policy: RetryPolicy | None = None,
151 clock=utcnow,
152 ) -> None:
153 self._uow_factory = uow_factory
154 self._settings = settings
155 self._policy = retry_policy or RetryPolicy(
156 base_delay_seconds=settings.retry_base_delay_seconds,
157 max_delay_seconds=settings.retry_max_delay_seconds,
158 jitter_ratio=settings.retry_jitter_ratio,
159 )
160 self._clock = clock
162 async def reap_once(self, *, limit: int = 100) -> int:
163 now = self._clock()
164 async with self._uow_factory() as uow:
165 expired = await uow.tasks.lock_expired_leases(now=now, limit=limit)
166 for task in expired:
167 await self._recover(uow, task, now)
168 await uow.commit()
169 if expired:
170 logger.warning("expired leases recovered", count=len(expired))
171 return len(expired)
173 async def _recover(self, uow: UnitOfWork, task: Task, now: datetime) -> None:
174 robot = await uow.robots.get(task.robot_id) if task.robot_id else None
175 if robot is not None:
176 robot.status = RobotStatus.IDLE.value
177 robot.current_task_id = None
178 robot.updated_at = now
180 previous = TaskStatus(task.status)
181 task.lease_expires_at = None
182 task.updated_at = now
184 if previous in {TaskStatus.PENDING, TaskStatus.RETRYING}:
185 return
187 if should_retry(task.attempts, task.max_attempts):
188 delay = self._policy.delay_seconds(task.attempts)
189 task.status = ensure_transition(previous, TaskStatus.RETRYING).value
190 task.available_at = now + timedelta(seconds=delay)
191 task.last_error = "lease expired: robot did not report progress"
192 task.version += 1
193 await uow.events.add(
194 _event(
195 task,
196 TaskEventKind.LEASE_EXPIRED,
197 from_status=previous.value,
198 to_status=task.status,
199 robot_id=task.robot_id,
200 message=f"retry in {delay:.1f}s",
201 payload={"attempt": task.attempts, "retry_in_seconds": round(delay, 3)},
202 )
203 )
204 return
206 task.status = ensure_transition(previous, TaskStatus.FAILED).value
207 task.last_error = task.last_error or "lease expired: robot did not report progress"
208 task.finished_at = now
209 task.version += 1
210 await uow.events.add(
211 _event(
212 task,
213 TaskEventKind.FAILED,
214 from_status=previous.value,
215 to_status=task.status,
216 robot_id=task.robot_id,
217 message="attempts exhausted after lease expiry",
218 payload={"attempts": task.attempts},
219 )
220 )
223def _event(task: Task, kind: TaskEventKind, **kwargs):
224 from robot_dispatch.db.models import TaskEvent
226 return TaskEvent.new(task_id=task.id, kind=kind.value, **kwargs)
229def _command_outbox_message(command: CommandMessage, *, timeout_seconds: int):
230 from robot_dispatch.db.models import OutboxMessage
232 return OutboxMessage.new(
233 exchange=COMMANDS_EXCHANGE,
234 routing_key=command_routing_key(command.robot_id),
235 payload=command.model_dump(mode="json"),
236 message_id=str(command.command_id),
237 headers={"task_id": str(command.task_id), "attempt": command.attempt},
238 expiration_ms=timeout_seconds * 1000,
239 )