Coverage for src/robot_dispatch/services/dispatcher.py: 99%

105 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-07 17:18 +0000

1from __future__ import annotations 

2 

3import uuid 

4from dataclasses import dataclass 

5from datetime import datetime, timedelta 

6 

7from robot_dispatch.config import Settings 

8from robot_dispatch.db.models import Robot, Task, utcnow 

9from robot_dispatch.domain.enums import RobotStatus, TaskEventKind, TaskStatus 

10from robot_dispatch.domain.policies import RetryPolicy, should_retry 

11from robot_dispatch.domain.state_machine import ensure_transition 

12from robot_dispatch.logging_setup import get_logger 

13from robot_dispatch.messaging.messages import ( 

14 COMMANDS_EXCHANGE, 

15 CommandMessage, 

16 command_routing_key, 

17) 

18from robot_dispatch.repositories.protocols import UnitOfWork, UnitOfWorkFactory 

19 

20logger = get_logger("dispatcher") 

21 

22 

23@dataclass(frozen=True) 

24class DispatchOutcome: 

25 dispatched: bool 

26 reason: str 

27 task_id: uuid.UUID | None = None 

28 robot_id: uuid.UUID | None = None 

29 attempt: int | None = None 

30 

31 

32class DispatcherService: 

33 """Assigns queued tasks to idle robots. 

34 

35 Reliability rules: 

36 * candidates are locked with `FOR UPDATE SKIP LOCKED`, so N replicas never 

37 grab the same task; 

38 * the assignment, the robot state change and the outgoing command (outbox) 

39 are committed in one transaction: a crash cannot leave a task assigned to 

40 a robot that never receives a command; 

41 * every attempt gets a fresh `command_id` and a lease, so a silent robot is 

42 detected by the reaper instead of stalling the task forever. 

43 """ 

44 

45 def __init__( 

46 self, 

47 uow_factory: UnitOfWorkFactory, 

48 settings: Settings, 

49 *, 

50 clock=utcnow, 

51 ) -> None: 

52 self._uow_factory = uow_factory 

53 self._settings = settings 

54 self._clock = clock 

55 

56 async def dispatch_once(self) -> DispatchOutcome: 

57 now = self._clock() 

58 async with self._uow_factory() as uow: 

59 candidates = await uow.tasks.lock_ready(now=now, limit=self._settings.dispatcher_batch_size) 

60 if not candidates: 

61 return DispatchOutcome(dispatched=False, reason="no_ready_tasks") 

62 

63 for task in candidates: 

64 robot = await uow.robots.find_idle_for( 

65 task_type=task.type, 

66 now=now, 

67 offline_after=self._settings.robot_offline_after_seconds, 

68 ) 

69 if robot is None: 

70 continue 

71 outcome = await self._assign(uow, task, robot, now) 

72 await uow.commit() 

73 logger.info( 

74 "task dispatched", 

75 task_id=str(task.id), 

76 robot=robot.name, 

77 attempt=task.attempts, 

78 ) 

79 return outcome 

80 

81 return DispatchOutcome(dispatched=False, reason="no_idle_robot") 

82 

83 async def _assign( 

84 self, uow: UnitOfWork, task: Task, robot: Robot, now: datetime 

85 ) -> DispatchOutcome: 

86 previous = TaskStatus(task.status) 

87 attempt = task.attempts + 1 

88 command = CommandMessage( 

89 command_id=uuid.uuid4(), 

90 task_id=task.id, 

91 task_type=task.type, 

92 robot_id=robot.id, 

93 namespace=robot.namespace, 

94 attempt=attempt, 

95 timeout_seconds=task.timeout_seconds, 

96 payload=task.payload or {}, 

97 issued_at=now, 

98 ) 

99 

100 task.status = ensure_transition(previous, TaskStatus.ASSIGNED).value 

101 task.attempts = attempt 

102 task.robot_id = robot.id 

103 task.command_id = command.command_id 

104 task.lease_expires_at = now + timedelta(seconds=task.timeout_seconds) 

105 task.available_at = now 

106 task.started_at = task.started_at or now 

107 task.updated_at = now 

108 task.version += 1 

109 

110 robot.status = RobotStatus.BUSY.value 

111 robot.current_task_id = task.id 

112 robot.updated_at = now 

113 

114 await uow.events.add( 

115 _event( 

116 task, 

117 TaskEventKind.DISPATCHED, 

118 from_status=previous.value, 

119 to_status=task.status, 

120 robot_id=robot.id, 

121 message=f"assigned to {robot.name}, attempt {attempt}/{task.max_attempts}", 

122 payload={"command_id": str(command.command_id), "attempt": attempt}, 

123 ) 

124 ) 

125 await uow.outbox.add( 

126 _command_outbox_message(command, timeout_seconds=task.timeout_seconds) 

127 ) 

128 return DispatchOutcome( 

129 dispatched=True, 

130 reason="assigned", 

131 task_id=task.id, 

132 robot_id=robot.id, 

133 attempt=attempt, 

134 ) 

135 

136 

137class LeaseReaper: 

138 """Owns every "robot went silent" case. 

139 

140 A task whose lease expired is re-queued with exponential backoff until 

141 `max_attempts`, then failed. The robot is always released, so a dead robot 

142 never blocks a capability slot. 

143 """ 

144 

145 def __init__( 

146 self, 

147 uow_factory: UnitOfWorkFactory, 

148 settings: Settings, 

149 *, 

150 retry_policy: RetryPolicy | None = None, 

151 clock=utcnow, 

152 ) -> None: 

153 self._uow_factory = uow_factory 

154 self._settings = settings 

155 self._policy = retry_policy or RetryPolicy( 

156 base_delay_seconds=settings.retry_base_delay_seconds, 

157 max_delay_seconds=settings.retry_max_delay_seconds, 

158 jitter_ratio=settings.retry_jitter_ratio, 

159 ) 

160 self._clock = clock 

161 

162 async def reap_once(self, *, limit: int = 100) -> int: 

163 now = self._clock() 

164 async with self._uow_factory() as uow: 

165 expired = await uow.tasks.lock_expired_leases(now=now, limit=limit) 

166 for task in expired: 

167 await self._recover(uow, task, now) 

168 await uow.commit() 

169 if expired: 

170 logger.warning("expired leases recovered", count=len(expired)) 

171 return len(expired) 

172 

173 async def _recover(self, uow: UnitOfWork, task: Task, now: datetime) -> None: 

174 robot = await uow.robots.get(task.robot_id) if task.robot_id else None 

175 if robot is not None: 

176 robot.status = RobotStatus.IDLE.value 

177 robot.current_task_id = None 

178 robot.updated_at = now 

179 

180 previous = TaskStatus(task.status) 

181 task.lease_expires_at = None 

182 task.updated_at = now 

183 

184 if previous in {TaskStatus.PENDING, TaskStatus.RETRYING}: 

185 return 

186 

187 if should_retry(task.attempts, task.max_attempts): 

188 delay = self._policy.delay_seconds(task.attempts) 

189 task.status = ensure_transition(previous, TaskStatus.RETRYING).value 

190 task.available_at = now + timedelta(seconds=delay) 

191 task.last_error = "lease expired: robot did not report progress" 

192 task.version += 1 

193 await uow.events.add( 

194 _event( 

195 task, 

196 TaskEventKind.LEASE_EXPIRED, 

197 from_status=previous.value, 

198 to_status=task.status, 

199 robot_id=task.robot_id, 

200 message=f"retry in {delay:.1f}s", 

201 payload={"attempt": task.attempts, "retry_in_seconds": round(delay, 3)}, 

202 ) 

203 ) 

204 return 

205 

206 task.status = ensure_transition(previous, TaskStatus.FAILED).value 

207 task.last_error = task.last_error or "lease expired: robot did not report progress" 

208 task.finished_at = now 

209 task.version += 1 

210 await uow.events.add( 

211 _event( 

212 task, 

213 TaskEventKind.FAILED, 

214 from_status=previous.value, 

215 to_status=task.status, 

216 robot_id=task.robot_id, 

217 message="attempts exhausted after lease expiry", 

218 payload={"attempts": task.attempts}, 

219 ) 

220 ) 

221 

222 

223def _event(task: Task, kind: TaskEventKind, **kwargs): 

224 from robot_dispatch.db.models import TaskEvent 

225 

226 return TaskEvent.new(task_id=task.id, kind=kind.value, **kwargs) 

227 

228 

229def _command_outbox_message(command: CommandMessage, *, timeout_seconds: int): 

230 from robot_dispatch.db.models import OutboxMessage 

231 

232 return OutboxMessage.new( 

233 exchange=COMMANDS_EXCHANGE, 

234 routing_key=command_routing_key(command.robot_id), 

235 payload=command.model_dump(mode="json"), 

236 message_id=str(command.command_id), 

237 headers={"task_id": str(command.task_id), "attempt": command.attempt}, 

238 expiration_ms=timeout_seconds * 1000, 

239 )