Coverage for src/robot_dispatch/services/results.py: 95%
113 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-07 17:18 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-07 17:18 +0000
1from __future__ import annotations
3from datetime import datetime, timedelta
5from robot_dispatch.config import Settings
6from robot_dispatch.db.models import Robot, Task, TaskEvent, utcnow
7from robot_dispatch.domain.enums import RobotStatus, TaskEventKind, TaskStatus
8from robot_dispatch.domain.policies import RetryPolicy, should_retry
9from robot_dispatch.domain.state_machine import ensure_transition
10from robot_dispatch.logging_setup import get_logger
11from robot_dispatch.messaging.messages import HeartbeatMessage, TaskResultMessage
12from robot_dispatch.repositories.protocols import UnitOfWork, UnitOfWorkFactory
14logger = get_logger("results")
17class ResultService:
18 """Applies robot feedback to tasks.
20 Every result is checked against the task's current `command_id`. A late
21 result from a previous attempt, a duplicate delivery or a message from a
22 lost robot must not resurrect a finished task, so stale messages are
23 recorded as `result_ignored` events and dropped.
24 """
26 def __init__(
27 self,
28 uow_factory: UnitOfWorkFactory,
29 settings: Settings,
30 *,
31 retry_policy: RetryPolicy | None = None,
32 clock=utcnow,
33 ) -> None:
34 self._uow_factory = uow_factory
35 self._settings = settings
36 self._policy = retry_policy or RetryPolicy(
37 base_delay_seconds=settings.retry_base_delay_seconds,
38 max_delay_seconds=settings.retry_max_delay_seconds,
39 jitter_ratio=settings.retry_jitter_ratio,
40 )
41 self._clock = clock
43 async def handle(self, message: TaskResultMessage) -> str:
44 now = self._clock()
45 async with self._uow_factory() as uow:
46 task = await uow.tasks.get(message.task_id)
47 if task is None:
48 logger.warning("result for unknown task", task_id=str(message.task_id))
49 return "unknown_task"
51 current = TaskStatus(task.status)
52 if current.is_terminal:
53 await self._ignore(uow, task, message, now, "task already finished")
54 await uow.commit()
55 return "terminal_task"
57 if message.command_id is not None and task.command_id != message.command_id:
58 await self._ignore(uow, task, message, now, "stale command_id")
59 await uow.commit()
60 return "stale_command"
62 if current not in {TaskStatus.ASSIGNED, TaskStatus.RUNNING}:
63 await self._ignore(uow, task, message, now, f"task is {current.value}")
64 await uow.commit()
65 return "unexpected_status"
67 if message.event == "progress":
68 await self._apply_progress(uow, task, message, now)
69 await uow.commit()
70 return "progress"
72 robot = await uow.robots.get(task.robot_id) if task.robot_id else None
73 self._release_robot(robot, now)
75 if message.event == "succeeded":
76 task.status = ensure_transition(current, TaskStatus.SUCCEEDED).value
77 task.result = message.result or {"detail": message.detail}
78 task.last_error = None
79 task.lease_expires_at = None
80 task.finished_at = now
81 task.updated_at = now
82 task.version += 1
83 await uow.events.add(
84 _event(
85 task,
86 TaskEventKind.SUCCEEDED,
87 from_status=current.value,
88 to_status=task.status,
89 robot_id=task.robot_id,
90 message=message.detail,
91 payload={"attempt": task.attempts},
92 )
93 )
94 else:
95 await self._apply_failure(uow, task, message, now, current)
97 await uow.commit()
98 return message.event
100 async def _apply_progress(
101 self, uow: UnitOfWork, task: Task, message: TaskResultMessage, now: datetime
102 ) -> None:
103 previous = TaskStatus(task.status)
104 if previous is TaskStatus.ASSIGNED:
105 task.status = ensure_transition(previous, TaskStatus.RUNNING).value
106 task.lease_expires_at = now + timedelta(seconds=task.timeout_seconds)
107 task.updated_at = now
108 task.version += 1
109 await uow.events.add(
110 _event(
111 task,
112 TaskEventKind.PROGRESS,
113 from_status=previous.value,
114 to_status=task.status,
115 robot_id=task.robot_id,
116 message=message.detail,
117 payload={"progress": message.progress, "attempt": task.attempts},
118 )
119 )
121 async def _apply_failure(
122 self,
123 uow: UnitOfWork,
124 task: Task,
125 message: TaskResultMessage,
126 now: datetime,
127 current: TaskStatus,
128 ) -> None:
129 error = message.error or message.detail or "robot reported failure"
130 task.last_error = error
132 if should_retry(task.attempts, task.max_attempts):
133 delay = self._policy.delay_seconds(task.attempts)
134 task.status = ensure_transition(current, TaskStatus.RETRYING).value
135 task.available_at = now + timedelta(seconds=delay)
136 task.lease_expires_at = None
137 task.updated_at = now
138 task.version += 1
139 await uow.events.add(
140 _event(
141 task,
142 TaskEventKind.RETRY_SCHEDULED,
143 from_status=current.value,
144 to_status=task.status,
145 robot_id=task.robot_id,
146 message=f"retry in {delay:.1f}s after: {error}",
147 payload={
148 "attempt": task.attempts,
149 "max_attempts": task.max_attempts,
150 "retry_in_seconds": round(delay, 3),
151 },
152 )
153 )
154 return
156 task.status = ensure_transition(current, TaskStatus.FAILED).value
157 task.lease_expires_at = None
158 task.finished_at = now
159 task.updated_at = now
160 task.version += 1
161 await uow.events.add(
162 _event(
163 task,
164 TaskEventKind.FAILED,
165 from_status=current.value,
166 to_status=task.status,
167 robot_id=task.robot_id,
168 message=error,
169 payload={"attempts": task.attempts, "max_attempts": task.max_attempts},
170 )
171 )
173 def _release_robot(self, robot: Robot | None, now: datetime) -> None:
174 if robot is None:
175 return
176 robot.status = RobotStatus.IDLE.value
177 robot.current_task_id = None
178 robot.last_seen_at = now
179 robot.updated_at = now
181 async def _ignore(
182 self,
183 uow: UnitOfWork,
184 task: Task,
185 message: TaskResultMessage,
186 now: datetime,
187 reason: str,
188 ) -> None:
189 logger.info("result ignored", task_id=str(task.id), reason=reason, event=message.event)
190 await uow.events.add(
191 _event(
192 task,
193 TaskEventKind.RESULT_IGNORED,
194 from_status=task.status,
195 to_status=task.status,
196 robot_id=message.robot_id,
197 message=reason,
198 payload={
199 "event": message.event,
200 "command_id": str(message.command_id) if message.command_id else None,
201 },
202 )
203 )
206class HeartbeatService:
207 """Keeps `robots.last_seen_at` fresh and revives robots that came back online."""
209 def __init__(self, uow_factory: UnitOfWorkFactory, *, clock=utcnow) -> None:
210 self._uow_factory = uow_factory
211 self._clock = clock
213 async def handle(self, message: HeartbeatMessage) -> str:
214 now = self._clock()
215 async with self._uow_factory() as uow:
216 robot = await uow.robots.get(message.robot_id)
217 if robot is None:
218 logger.warning("heartbeat from unknown robot", robot_id=str(message.robot_id))
219 return "unknown_robot"
221 robot.last_seen_at = now
222 robot.updated_at = now
223 if robot.current_task_id is not None:
224 robot.status = RobotStatus.BUSY.value
225 elif message.status in {RobotStatus.IDLE.value, RobotStatus.BUSY.value}:
226 robot.status = message.status
227 else:
228 robot.status = RobotStatus.IDLE.value
229 await uow.commit()
230 return "ok"
233def _event(task: Task, kind: TaskEventKind, **kwargs) -> TaskEvent:
234 return TaskEvent.new(task_id=task.id, kind=kind.value, **kwargs)