Coverage for src/robot_dispatch/services/results.py: 95%

113 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-07 17:18 +0000

1from __future__ import annotations 

2 

3from datetime import datetime, timedelta 

4 

5from robot_dispatch.config import Settings 

6from robot_dispatch.db.models import Robot, Task, TaskEvent, utcnow 

7from robot_dispatch.domain.enums import RobotStatus, TaskEventKind, TaskStatus 

8from robot_dispatch.domain.policies import RetryPolicy, should_retry 

9from robot_dispatch.domain.state_machine import ensure_transition 

10from robot_dispatch.logging_setup import get_logger 

11from robot_dispatch.messaging.messages import HeartbeatMessage, TaskResultMessage 

12from robot_dispatch.repositories.protocols import UnitOfWork, UnitOfWorkFactory 

13 

14logger = get_logger("results") 

15 

16 

17class ResultService: 

18 """Applies robot feedback to tasks. 

19 

20 Every result is checked against the task's current `command_id`. A late 

21 result from a previous attempt, a duplicate delivery or a message from a 

22 lost robot must not resurrect a finished task, so stale messages are 

23 recorded as `result_ignored` events and dropped. 

24 """ 

25 

26 def __init__( 

27 self, 

28 uow_factory: UnitOfWorkFactory, 

29 settings: Settings, 

30 *, 

31 retry_policy: RetryPolicy | None = None, 

32 clock=utcnow, 

33 ) -> None: 

34 self._uow_factory = uow_factory 

35 self._settings = settings 

36 self._policy = retry_policy or RetryPolicy( 

37 base_delay_seconds=settings.retry_base_delay_seconds, 

38 max_delay_seconds=settings.retry_max_delay_seconds, 

39 jitter_ratio=settings.retry_jitter_ratio, 

40 ) 

41 self._clock = clock 

42 

43 async def handle(self, message: TaskResultMessage) -> str: 

44 now = self._clock() 

45 async with self._uow_factory() as uow: 

46 task = await uow.tasks.get(message.task_id) 

47 if task is None: 

48 logger.warning("result for unknown task", task_id=str(message.task_id)) 

49 return "unknown_task" 

50 

51 current = TaskStatus(task.status) 

52 if current.is_terminal: 

53 await self._ignore(uow, task, message, now, "task already finished") 

54 await uow.commit() 

55 return "terminal_task" 

56 

57 if message.command_id is not None and task.command_id != message.command_id: 

58 await self._ignore(uow, task, message, now, "stale command_id") 

59 await uow.commit() 

60 return "stale_command" 

61 

62 if current not in {TaskStatus.ASSIGNED, TaskStatus.RUNNING}: 

63 await self._ignore(uow, task, message, now, f"task is {current.value}") 

64 await uow.commit() 

65 return "unexpected_status" 

66 

67 if message.event == "progress": 

68 await self._apply_progress(uow, task, message, now) 

69 await uow.commit() 

70 return "progress" 

71 

72 robot = await uow.robots.get(task.robot_id) if task.robot_id else None 

73 self._release_robot(robot, now) 

74 

75 if message.event == "succeeded": 

76 task.status = ensure_transition(current, TaskStatus.SUCCEEDED).value 

77 task.result = message.result or {"detail": message.detail} 

78 task.last_error = None 

79 task.lease_expires_at = None 

80 task.finished_at = now 

81 task.updated_at = now 

82 task.version += 1 

83 await uow.events.add( 

84 _event( 

85 task, 

86 TaskEventKind.SUCCEEDED, 

87 from_status=current.value, 

88 to_status=task.status, 

89 robot_id=task.robot_id, 

90 message=message.detail, 

91 payload={"attempt": task.attempts}, 

92 ) 

93 ) 

94 else: 

95 await self._apply_failure(uow, task, message, now, current) 

96 

97 await uow.commit() 

98 return message.event 

99 

100 async def _apply_progress( 

101 self, uow: UnitOfWork, task: Task, message: TaskResultMessage, now: datetime 

102 ) -> None: 

103 previous = TaskStatus(task.status) 

104 if previous is TaskStatus.ASSIGNED: 

105 task.status = ensure_transition(previous, TaskStatus.RUNNING).value 

106 task.lease_expires_at = now + timedelta(seconds=task.timeout_seconds) 

107 task.updated_at = now 

108 task.version += 1 

109 await uow.events.add( 

110 _event( 

111 task, 

112 TaskEventKind.PROGRESS, 

113 from_status=previous.value, 

114 to_status=task.status, 

115 robot_id=task.robot_id, 

116 message=message.detail, 

117 payload={"progress": message.progress, "attempt": task.attempts}, 

118 ) 

119 ) 

120 

121 async def _apply_failure( 

122 self, 

123 uow: UnitOfWork, 

124 task: Task, 

125 message: TaskResultMessage, 

126 now: datetime, 

127 current: TaskStatus, 

128 ) -> None: 

129 error = message.error or message.detail or "robot reported failure" 

130 task.last_error = error 

131 

132 if should_retry(task.attempts, task.max_attempts): 

133 delay = self._policy.delay_seconds(task.attempts) 

134 task.status = ensure_transition(current, TaskStatus.RETRYING).value 

135 task.available_at = now + timedelta(seconds=delay) 

136 task.lease_expires_at = None 

137 task.updated_at = now 

138 task.version += 1 

139 await uow.events.add( 

140 _event( 

141 task, 

142 TaskEventKind.RETRY_SCHEDULED, 

143 from_status=current.value, 

144 to_status=task.status, 

145 robot_id=task.robot_id, 

146 message=f"retry in {delay:.1f}s after: {error}", 

147 payload={ 

148 "attempt": task.attempts, 

149 "max_attempts": task.max_attempts, 

150 "retry_in_seconds": round(delay, 3), 

151 }, 

152 ) 

153 ) 

154 return 

155 

156 task.status = ensure_transition(current, TaskStatus.FAILED).value 

157 task.lease_expires_at = None 

158 task.finished_at = now 

159 task.updated_at = now 

160 task.version += 1 

161 await uow.events.add( 

162 _event( 

163 task, 

164 TaskEventKind.FAILED, 

165 from_status=current.value, 

166 to_status=task.status, 

167 robot_id=task.robot_id, 

168 message=error, 

169 payload={"attempts": task.attempts, "max_attempts": task.max_attempts}, 

170 ) 

171 ) 

172 

173 def _release_robot(self, robot: Robot | None, now: datetime) -> None: 

174 if robot is None: 

175 return 

176 robot.status = RobotStatus.IDLE.value 

177 robot.current_task_id = None 

178 robot.last_seen_at = now 

179 robot.updated_at = now 

180 

181 async def _ignore( 

182 self, 

183 uow: UnitOfWork, 

184 task: Task, 

185 message: TaskResultMessage, 

186 now: datetime, 

187 reason: str, 

188 ) -> None: 

189 logger.info("result ignored", task_id=str(task.id), reason=reason, event=message.event) 

190 await uow.events.add( 

191 _event( 

192 task, 

193 TaskEventKind.RESULT_IGNORED, 

194 from_status=task.status, 

195 to_status=task.status, 

196 robot_id=message.robot_id, 

197 message=reason, 

198 payload={ 

199 "event": message.event, 

200 "command_id": str(message.command_id) if message.command_id else None, 

201 }, 

202 ) 

203 ) 

204 

205 

206class HeartbeatService: 

207 """Keeps `robots.last_seen_at` fresh and revives robots that came back online.""" 

208 

209 def __init__(self, uow_factory: UnitOfWorkFactory, *, clock=utcnow) -> None: 

210 self._uow_factory = uow_factory 

211 self._clock = clock 

212 

213 async def handle(self, message: HeartbeatMessage) -> str: 

214 now = self._clock() 

215 async with self._uow_factory() as uow: 

216 robot = await uow.robots.get(message.robot_id) 

217 if robot is None: 

218 logger.warning("heartbeat from unknown robot", robot_id=str(message.robot_id)) 

219 return "unknown_robot" 

220 

221 robot.last_seen_at = now 

222 robot.updated_at = now 

223 if robot.current_task_id is not None: 

224 robot.status = RobotStatus.BUSY.value 

225 elif message.status in {RobotStatus.IDLE.value, RobotStatus.BUSY.value}: 

226 robot.status = message.status 

227 else: 

228 robot.status = RobotStatus.IDLE.value 

229 await uow.commit() 

230 return "ok" 

231 

232 

233def _event(task: Task, kind: TaskEventKind, **kwargs) -> TaskEvent: 

234 return TaskEvent.new(task_id=task.id, kind=kind.value, **kwargs)