Coverage for src/robot_dispatch/messaging/outbox_relay.py: 95%

43 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-07 17:18 +0000

1from __future__ import annotations 

2 

3from datetime import datetime, timedelta 

4 

5from robot_dispatch.config import Settings 

6from robot_dispatch.db.models import utcnow 

7from robot_dispatch.logging_setup import get_logger 

8from robot_dispatch.messaging.publisher import Publisher 

9from robot_dispatch.repositories.protocols import UnitOfWorkFactory 

10 

11logger = get_logger("outbox") 

12 

13 

14class OutboxRelay: 

15 """Transactional outbox publisher. 

16 

17 The dispatcher writes the outgoing command into `outbox_messages` inside the 

18 same transaction that assigns the task. This relay is the only place that 

19 talks to RabbitMQ, so a broker outage or a crash never loses a command: the 

20 row stays pending and is retried until the broker confirms the publish. 

21 """ 

22 

23 def __init__( 

24 self, 

25 uow_factory: UnitOfWorkFactory, 

26 publisher: Publisher, 

27 settings: Settings, 

28 *, 

29 clock=utcnow, 

30 ) -> None: 

31 self._uow_factory = uow_factory 

32 self._publisher = publisher 

33 self._settings = settings 

34 self._clock = clock 

35 

36 async def relay_once(self) -> int: 

37 published = 0 

38 now = self._clock() 

39 async with self._uow_factory() as uow: 

40 messages = await uow.outbox.lock_pending(now=now, limit=self._settings.outbox_batch_size) 

41 for message in messages: 

42 try: 

43 await self._publisher.publish( 

44 exchange=message.exchange, 

45 routing_key=message.routing_key, 

46 payload=message.payload, 

47 message_id=message.message_id, 

48 headers=message.headers, 

49 expiration_ms=message.expiration_ms, 

50 ) 

51 except Exception as exc: # noqa: BLE001 - any publish failure is retried 

52 attempts = message.attempts + 1 

53 delay = min(2 ** attempts, self._settings.retry_max_delay_seconds) 

54 await uow.outbox.mark_failed( 

55 message, 

56 str(exc), 

57 now + timedelta(seconds=delay), 

58 attempts, 

59 ) 

60 level = logger.error if attempts >= self._settings.outbox_max_attempts else logger.warning 

61 level( 

62 "outbox publish failed", 

63 message_id=message.message_id, 

64 attempts=attempts, 

65 retry_in_seconds=delay, 

66 error=str(exc), 

67 ) 

68 continue 

69 await uow.outbox.mark_sent(message, self._clock()) 

70 published += 1 

71 await uow.commit() 

72 if published: 

73 logger.info("outbox relayed", count=published) 

74 return published 

75 

76 

77class HeartbeatWatchdog: 

78 """Marks robots offline when their heartbeats stop.""" 

79 

80 def __init__(self, robot_service, settings: Settings, *, clock=utcnow) -> None: 

81 self._robots = robot_service 

82 self._settings = settings 

83 self._clock = clock 

84 

85 async def sweep_once(self) -> int: 

86 threshold: datetime = self._clock() - timedelta(seconds=self._settings.robot_offline_after_seconds) 

87 return await self._robots.mark_offline(threshold)