refactor: simplify mqtt pipeline state and remove event table

backend/app/mqtt/ingestor.py

@@ -1,24 +1,32 @@
 from __future__ import annotations
 
 import logging
+import re
+from typing import Any
 
-import paho.mqtt.client as mqtt  # type: ignore[import-not-found,import-untyped]
 from sqlmodel import Session
 
 from app.core.config import settings
 from app.core.db import engine
 from app.mqtt.repository import insert_raw_message
-from app.mqtt.topic import extract_device_id
 
 logger = logging.getLogger(__name__)
+TOPIC_PATTERN = re.compile(r"^terminal/(?P<device_id>[^/]+)/raw$")
+
+
+def extract_device_id(topic: str) -> str:
+    match = TOPIC_PATTERN.match(topic)
+    if match is None:
+        raise ValueError("topic must match terminal/{device_id}/raw")
+    return match.group("device_id")
 
 
 def _on_connect(
-    client: mqtt.Client,
+    client: Any,
     _userdata: object,
     _flags: dict[str, int],
     reason_code: int,
-    _properties: mqtt.Properties | None = None,
+    _properties: object | None = None,
 ) -> None:
     if reason_code != 0:
         logger.error("MQTT connect failed: rc=%s", reason_code)
@@ -31,9 +39,9 @@ def _on_connect(
 
 
 def _on_message(
-    _client: mqtt.Client,
+    _client: Any,
     _userdata: object,
-    message: mqtt.MQTTMessage,
+    message: Any,
 ) -> None:
     try:
         device_id = extract_device_id(message.topic)
@@ -52,7 +60,14 @@ def _on_message(
         )
 
 
-def create_client() -> mqtt.Client:
+def create_client() -> Any:
+    try:
+        import paho.mqtt.client as mqtt  # type: ignore[import-not-found,import-untyped]
+    except ModuleNotFoundError as exc:  # pragma: no cover - runtime guard
+        raise RuntimeError(
+            "paho-mqtt is required to run mqtt-ingestor. Install backend dependencies first."
+        ) from exc
+
     client = mqtt.Client(
         callback_api_version=mqtt.CallbackAPIVersion.VERSION2,
         client_id=settings.MQTT_CLIENT_ID,

backend/app/mqtt/parser_worker.py

@@ -2,45 +2,33 @@ from __future__ import annotations
 
 import logging
 import time
-from datetime import timedelta
 
 from sqlmodel import Session
 
 from app.core.config import settings
 from app.core.db import engine
-from app.models import TerminalMessageRaw, get_datetime_utc
+from app.models import TerminalMessageRaw
 from app.mqtt.parsers.mixed import parse_mixed_payload
 from app.mqtt.repository import (
     fetch_pending_rows_for_update,
     mark_failed,
-    mark_processing,
     mark_succeeded,
-    write_events_and_update_state,
+    update_device_state_from_events,
 )
 
 logger = logging.getLogger(__name__)
 
 
-def _next_retry_delay_seconds(attempts: int) -> int:
-    # Bounded exponential backoff: 30s, 60s, 120s, ... capped to 15m.
-    return int(min(30 * (2 ** max(attempts - 1, 0)), 900))
-
-
 def _process_one_row(*, session: Session, row: TerminalMessageRaw) -> None:
-    mark_processing(session=session, row=row)
-    session.commit()
-
     try:
         events = parse_mixed_payload(row.payload_bytes)
-        write_events_and_update_state(session=session, row=row, events=events)
+        update_device_state_from_events(session=session, row=row, events=events)
         mark_succeeded(session=session, row=row)
     except Exception as exc:  # pragma: no cover - safety net path
-        delay = _next_retry_delay_seconds(row.parse_attempts)
         mark_failed(
             session=session,
             row=row,
             parse_error=str(exc),
-            next_retry_at=get_datetime_utc() + timedelta(seconds=delay),
         )
 
     session.commit()

backend/app/mqtt/parsers/__init__.py

@@ -1,5 +1,4 @@
-from app.mqtt.parsers.mixed import parse_mixed_payload
+from app.mqtt.parsers.mixed import parse_mixed_payload, summarize_rtcm_types
 from app.mqtt.parsers.nmea import parse_nmea_sentences
-from app.mqtt.parsers.rtcm import summarize_rtcm_types
 
 __all__ = ["parse_mixed_payload", "parse_nmea_sentences", "summarize_rtcm_types"]

backend/app/mqtt/parsers/mixed.py

@@ -3,7 +3,27 @@ from __future__ import annotations
 from typing import Any
 
 from app.mqtt.parsers.nmea import parse_nmea_sentences
-from app.mqtt.parsers.rtcm import summarize_rtcm_types
+
+
+def summarize_rtcm_types(payload: bytes) -> dict[int, int]:
+    counts: dict[int, int] = {}
+    i = 0
+
+    while i + 6 <= len(payload):
+        if payload[i] != 0xD3:
+            i += 1
+            continue
+
+        length = ((payload[i + 1] & 0x03) << 8) | payload[i + 2]
+        frame_end = i + 3 + length + 3
+        if frame_end > len(payload):
+            break
+
+        message_type = ((payload[i + 3] << 4) | (payload[i + 4] >> 4)) & 0x0FFF
+        counts[message_type] = counts.get(message_type, 0) + 1
+        i = frame_end
+
+    return counts
 
 
 def parse_mixed_payload(payload: bytes) -> list[dict[str, Any]]:

backend/app/mqtt/parsers/rtcm.py

@@ -1,22 +0,0 @@
-from __future__ import annotations
-
-
-def summarize_rtcm_types(payload: bytes) -> dict[int, int]:
-    counts: dict[int, int] = {}
-    i = 0
-
-    while i + 6 <= len(payload):
-        if payload[i] != 0xD3:
-            i += 1
-            continue
-
-        length = ((payload[i + 1] & 0x03) << 8) | payload[i + 2]
-        frame_end = i + 3 + length + 3
-        if frame_end > len(payload):
-            break
-
-        message_type = ((payload[i + 3] << 4) | (payload[i + 4] >> 4)) & 0x0FFF
-        counts[message_type] = counts.get(message_type, 0) + 1
-        i = frame_end
-
-    return counts

backend/app/mqtt/repository.py

@@ -9,7 +9,6 @@ from sqlmodel import Session, col, select
 from app.models import (
     TerminalDeviceState,
     TerminalMessageRaw,
-    TerminalObservationEvent,
     get_datetime_utc,
 )
 
@@ -34,7 +33,6 @@ def insert_raw_message(
         payload_sha256=hashlib.sha256(payload).hexdigest(),
         qos=qos,
         retain=retain,
-        parse_status="pending",
         created_at=ts,
     )
     session.add(row)
@@ -48,7 +46,8 @@ def fetch_pending_rows_for_update(
 ) -> list[TerminalMessageRaw]:
     statement = (
         select(TerminalMessageRaw)
-        .where(TerminalMessageRaw.parse_status == "pending")
+        .where(col(TerminalMessageRaw.parsed_at).is_(None))
+        .where(col(TerminalMessageRaw.parse_error).is_(None))
         .order_by(col(TerminalMessageRaw.received_at))
         .limit(batch_size)
     )
@@ -61,17 +60,9 @@ def fetch_pending_rows_for_update(
     return list(rows)
 
 
-def mark_processing(*, session: Session, row: TerminalMessageRaw) -> None:
-    row.parse_status = "processing"
-    row.parse_attempts += 1
-    row.parse_error = None
-    session.add(row)
-
-
 def mark_succeeded(*, session: Session, row: TerminalMessageRaw) -> None:
-    row.parse_status = "succeeded"
+    row.parsed_at = get_datetime_utc()
     row.parse_error = None
-    row.next_retry_at = None
     session.add(row)
 
 
@@ -80,15 +71,13 @@ def mark_failed(
     session: Session,
     row: TerminalMessageRaw,
     parse_error: str,
-    next_retry_at: datetime | None,
 ) -> None:
-    row.parse_status = "failed"
+    row.parsed_at = get_datetime_utc()
     row.parse_error = parse_error[:1024]
-    row.next_retry_at = next_retry_at
     session.add(row)
 
 
-def write_events_and_update_state(
+def update_device_state_from_events(
     *,
     session: Session,
     row: TerminalMessageRaw,
@@ -110,14 +99,6 @@ def write_events_and_update_state(
     for event in events:
         payload = event.get("payload", {})
         event_type = str(event.get("event_type", "unknown"))
-        db_event = TerminalObservationEvent(
-            raw_id=row.id,
-            device_id=row.device_id,
-            event_time=row.received_at,
-            event_type=event_type,
-            payload_json=payload if isinstance(payload, dict) else {"value": payload},
-        )
-        session.add(db_event)
 
         if event_type == "nmea_gga":
             state.lat = payload.get("lat")

backend/app/mqtt/topic.py

@@ -1,10 +0,0 @@
-import re
-
-TOPIC_PATTERN = re.compile(r"^terminal/(?P<device_id>[^/]+)/raw$")
-
-
-def extract_device_id(topic: str) -> str:
-    match = TOPIC_PATTERN.match(topic)
-    if match is None:
-        raise ValueError("topic must match terminal/{device_id}/raw")
-    return match.group("device_id")