feat: implement mqtt ingestion, parsing, and archive pipeline

backend/app/mqtt/parser_worker.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import logging
+import time
+from datetime import timedelta
+
+from sqlmodel import Session
+
+from app.core.config import settings
+from app.core.db import engine
+from app.models import TerminalMessageRaw, get_datetime_utc
+from app.mqtt.parsers.mixed import parse_mixed_payload
+from app.mqtt.repository import (
+    fetch_pending_rows_for_update,
+    mark_failed,
+    mark_processing,
+    mark_succeeded,
+    write_events_and_update_state,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def _next_retry_delay_seconds(attempts: int) -> int:
+    # Bounded exponential backoff: 30s, 60s, 120s, ... capped to 15m.
+    return int(min(30 * (2 ** max(attempts - 1, 0)), 900))
+
+
+def _process_one_row(*, session: Session, row: TerminalMessageRaw) -> None:
+    mark_processing(session=session, row=row)
+    session.commit()
+
+    try:
+        events = parse_mixed_payload(row.payload_bytes)
+        write_events_and_update_state(session=session, row=row, events=events)
+        mark_succeeded(session=session, row=row)
+    except Exception as exc:  # pragma: no cover - safety net path
+        delay = _next_retry_delay_seconds(row.parse_attempts)
+        mark_failed(
+            session=session,
+            row=row,
+            parse_error=str(exc),
+            next_retry_at=get_datetime_utc() + timedelta(seconds=delay),
+        )
+
+    session.commit()
+
+
+def process_pending_batch(*, session: Session, batch_size: int) -> int:
+    rows = fetch_pending_rows_for_update(session=session, batch_size=batch_size)
+    for row in rows:
+        _process_one_row(session=session, row=row)
+    return len(rows)
+
+
+def run() -> None:
+    logging.basicConfig(level=logging.INFO)
+    sleep_seconds = max(settings.PARSER_POLL_INTERVAL_MS, 50) / 1000.0
+
+    while True:
+        with Session(engine) as session:
+            processed = process_pending_batch(
+                session=session, batch_size=settings.PARSER_BATCH_SIZE
+            )
+        if processed == 0:
+            time.sleep(sleep_seconds)
+        else:
+            logger.info("Processed %s pending MQTT raw messages", processed)
+
+
+if __name__ == "__main__":
+    run()