feat: implement mqtt ingestion, parsing, and archive pipeline

backend/app/mqtt/repository.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime
+from typing import Any
+
+from sqlmodel import Session, col, select
+
+from app.models import (
+    TerminalDeviceState,
+    TerminalMessageRaw,
+    TerminalObservationEvent,
+    get_datetime_utc,
+)
+
+
+def insert_raw_message(
+    *,
+    session: Session,
+    device_id: str,
+    topic: str,
+    payload: bytes,
+    qos: int,
+    retain: bool,
+    received_at: datetime | None = None,
+) -> TerminalMessageRaw:
+    ts = received_at or get_datetime_utc()
+    row = TerminalMessageRaw(
+        device_id=device_id,
+        topic=topic,
+        received_at=ts,
+        payload_bytes=payload,
+        payload_size=len(payload),
+        payload_sha256=hashlib.sha256(payload).hexdigest(),
+        qos=qos,
+        retain=retain,
+        parse_status="pending",
+        created_at=ts,
+    )
+    session.add(row)
+    session.commit()
+    session.refresh(row)
+    return row
+
+
+def fetch_pending_rows_for_update(
+    *, session: Session, batch_size: int
+) -> list[TerminalMessageRaw]:
+    statement = (
+        select(TerminalMessageRaw)
+        .where(TerminalMessageRaw.parse_status == "pending")
+        .order_by(col(TerminalMessageRaw.received_at))
+        .limit(batch_size)
+    )
+
+    bind = session.get_bind()
+    if bind is not None and bind.dialect.name != "sqlite":
+        statement = statement.with_for_update(skip_locked=True)
+
+    rows = session.exec(statement).all()
+    return list(rows)
+
+
+def mark_processing(*, session: Session, row: TerminalMessageRaw) -> None:
+    row.parse_status = "processing"
+    row.parse_attempts += 1
+    row.parse_error = None
+    session.add(row)
+
+
+def mark_succeeded(*, session: Session, row: TerminalMessageRaw) -> None:
+    row.parse_status = "succeeded"
+    row.parse_error = None
+    row.next_retry_at = None
+    session.add(row)
+
+
+def mark_failed(
+    *,
+    session: Session,
+    row: TerminalMessageRaw,
+    parse_error: str,
+    next_retry_at: datetime | None,
+) -> None:
+    row.parse_status = "failed"
+    row.parse_error = parse_error[:1024]
+    row.next_retry_at = next_retry_at
+    session.add(row)
+
+
+def write_events_and_update_state(
+    *,
+    session: Session,
+    row: TerminalMessageRaw,
+    events: list[dict[str, Any]],
+) -> None:
+    if not events:
+        state = session.get(TerminalDeviceState, row.device_id)
+        if state is None:
+            state = TerminalDeviceState(device_id=row.device_id)
+        state.last_event_time = row.received_at
+        state.updated_at = get_datetime_utc()
+        session.add(state)
+        return
+
+    state = session.get(TerminalDeviceState, row.device_id)
+    if state is None:
+        state = TerminalDeviceState(device_id=row.device_id)
+
+    for event in events:
+        payload = event.get("payload", {})
+        event_type = str(event.get("event_type", "unknown"))
+        db_event = TerminalObservationEvent(
+            raw_id=row.id,
+            device_id=row.device_id,
+            event_time=row.received_at,
+            event_type=event_type,
+            payload_json=payload if isinstance(payload, dict) else {"value": payload},
+        )
+        session.add(db_event)
+
+        if event_type == "nmea_gga":
+            state.lat = payload.get("lat")
+            state.lon = payload.get("lon")
+            state.alt_m = payload.get("altitude_m")
+            state.satellites_used = payload.get("satellites_used")
+            fix_quality = payload.get("fix_quality")
+            if fix_quality is not None:
+                state.fix_status = str(fix_quality)
+        elif event_type == "nmea_rmc":
+            if payload.get("lat") is not None:
+                state.lat = payload.get("lat")
+            if payload.get("lon") is not None:
+                state.lon = payload.get("lon")
+            status = payload.get("status")
+            if status is not None:
+                state.fix_status = str(status)
+        elif event_type == "nmea_gsv":
+            state.satellite_signal_summary = {
+                "snr_values": payload.get("snr_values", []),
+                "satellites_in_view": payload.get("satellites_in_view"),
+            }
+        elif event_type == "rtcm_summary":
+            state.rtcm_type_summary = payload.get("rtcm_types", {})
+
+    state.last_event_time = row.received_at
+    state.updated_at = get_datetime_utc()
+    session.add(state)