feat: implement mqtt ingestion, parsing, and archive pipeline

2026-03-17 13:27:39 +08:00
parent 27101e9bff
commit f02d324335
16 changed files with 896 additions and 5 deletions
--- a/backend/app/mqtt/archive_job.py
+++ b/backend/app/mqtt/archive_job.py
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from sqlmodel import Session, col, select
+
+from app.core.config import settings
+from app.core.db import engine
+from app.models import TerminalMessageRaw, TerminalRawArchiveManifest, get_datetime_utc
+
+logger = logging.getLogger(__name__)
+
+
+def _format_timestamp(value: datetime) -> str:
+    return value.strftime("%Y-%m-%dT%H-%M-%S")
+
+
+def _build_archive_path(
+    *,
+    archive_base_dir: Path,
+    device_id: str,
+    range_start: datetime,
+    range_end: datetime,
+) -> Path:
+    day = range_start.strftime("%Y-%m-%d")
+    output_dir = archive_base_dir / device_id / day
+    output_dir.mkdir(parents=True, exist_ok=True)
+    filename = (
+        f"device_{device_id}_{_format_timestamp(range_start)}"
+        f"_{_format_timestamp(range_end)}.txt"
+    )
+    return output_dir / filename
+
+
+def run_archive_once(
+    *,
+    session: Session,
+    archive_base_dir: Path | str,
+    retention_days: int,
+    now: datetime | None = None,
+) -> list[TerminalRawArchiveManifest]:
+    archive_root = Path(archive_base_dir)
+    ts_now = now or get_datetime_utc()
+    cutoff = ts_now - timedelta(days=retention_days)
+
+    statement = (
+        select(TerminalMessageRaw)
+        .where(TerminalMessageRaw.received_at < cutoff)
+        .order_by(
+            col(TerminalMessageRaw.device_id), col(TerminalMessageRaw.received_at)
+        )
+    )
+    rows = session.exec(statement).all()
+    if not rows:
+        return []
+
+    grouped: dict[tuple[str, str], list[TerminalMessageRaw]] = {}
+    for row in rows:
+        key = (row.device_id, row.received_at.strftime("%Y-%m-%d"))
+        grouped.setdefault(key, []).append(row)
+
+    manifests: list[TerminalRawArchiveManifest] = []
+    for (_, _), group_rows in grouped.items():
+        range_start = group_rows[0].received_at
+        range_end = group_rows[-1].received_at
+        output_path = _build_archive_path(
+            archive_base_dir=archive_root,
+            device_id=group_rows[0].device_id,
+            range_start=range_start,
+            range_end=range_end,
+        )
+
+        digest = hashlib.sha256()
+        record_count = 0
+        with output_path.open("wb") as out:
+            for row in group_rows:
+                out.write(row.payload_bytes)
+                digest.update(row.payload_bytes)
+                record_count += 1
+
+        manifest = TerminalRawArchiveManifest(
+            device_id=group_rows[0].device_id,
+            range_start=range_start,
+            range_end=range_end,
+            file_path=str(output_path),
+            file_size=output_path.stat().st_size,
+            sha256=digest.hexdigest(),
+            record_count=record_count,
+        )
+        session.add(manifest)
+        manifests.append(manifest)
+
+        for row in group_rows:
+            session.delete(row)
+
+    session.commit()
+
+    for manifest in manifests:
+        session.refresh(manifest)
+
+    return manifests
+
+
+def run() -> None:
+    logging.basicConfig(level=logging.INFO)
+    with Session(engine) as session:
+        manifests = run_archive_once(
+            session=session,
+            archive_base_dir=settings.RAW_ARCHIVE_BASE_DIR,
+            retention_days=settings.RAW_HOT_RETENTION_DAYS,
+        )
+    logger.info("Archived %s file(s)", len(manifests))
+
+
+if __name__ == "__main__":
+    run()