from __future__ import annotations import hashlib import logging from datetime import datetime, timedelta from pathlib import Path from sqlmodel import Session, col, select from app.core.config import settings from app.core.db import engine from app.models import TerminalMessageRaw, TerminalRawArchiveManifest, get_datetime_utc logger = logging.getLogger(__name__) def _format_timestamp(value: datetime) -> str: return value.strftime("%Y-%m-%dT%H-%M-%S") def _build_archive_path( *, archive_base_dir: Path, device_id: str, range_start: datetime, range_end: datetime, ) -> Path: day = range_start.strftime("%Y-%m-%d") output_dir = archive_base_dir / device_id / day output_dir.mkdir(parents=True, exist_ok=True) filename = ( f"device_{device_id}_{_format_timestamp(range_start)}" f"_{_format_timestamp(range_end)}.txt" ) return output_dir / filename def run_archive_once( *, session: Session, archive_base_dir: Path | str, retention_days: int, now: datetime | None = None, ) -> list[TerminalRawArchiveManifest]: archive_root = Path(archive_base_dir) ts_now = now or get_datetime_utc() cutoff = ts_now - timedelta(days=retention_days) statement = ( select(TerminalMessageRaw) .where(TerminalMessageRaw.received_at < cutoff) .order_by( col(TerminalMessageRaw.device_id), col(TerminalMessageRaw.received_at) ) ) rows = session.exec(statement).all() if not rows: return [] grouped: dict[tuple[str, str], list[TerminalMessageRaw]] = {} for row in rows: key = (row.device_id, row.received_at.strftime("%Y-%m-%d")) grouped.setdefault(key, []).append(row) manifests: list[TerminalRawArchiveManifest] = [] for (_, _), group_rows in grouped.items(): range_start = group_rows[0].received_at range_end = group_rows[-1].received_at output_path = _build_archive_path( archive_base_dir=archive_root, device_id=group_rows[0].device_id, range_start=range_start, range_end=range_end, ) digest = hashlib.sha256() record_count = 0 with output_path.open("wb") as out: for row in group_rows: out.write(row.payload_bytes) digest.update(row.payload_bytes) record_count += 1 manifest = TerminalRawArchiveManifest( device_id=group_rows[0].device_id, range_start=range_start, range_end=range_end, file_path=str(output_path), file_size=output_path.stat().st_size, sha256=digest.hexdigest(), record_count=record_count, ) session.add(manifest) manifests.append(manifest) for row in group_rows: session.delete(row) session.commit() for manifest in manifests: session.refresh(manifest) return manifests def run() -> None: logging.basicConfig(level=logging.INFO) with Session(engine) as session: manifests = run_archive_once( session=session, archive_base_dir=settings.RAW_ARCHIVE_BASE_DIR, retention_days=settings.RAW_HOT_RETENTION_DAYS, ) logger.info("Archived %s file(s)", len(manifests)) if __name__ == "__main__": run()