120 lines
3.3 KiB
Python
120 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from sqlmodel import Session, col, select
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import engine
|
|
from app.models import TerminalMessageRaw, TerminalRawArchiveManifest, get_datetime_utc
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _format_timestamp(value: datetime) -> str:
|
|
return value.strftime("%Y-%m-%dT%H-%M-%S")
|
|
|
|
|
|
def _build_archive_path(
|
|
*,
|
|
archive_base_dir: Path,
|
|
device_id: str,
|
|
range_start: datetime,
|
|
range_end: datetime,
|
|
) -> Path:
|
|
day = range_start.strftime("%Y-%m-%d")
|
|
output_dir = archive_base_dir / device_id / day
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
filename = (
|
|
f"device_{device_id}_{_format_timestamp(range_start)}"
|
|
f"_{_format_timestamp(range_end)}.txt"
|
|
)
|
|
return output_dir / filename
|
|
|
|
|
|
def run_archive_once(
|
|
*,
|
|
session: Session,
|
|
archive_base_dir: Path | str,
|
|
retention_days: int,
|
|
now: datetime | None = None,
|
|
) -> list[TerminalRawArchiveManifest]:
|
|
archive_root = Path(archive_base_dir)
|
|
ts_now = now or get_datetime_utc()
|
|
cutoff = ts_now - timedelta(days=retention_days)
|
|
|
|
statement = (
|
|
select(TerminalMessageRaw)
|
|
.where(TerminalMessageRaw.received_at < cutoff)
|
|
.order_by(
|
|
col(TerminalMessageRaw.device_id), col(TerminalMessageRaw.received_at)
|
|
)
|
|
)
|
|
rows = session.exec(statement).all()
|
|
if not rows:
|
|
return []
|
|
|
|
grouped: dict[tuple[str, str], list[TerminalMessageRaw]] = {}
|
|
for row in rows:
|
|
key = (row.device_id, row.received_at.strftime("%Y-%m-%d"))
|
|
grouped.setdefault(key, []).append(row)
|
|
|
|
manifests: list[TerminalRawArchiveManifest] = []
|
|
for (_, _), group_rows in grouped.items():
|
|
range_start = group_rows[0].received_at
|
|
range_end = group_rows[-1].received_at
|
|
output_path = _build_archive_path(
|
|
archive_base_dir=archive_root,
|
|
device_id=group_rows[0].device_id,
|
|
range_start=range_start,
|
|
range_end=range_end,
|
|
)
|
|
|
|
digest = hashlib.sha256()
|
|
record_count = 0
|
|
with output_path.open("wb") as out:
|
|
for row in group_rows:
|
|
out.write(row.payload_bytes)
|
|
digest.update(row.payload_bytes)
|
|
record_count += 1
|
|
|
|
manifest = TerminalRawArchiveManifest(
|
|
device_id=group_rows[0].device_id,
|
|
range_start=range_start,
|
|
range_end=range_end,
|
|
file_path=str(output_path),
|
|
file_size=output_path.stat().st_size,
|
|
sha256=digest.hexdigest(),
|
|
record_count=record_count,
|
|
)
|
|
session.add(manifest)
|
|
manifests.append(manifest)
|
|
|
|
for row in group_rows:
|
|
session.delete(row)
|
|
|
|
session.commit()
|
|
|
|
for manifest in manifests:
|
|
session.refresh(manifest)
|
|
|
|
return manifests
|
|
|
|
|
|
def run() -> None:
|
|
logging.basicConfig(level=logging.INFO)
|
|
with Session(engine) as session:
|
|
manifests = run_archive_once(
|
|
session=session,
|
|
archive_base_dir=settings.RAW_ARCHIVE_BASE_DIR,
|
|
retention_days=settings.RAW_HOT_RETENTION_DAYS,
|
|
)
|
|
logger.info("Archived %s file(s)", len(manifests))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|