feat: implement mqtt ingestion, parsing, and archive pipeline

compose.yml

@@ -136,6 +136,83 @@ services:
       # Enable redirection for HTTP and HTTPS
       - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.middlewares=https-redirect
 
+  mqtt-ingestor:
+    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    restart: always
+    networks:
+      - default
+    depends_on:
+      db:
+        condition: service_healthy
+        restart: true
+      prestart:
+        condition: service_completed_successfully
+    command:
+      - python
+      - -m
+      - app.mqtt.ingestor
+    env_file:
+      - .env
+    environment:
+      - ENVIRONMENT=${ENVIRONMENT}
+      - SECRET_KEY=${SECRET_KEY?Variable not set}
+      - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set}
+      - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set}
+      - POSTGRES_SERVER=db
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER?Variable not set}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set}
+      - MQTT_HOST=${MQTT_HOST-101.35.119.226}
+      - MQTT_PORT=${MQTT_PORT-1883}
+      - MQTT_USERNAME=${MQTT_USERNAME}
+      - MQTT_PASSWORD=${MQTT_PASSWORD}
+      - MQTT_TOPIC_PATTERN=${MQTT_TOPIC_PATTERN-terminal/+/raw}
+      - MQTT_CLIENT_ID=${MQTT_CLIENT_ID-spatialhub-mqtt-ingestor}
+      - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90}
+      - RAW_ARCHIVE_BASE_DIR=${RAW_ARCHIVE_BASE_DIR-/data/archive/gnss}
+      - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200}
+      - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300}
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
+    volumes:
+      - mqtt-archive-data:/data/archive/gnss
+
+  mqtt-parser-worker:
+    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    restart: always
+    networks:
+      - default
+    depends_on:
+      db:
+        condition: service_healthy
+        restart: true
+      prestart:
+        condition: service_completed_successfully
+    command:
+      - python
+      - -m
+      - app.mqtt.parser_worker
+    env_file:
+      - .env
+    environment:
+      - ENVIRONMENT=${ENVIRONMENT}
+      - SECRET_KEY=${SECRET_KEY?Variable not set}
+      - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set}
+      - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set}
+      - POSTGRES_SERVER=db
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER?Variable not set}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set}
+      - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90}
+      - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200}
+      - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300}
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
+
   frontend:
     image: '${DOCKER_IMAGE_FRONTEND?Variable not set}:${TAG-latest}'
     restart: always
@@ -167,6 +244,7 @@ services:
       - traefik.http.routers.${STACK_NAME?Variable not set}-frontend-http.middlewares=https-redirect
 volumes:
   app-db-data:
+  mqtt-archive-data:
 
 networks:
   traefik-public: