deploy: run mqtt workers in prod and default to EMQX on 1panel network

.gitea/workflows/deploy.yml

@@ -40,6 +40,16 @@ jobs:
           POSTGRES_USER=${{ secrets.POSTGRES_USER }}
           POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
           SENTRY_DSN=${{ secrets.SENTRY_DSN }}
+          MQTT_HOST=${{ secrets.MQTT_HOST }}
+          MQTT_PORT=${{ secrets.MQTT_PORT }}
+          MQTT_USERNAME=${{ secrets.MQTT_USERNAME }}
+          MQTT_PASSWORD=${{ secrets.MQTT_PASSWORD }}
+          MQTT_TOPIC_PATTERN=${{ secrets.MQTT_TOPIC_PATTERN }}
+          MQTT_CLIENT_ID=${{ secrets.MQTT_CLIENT_ID }}
+          RAW_HOT_RETENTION_DAYS=${{ secrets.RAW_HOT_RETENTION_DAYS }}
+          RAW_ARCHIVE_BASE_DIR=${{ secrets.RAW_ARCHIVE_BASE_DIR }}
+          PARSER_BATCH_SIZE=${{ secrets.PARSER_BATCH_SIZE }}
+          PARSER_POLL_INTERVAL_MS=${{ secrets.PARSER_POLL_INTERVAL_MS }}
           DOCKER_IMAGE_BACKEND=${{ secrets.DOCKER_IMAGE_BACKEND }}
           DOCKER_IMAGE_FRONTEND=${{ secrets.DOCKER_IMAGE_FRONTEND }}
           ENVEOF
@@ -47,6 +57,16 @@ jobs:
           # Fallback defaults if image name secrets are empty
           sed -i 's/^DOCKER_IMAGE_BACKEND=$/DOCKER_IMAGE_BACKEND=backend/' .env.production
           sed -i 's/^DOCKER_IMAGE_FRONTEND=$/DOCKER_IMAGE_FRONTEND=frontend/' .env.production
+          sed -i 's/^MQTT_HOST=$/MQTT_HOST=1Panel-emqx-Rniy/' .env.production
+          sed -i 's/^MQTT_PORT=$/MQTT_PORT=1883/' .env.production
+          sed -i 's/^MQTT_USERNAME=$/MQTT_USERNAME=weefnn/' .env.production
+          sed -i 's/^MQTT_PASSWORD=$/MQTT_PASSWORD=123456/' .env.production
+          sed -i 's%^MQTT_TOPIC_PATTERN=$%MQTT_TOPIC_PATTERN=terminal/+/raw%' .env.production
+          sed -i 's/^MQTT_CLIENT_ID=$/MQTT_CLIENT_ID=spatialhub-mqtt-ingestor/' .env.production
+          sed -i 's/^RAW_HOT_RETENTION_DAYS=$/RAW_HOT_RETENTION_DAYS=90/' .env.production
+          sed -i 's%^RAW_ARCHIVE_BASE_DIR=$%RAW_ARCHIVE_BASE_DIR=/data/archive/gnss%' .env.production
+          sed -i 's/^PARSER_BATCH_SIZE=$/PARSER_BATCH_SIZE=200/' .env.production
+          sed -i 's/^PARSER_POLL_INTERVAL_MS=$/PARSER_POLL_INTERVAL_MS=300/' .env.production
 
       - name: Build Docker images
         run: docker compose --env-file .env.production -f compose.prod.yml build
@@ -83,5 +103,22 @@ jobs:
             exit 1
           fi
 
+      - name: Verify mqtt workers
+        run: |
+          if docker compose --env-file .env.production -f compose.prod.yml ps mqtt-ingestor --status running | grep -q mqtt-ingestor; then
+            echo "✅ mqtt-ingestor is running!"
+          else
+            echo "❌ mqtt-ingestor is not running"
+            docker compose --env-file .env.production -f compose.prod.yml logs mqtt-ingestor
+            exit 1
+          fi
+          if docker compose --env-file .env.production -f compose.prod.yml ps mqtt-parser-worker --status running | grep -q mqtt-parser-worker; then
+            echo "✅ mqtt-parser-worker is running!"
+          else
+            echo "❌ mqtt-parser-worker is not running"
+            docker compose --env-file .env.production -f compose.prod.yml logs mqtt-parser-worker
+            exit 1
+          fi
+
       - name: Cleanup old Docker images
         run: docker image prune -f || true

compose.prod.yml

@@ -70,6 +70,77 @@ services:
       context: .
       dockerfile: backend/Dockerfile
 
+  mqtt-ingestor:
+    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    restart: always
+    networks:
+      - 1panel-network
+    depends_on:
+      prestart:
+        condition: service_completed_successfully
+    env_file:
+      - .env.production
+    environment:
+      - ENVIRONMENT=${ENVIRONMENT}
+      - SECRET_KEY=${SECRET_KEY?Variable not set}
+      - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set}
+      - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set}
+      - POSTGRES_SERVER=${POSTGRES_SERVER}
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER?Variable not set}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set}
+      - MQTT_HOST=${MQTT_HOST-1Panel-emqx-Rniy}
+      - MQTT_PORT=${MQTT_PORT-1883}
+      - MQTT_USERNAME=${MQTT_USERNAME}
+      - MQTT_PASSWORD=${MQTT_PASSWORD}
+      - MQTT_TOPIC_PATTERN=${MQTT_TOPIC_PATTERN-terminal/+/raw}
+      - MQTT_CLIENT_ID=${MQTT_CLIENT_ID-spatialhub-mqtt-ingestor}
+      - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90}
+      - RAW_ARCHIVE_BASE_DIR=${RAW_ARCHIVE_BASE_DIR-/data/archive/gnss}
+      - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200}
+      - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300}
+    command:
+      - python
+      - -m
+      - app.mqtt.ingestor
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
+    volumes:
+      - mqtt-archive-data:/data/archive/gnss
+
+  mqtt-parser-worker:
+    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    restart: always
+    networks:
+      - 1panel-network
+    depends_on:
+      prestart:
+        condition: service_completed_successfully
+    env_file:
+      - .env.production
+    environment:
+      - ENVIRONMENT=${ENVIRONMENT}
+      - SECRET_KEY=${SECRET_KEY?Variable not set}
+      - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set}
+      - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set}
+      - POSTGRES_SERVER=${POSTGRES_SERVER}
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER?Variable not set}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set}
+      - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90}
+      - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200}
+      - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300}
+    command:
+      - python
+      - -m
+      - app.mqtt.parser_worker
+    build:
+      context: .
+      dockerfile: backend/Dockerfile
+
   frontend:
     image: '${DOCKER_IMAGE_FRONTEND?Variable not set}:${TAG-latest}'
     restart: always
@@ -87,3 +158,6 @@ services:
 networks:
   1panel-network:
     external: true
+
+volumes:
+  mqtt-archive-data: