From 8717b9c07b1a8c59c05aad0e09dfaafb38d2be9e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 17 Mar 2026 16:22:44 +0800 Subject: [PATCH] deploy: run mqtt workers in prod and default to EMQX on 1panel network --- .gitea/workflows/deploy.yml | 37 +++++++++++++++++++ compose.prod.yml | 74 +++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ea943ab..2eefed8 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -40,6 +40,16 @@ jobs: POSTGRES_USER=${{ secrets.POSTGRES_USER }} POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} SENTRY_DSN=${{ secrets.SENTRY_DSN }} + MQTT_HOST=${{ secrets.MQTT_HOST }} + MQTT_PORT=${{ secrets.MQTT_PORT }} + MQTT_USERNAME=${{ secrets.MQTT_USERNAME }} + MQTT_PASSWORD=${{ secrets.MQTT_PASSWORD }} + MQTT_TOPIC_PATTERN=${{ secrets.MQTT_TOPIC_PATTERN }} + MQTT_CLIENT_ID=${{ secrets.MQTT_CLIENT_ID }} + RAW_HOT_RETENTION_DAYS=${{ secrets.RAW_HOT_RETENTION_DAYS }} + RAW_ARCHIVE_BASE_DIR=${{ secrets.RAW_ARCHIVE_BASE_DIR }} + PARSER_BATCH_SIZE=${{ secrets.PARSER_BATCH_SIZE }} + PARSER_POLL_INTERVAL_MS=${{ secrets.PARSER_POLL_INTERVAL_MS }} DOCKER_IMAGE_BACKEND=${{ secrets.DOCKER_IMAGE_BACKEND }} DOCKER_IMAGE_FRONTEND=${{ secrets.DOCKER_IMAGE_FRONTEND }} ENVEOF @@ -47,6 +57,16 @@ jobs: # Fallback defaults if image name secrets are empty sed -i 's/^DOCKER_IMAGE_BACKEND=$/DOCKER_IMAGE_BACKEND=backend/' .env.production sed -i 's/^DOCKER_IMAGE_FRONTEND=$/DOCKER_IMAGE_FRONTEND=frontend/' .env.production + sed -i 's/^MQTT_HOST=$/MQTT_HOST=1Panel-emqx-Rniy/' .env.production + sed -i 's/^MQTT_PORT=$/MQTT_PORT=1883/' .env.production + sed -i 's/^MQTT_USERNAME=$/MQTT_USERNAME=weefnn/' .env.production + sed -i 's/^MQTT_PASSWORD=$/MQTT_PASSWORD=123456/' .env.production + sed -i 's%^MQTT_TOPIC_PATTERN=$%MQTT_TOPIC_PATTERN=terminal/+/raw%' .env.production + sed -i 's/^MQTT_CLIENT_ID=$/MQTT_CLIENT_ID=spatialhub-mqtt-ingestor/' .env.production + sed -i 's/^RAW_HOT_RETENTION_DAYS=$/RAW_HOT_RETENTION_DAYS=90/' .env.production + sed -i 's%^RAW_ARCHIVE_BASE_DIR=$%RAW_ARCHIVE_BASE_DIR=/data/archive/gnss%' .env.production + sed -i 's/^PARSER_BATCH_SIZE=$/PARSER_BATCH_SIZE=200/' .env.production + sed -i 's/^PARSER_POLL_INTERVAL_MS=$/PARSER_POLL_INTERVAL_MS=300/' .env.production - name: Build Docker images run: docker compose --env-file .env.production -f compose.prod.yml build @@ -83,5 +103,22 @@ jobs: exit 1 fi + - name: Verify mqtt workers + run: | + if docker compose --env-file .env.production -f compose.prod.yml ps mqtt-ingestor --status running | grep -q mqtt-ingestor; then + echo "✅ mqtt-ingestor is running!" + else + echo "❌ mqtt-ingestor is not running" + docker compose --env-file .env.production -f compose.prod.yml logs mqtt-ingestor + exit 1 + fi + if docker compose --env-file .env.production -f compose.prod.yml ps mqtt-parser-worker --status running | grep -q mqtt-parser-worker; then + echo "✅ mqtt-parser-worker is running!" + else + echo "❌ mqtt-parser-worker is not running" + docker compose --env-file .env.production -f compose.prod.yml logs mqtt-parser-worker + exit 1 + fi + - name: Cleanup old Docker images run: docker image prune -f || true diff --git a/compose.prod.yml b/compose.prod.yml index 72984e5..2e46cda 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -70,6 +70,77 @@ services: context: . dockerfile: backend/Dockerfile + mqtt-ingestor: + image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}' + restart: always + networks: + - 1panel-network + depends_on: + prestart: + condition: service_completed_successfully + env_file: + - .env.production + environment: + - ENVIRONMENT=${ENVIRONMENT} + - SECRET_KEY=${SECRET_KEY?Variable not set} + - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set} + - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set} + - POSTGRES_SERVER=${POSTGRES_SERVER} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER?Variable not set} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set} + - MQTT_HOST=${MQTT_HOST-1Panel-emqx-Rniy} + - MQTT_PORT=${MQTT_PORT-1883} + - MQTT_USERNAME=${MQTT_USERNAME} + - MQTT_PASSWORD=${MQTT_PASSWORD} + - MQTT_TOPIC_PATTERN=${MQTT_TOPIC_PATTERN-terminal/+/raw} + - MQTT_CLIENT_ID=${MQTT_CLIENT_ID-spatialhub-mqtt-ingestor} + - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90} + - RAW_ARCHIVE_BASE_DIR=${RAW_ARCHIVE_BASE_DIR-/data/archive/gnss} + - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200} + - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300} + command: + - python + - -m + - app.mqtt.ingestor + build: + context: . + dockerfile: backend/Dockerfile + volumes: + - mqtt-archive-data:/data/archive/gnss + + mqtt-parser-worker: + image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}' + restart: always + networks: + - 1panel-network + depends_on: + prestart: + condition: service_completed_successfully + env_file: + - .env.production + environment: + - ENVIRONMENT=${ENVIRONMENT} + - SECRET_KEY=${SECRET_KEY?Variable not set} + - FIRST_SUPERUSER=${FIRST_SUPERUSER?Variable not set} + - FIRST_SUPERUSER_PASSWORD=${FIRST_SUPERUSER_PASSWORD?Variable not set} + - POSTGRES_SERVER=${POSTGRES_SERVER} + - POSTGRES_PORT=${POSTGRES_PORT} + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER?Variable not set} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD?Variable not set} + - RAW_HOT_RETENTION_DAYS=${RAW_HOT_RETENTION_DAYS-90} + - PARSER_BATCH_SIZE=${PARSER_BATCH_SIZE-200} + - PARSER_POLL_INTERVAL_MS=${PARSER_POLL_INTERVAL_MS-300} + command: + - python + - -m + - app.mqtt.parser_worker + build: + context: . + dockerfile: backend/Dockerfile + frontend: image: '${DOCKER_IMAGE_FRONTEND?Variable not set}:${TAG-latest}' restart: always @@ -87,3 +158,6 @@ services: networks: 1panel-network: external: true + +volumes: + mqtt-archive-data: