156cd43a14
nginx -t resolves upstream hostnames — if the config points to a stopped slot from a previous failed deploy, the health check fails and the router stays unhealthy indefinitely, blocking all future deploys. Before up -d --wait, write the router config to point to the CURRENT live slot (which is still running) and restart the router. This clears the stale unhealthy state. After the new slot passes health checks, switch the router config to the new slot and reload. Also extracted _write_router_conf() to avoid duplicating the nginx config template. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
116 lines
4.2 KiB
Bash
Executable File
116 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
COMPOSE="docker compose -f docker-compose.prod.yml"
|
|
LIVE_FILE=".live-slot"
|
|
ROUTER_CONF="router/default.conf"
|
|
|
|
# ── Determine slots ─────────────────────────────────────────
|
|
|
|
CURRENT=$(cat "$LIVE_FILE" 2>/dev/null || echo "none")
|
|
|
|
if [ "$CURRENT" = "blue" ]; then
|
|
TARGET="green"
|
|
else
|
|
TARGET="blue"
|
|
fi
|
|
|
|
echo "==> Current: $CURRENT → Deploying: $TARGET"
|
|
|
|
# ── Build ───────────────────────────────────────────────────
|
|
|
|
echo "==> Building $TARGET..."
|
|
$COMPOSE --profile "$TARGET" build
|
|
|
|
# ── Backup DB before migration ────────────────────────────────
|
|
|
|
BACKUP_TAG="pre-deploy-$(date +%Y%m%d-%H%M%S)"
|
|
echo "==> Backing up database (${BACKUP_TAG})..."
|
|
$COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
|
|
sh -c "cp /app/data/app.db /app/data/app.db.${BACKUP_TAG} 2>/dev/null || true"
|
|
|
|
# ── Migrate ─────────────────────────────────────────────────
|
|
|
|
echo "==> Running migrations..."
|
|
$COMPOSE --profile "$TARGET" run --rm "${TARGET}-app" \
|
|
python -m padelnomics.migrations.migrate
|
|
|
|
# ── Ensure router is healthy before waiting ──────────────────
|
|
# nginx -t resolves upstream hostnames — if the config points to a stopped
|
|
# slot, the health check fails. Write config for the CURRENT live slot
|
|
# (which is still running) so the router stays healthy during --wait.
|
|
|
|
_write_router_conf() {
|
|
local SLOT="$1"
|
|
mkdir -p "$(dirname "$ROUTER_CONF")"
|
|
cat > "$ROUTER_CONF" <<NGINX
|
|
upstream app {
|
|
server ${SLOT}-app:5000;
|
|
}
|
|
|
|
server {
|
|
listen 80;
|
|
|
|
location / {
|
|
proxy_pass http://app;
|
|
proxy_set_header Host \$host;
|
|
proxy_set_header X-Real-IP \$remote_addr;
|
|
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
|
proxy_set_header X-Forwarded-Proto \$scheme;
|
|
}
|
|
}
|
|
NGINX
|
|
}
|
|
|
|
if [ "$CURRENT" != "none" ]; then
|
|
echo "==> Resetting router to current slot ($CURRENT)..."
|
|
_write_router_conf "$CURRENT"
|
|
$COMPOSE restart router
|
|
fi
|
|
|
|
# ── Start & health check ───────────────────────────────────
|
|
|
|
echo "==> Starting $TARGET (waiting for health check)..."
|
|
if ! $COMPOSE --profile "$TARGET" up -d --wait; then
|
|
echo "!!! Health check failed — dumping logs"
|
|
echo "--- ${TARGET}-app logs ---"
|
|
$COMPOSE --profile "$TARGET" logs --tail=60 "${TARGET}-app" 2>&1 || true
|
|
echo "--- router logs ---"
|
|
$COMPOSE logs --tail=10 router 2>&1 || true
|
|
echo "--- litestream logs ---"
|
|
$COMPOSE logs --tail=10 litestream 2>&1 || true
|
|
echo "!!! Rolling back"
|
|
$COMPOSE stop "${TARGET}-app" "${TARGET}-worker" "${TARGET}-scheduler"
|
|
LATEST=$($COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
|
|
sh -c "ls -t /app/data/app.db.pre-deploy-* 2>/dev/null | head -1")
|
|
if [ -n "$LATEST" ]; then
|
|
echo "==> Restoring database from ${LATEST}..."
|
|
$COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
|
|
sh -c "cp '${LATEST}' /app/data/app.db"
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# ── Write router config and reload (new slot is healthy) ────
|
|
|
|
echo "==> Switching router to $TARGET..."
|
|
_write_router_conf "$TARGET"
|
|
$COMPOSE exec router nginx -s reload
|
|
|
|
# ── Cleanup old pre-deploy backups (keep last 3) ─────────────
|
|
|
|
$COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
|
|
sh -c "ls -t /app/data/app.db.pre-deploy-* 2>/dev/null | tail -n +4 | xargs rm -f" || true
|
|
|
|
# ── Stop old slot ───────────────────────────────────────────
|
|
|
|
if [ "$CURRENT" != "none" ]; then
|
|
echo "==> Stopping $CURRENT..."
|
|
$COMPOSE stop "${CURRENT}-app" "${CURRENT}-worker" "${CURRENT}-scheduler"
|
|
fi
|
|
|
|
# ── Record live slot ────────────────────────────────────────
|
|
|
|
echo "$TARGET" > "$LIVE_FILE"
|
|
echo "==> Deployed $TARGET successfully!"
|