merge: SOPS migration + Python supervisor + docs (3 repos)

web/deploy.sh

@@ -1,6 +1,64 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# ── Ensure sops + age are installed ───────────────────────
+APP_DIR="$(cd "$(dirname "$0")" && pwd)"
+BIN_DIR="$APP_DIR/bin"
+mkdir -p "$BIN_DIR"
+export PATH="$BIN_DIR:$PATH"
+
+ARCH=$(uname -m)
+case "$ARCH" in
+    x86_64)  ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
+    aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
+    *) echo "Unsupported architecture: $ARCH"; exit 1 ;;
+esac
+
+if ! command -v age &>/dev/null; then
+    echo "==> Installing age to $BIN_DIR..."
+    AGE_VERSION="v1.3.1"
+    curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
+    tar -xzf /tmp/age.tar.gz -C "$BIN_DIR" --strip-components=1 age/age age/age-keygen
+    chmod +x "$BIN_DIR/age" "$BIN_DIR/age-keygen"
+    rm /tmp/age.tar.gz
+fi
+
+if ! command -v sops &>/dev/null; then
+    echo "==> Installing sops to $BIN_DIR..."
+    SOPS_VERSION="v3.12.1"
+    curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o "$BIN_DIR/sops"
+    chmod +x "$BIN_DIR/sops"
+fi
+
+# ── Ensure age keypair exists ─────────────────────────────
+# Key file lives at repo root (one level up from web/)
+AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$APP_DIR/../age-key.txt}"
+AGE_KEY_FILE="$(realpath "$AGE_KEY_FILE")"
+export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
+
+if [ ! -f "$AGE_KEY_FILE" ]; then
+    echo "==> Generating age keypair at $AGE_KEY_FILE..."
+    age-keygen -o "$AGE_KEY_FILE" 2>&1
+    chmod 600 "$AGE_KEY_FILE"
+    AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
+    echo ""
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo "!! NEW SERVER — add this public key to .sops.yaml:    !!"
+    echo "!!                                                     !!"
+    echo "!! $AGE_PUB !!"
+    echo "!!                                                     !!"
+    echo "!! Then run: sops updatekeys .env.prod.sops            !!"
+    echo "!! Commit, push, and re-deploy.                        !!"
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo ""
+    exit 1
+fi
+
+# ── Decrypt secrets ───────────────────────────────────────
+echo "==> Decrypting secrets from .env.prod.sops..."
+sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env"
+chmod 600 "$APP_DIR/.env"
+
 COMPOSE="docker compose -f docker-compose.prod.yml"
 LIVE_FILE=".live-slot"
 ROUTER_CONF="router/default.conf"
@@ -22,28 +80,29 @@ echo "==> Current: $CURRENT → Deploying: $TARGET"
 echo "==> Building $TARGET..."
 $COMPOSE --profile "$TARGET" build
 
+# ── Backup DB before migration ────────────────────────────────
+
+BACKUP_TAG="pre-deploy-$(date +%Y%m%d-%H%M%S)"
+echo "==> Backing up database (${BACKUP_TAG})..."
+$COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
+    sh -c "cp /app/data/app.db /app/data/app.db.${BACKUP_TAG} 2>/dev/null || true"
+
 # ── Migrate ─────────────────────────────────────────────────
 
 echo "==> Running migrations..."
 $COMPOSE --profile "$TARGET" run --rm "${TARGET}-app" \
     python -m beanflows.migrations.migrate
 
-# ── Start & health check ───────────────────────────────────
+# ── Ensure router points to current live slot before --wait ──
+# nginx resolves upstream hostnames — if config points to a stopped slot,
+# the health check fails. Reset router to current slot while target starts.
 
-echo "==> Starting $TARGET (waiting for health check)..."
-if ! $COMPOSE --profile "$TARGET" up -d --wait; then
-    echo "!!! Health check failed — rolling back"
-    $COMPOSE stop "${TARGET}-app" "${TARGET}-worker" "${TARGET}-scheduler"
-    exit 1
-fi
-
-# ── Switch router ───────────────────────────────────────────
-
-echo "==> Switching router to $TARGET..."
-mkdir -p "$(dirname "$ROUTER_CONF")"
-cat > "$ROUTER_CONF" <<NGINX
+_write_router_conf() {
+    local SLOT="$1"
+    mkdir -p "$(dirname "$ROUTER_CONF")"
+    cat > "$ROUTER_CONF" <<NGINX
 upstream app {
-    server ${TARGET}-app:5000;
+    server ${SLOT}-app:5000;
 }
 
 server {
@@ -58,11 +117,46 @@ server {
     }
 }
 NGINX
+}
 
-# Ensure router is running, then reload
-$COMPOSE up -d router
+if [ "$CURRENT" != "none" ]; then
+    echo "==> Resetting router to current slot ($CURRENT)..."
+    _write_router_conf "$CURRENT"
+    $COMPOSE restart router
+fi
+
+# ── Start & health check ───────────────────────────────────
+
+echo "==> Starting $TARGET (waiting for health check)..."
+if ! $COMPOSE --profile "$TARGET" up -d --wait; then
+    echo "!!! Health check failed — dumping logs"
+    echo "--- ${TARGET}-app logs ---"
+    $COMPOSE --profile "$TARGET" logs --tail=60 "${TARGET}-app" 2>&1 || true
+    echo "--- router logs ---"
+    $COMPOSE logs --tail=10 router 2>&1 || true
+    echo "!!! Rolling back"
+    $COMPOSE stop "${TARGET}-app" "${TARGET}-worker" "${TARGET}-scheduler"
+    LATEST=$($COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
+        sh -c "ls -t /app/data/app.db.pre-deploy-* 2>/dev/null | head -1")
+    if [ -n "$LATEST" ]; then
+        echo "==> Restoring database from ${LATEST}..."
+        $COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
+            sh -c "cp '${LATEST}' /app/data/app.db"
+    fi
+    exit 1
+fi
+
+# ── Write router config and reload (new slot is healthy) ────
+
+echo "==> Switching router to $TARGET..."
+_write_router_conf "$TARGET"
 $COMPOSE exec router nginx -s reload
 
+# ── Cleanup old pre-deploy backups (keep last 3) ─────────────
+
+$COMPOSE run --rm --entrypoint "" "${TARGET}-app" \
+    sh -c "ls -t /app/data/app.db.pre-deploy-* 2>/dev/null | tail -n +4 | xargs rm -f" || true
+
 # ── Stop old slot ───────────────────────────────────────────
 
 if [ "$CURRENT" != "none" ]; then

web/docker-compose.prod.yml

@@ -33,8 +33,10 @@ services:
     env_file: ./.env
     environment:
       - DATABASE_PATH=/app/data/app.db
+      - SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb
     volumes:
       - app-data:/app/data
+      - /data/materia/analytics.duckdb:/data/materia/analytics.duckdb:ro
     networks:
       - net
     healthcheck:
@@ -82,8 +84,10 @@ services:
     env_file: ./.env
     environment:
       - DATABASE_PATH=/app/data/app.db
+      - SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb
     volumes:
       - app-data:/app/data
+      - /data/materia/analytics.duckdb:/data/materia/analytics.duckdb:ro
     networks:
       - net
     healthcheck:

web/scripts/deploy.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-set -e
-
-# BeanFlows Deployment Script
-
-echo "🚀 Deploying BeanFlows..."
-
-# Pull latest code
-git pull origin main
-
-# Build and restart containers
-docker compose build
-docker compose up -d
-
-# Run migrations
-docker compose exec app uv run python -m beanflows.migrations.migrate
-
-# Health check
-sleep 5
-curl -f http://localhost:5000/health || exit 1
-
-echo "✅ Deployment complete!"

web/src/beanflows/core.py

@@ -1,29 +1,33 @@
 """
 Core infrastructure: database, config, email, and shared utilities.
 """
+
+import hashlib
+import hmac
 import os
 import random
 import secrets
-import hashlib
-import hmac
+from contextvars import ContextVar
+from datetime import datetime, timedelta
+from functools import wraps
+from pathlib import Path
 
 import aiosqlite
 import resend
-import httpx
-from pathlib import Path
-from functools import wraps
-from datetime import datetime, timedelta
-from contextvars import ContextVar
-from quart import g, make_response, render_template, request, session
 from dotenv import load_dotenv
+from quart import g, make_response, render_template, request, session
 
-# web/.env is three levels up from web/src/beanflows/core.py
-load_dotenv(Path(__file__).parent.parent.parent / ".env", override=False)
+# Load .env from repo root first (created by `make secrets-decrypt-dev`),
+# fall back to web/.env for legacy local dev setups.
+_repo_root = Path(__file__).parent.parent.parent.parent
+load_dotenv(_repo_root / ".env", override=False)
+load_dotenv(_repo_root / "web" / ".env", override=False)
 
 # =============================================================================
 # Configuration
 # =============================================================================
 
+
 class Config:
     APP_NAME: str = os.getenv("APP_NAME", "BeanFlows")
     SECRET_KEY: str = os.getenv("SECRET_KEY", "change-me-in-production")
@@ -53,7 +57,9 @@ class Config:
 
     ADMIN_EMAILS: list[str] = [
         e.strip().lower()
-        for e in os.getenv("ADMIN_EMAILS", "hendrik@beanflow.coffee,simon@beanflows.coffee").split(",")
+        for e in os.getenv("ADMIN_EMAILS", "hendrik@beanflow.coffee,simon@beanflows.coffee").split(
+            ","
+        )
         if e.strip()
     ]
 
@@ -66,7 +72,14 @@ class Config:
     PLAN_FEATURES: dict = {
         "free": ["dashboard", "coffee_only", "limited_history"],
         "starter": ["dashboard", "coffee_only", "full_history", "export", "api"],
-        "pro": ["dashboard", "all_commodities", "full_history", "export", "api", "priority_support"],
+        "pro": [
+            "dashboard",
+            "all_commodities",
+            "full_history",
+            "export",
+            "api",
+            "priority_support",
+        ],
     }
 
     PLAN_LIMITS: dict = {
@@ -165,6 +178,7 @@ class transaction:
             await self.db.rollback()
         return False
 
+
 # =============================================================================
 # Email
 # =============================================================================
@@ -175,8 +189,12 @@ EMAIL_ADDRESSES = {
 
 
 async def send_email(
-    to: str, subject: str, html: str, text: str = None,
-    from_addr: str = None, template: str = None,
+    to: str,
+    subject: str,
+    html: str,
+    text: str = None,
+    from_addr: str = None,
+    template: str = None,
 ) -> bool:
     """Send email via Resend SDK and log to email_log table."""
     if not config.RESEND_API_KEY:
@@ -191,13 +209,15 @@ async def send_email(
     provider_id = None
     error_msg = None
     try:
-        result = resend.Emails.send({
-            "from": from_addr or config.EMAIL_FROM,
-            "to": to,
-            "subject": subject,
-            "html": html,
-            "text": text or html,
-        })
+        result = resend.Emails.send(
+            {
+                "from": from_addr or config.EMAIL_FROM,
+                "to": to,
+                "subject": subject,
+                "html": html,
+                "text": text or html,
+            }
+        )
         provider_id = result.get("id") if isinstance(result, dict) else None
     except Exception as e:
         error_msg = str(e)
@@ -206,15 +226,24 @@ async def send_email(
     await execute(
         """INSERT INTO email_log (recipient, subject, template, status, provider_id, error, created_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)""",
-        (to, subject, template, "error" if error_msg else "sent",
-         provider_id, error_msg, datetime.utcnow().isoformat()),
+        (
+            to,
+            subject,
+            template,
+            "error" if error_msg else "sent",
+            provider_id,
+            error_msg,
+            datetime.utcnow().isoformat(),
+        ),
     )
     return error_msg is None
 
+
 # =============================================================================
 # CSRF Protection
 # =============================================================================
 
+
 def get_csrf_token() -> str:
     """Get or create CSRF token for current session."""
     if "csrf_token" not in session:
@@ -229,6 +258,7 @@ def validate_csrf_token(token: str) -> bool:
 
 def csrf_protect(f):
     """Decorator to require valid CSRF token for POST requests."""
+
     @wraps(f)
     async def decorated(*args, **kwargs):
         if request.method == "POST":
@@ -237,12 +267,15 @@ def csrf_protect(f):
             if not validate_csrf_token(token):
                 return {"error": "Invalid CSRF token"}, 403
         return await f(*args, **kwargs)
+
     return decorated
 
+
 # =============================================================================
 # Rate Limiting (SQLite-based)
 # =============================================================================
 
+
 async def check_rate_limit(key: str, limit: int = None, window: int = None) -> tuple[bool, dict]:
     """
     Check if rate limit exceeded. Returns (is_allowed, info).
@@ -255,13 +288,12 @@ async def check_rate_limit(key: str, limit: int = None, window: int = None) -> t
 
     # Clean old entries and count recent
     await execute(
-        "DELETE FROM rate_limits WHERE key = ? AND timestamp < ?",
-        (key, window_start.isoformat())
+        "DELETE FROM rate_limits WHERE key = ? AND timestamp < ?", (key, window_start.isoformat())
     )
 
     result = await fetch_one(
         "SELECT COUNT(*) as count FROM rate_limits WHERE key = ? AND timestamp > ?",
-        (key, window_start.isoformat())
+        (key, window_start.isoformat()),
     )
     count = result["count"] if result else 0
 
@@ -275,16 +307,14 @@ async def check_rate_limit(key: str, limit: int = None, window: int = None) -> t
         return False, info
 
     # Record this request
-    await execute(
-        "INSERT INTO rate_limits (key, timestamp) VALUES (?, ?)",
-        (key, now.isoformat())
-    )
+    await execute("INSERT INTO rate_limits (key, timestamp) VALUES (?, ?)", (key, now.isoformat()))
 
     return True, info
 
 
 def rate_limit(limit: int = None, window: int = None, key_func=None):
     """Decorator for rate limiting routes."""
+
     def decorator(f):
         @wraps(f)
         async def decorated(*args, **kwargs):
@@ -300,9 +330,12 @@ def rate_limit(limit: int = None, window: int = None, key_func=None):
                 return response, 429
 
             return await f(*args, **kwargs)
+
         return decorated
+
     return decorator
 
+
 # =============================================================================
 # Request ID Tracking
 # =============================================================================
@@ -317,6 +350,7 @@ def get_request_id() -> str:
 
 def setup_request_id(app):
     """Setup request ID middleware."""
+
     @app.before_request
     async def set_request_id():
         rid = request.headers.get("X-Request-ID") or secrets.token_hex(8)
@@ -328,34 +362,35 @@ def setup_request_id(app):
         response.headers["X-Request-ID"] = get_request_id()
         return response
 
+
 # =============================================================================
 # Webhook Signature Verification
 # =============================================================================
 
+
 def verify_hmac_signature(payload: bytes, signature: str, secret: str) -> bool:
     """Verify HMAC-SHA256 webhook signature."""
     expected = hmac.new(secret.encode(), payload, hashlib.sha256).hexdigest()
     return hmac.compare_digest(signature, expected)
 
+
 # =============================================================================
 # Soft Delete Helpers
 # =============================================================================
 
+
 async def soft_delete(table: str, id: int) -> bool:
     """Mark record as deleted."""
     result = await execute(
         f"UPDATE {table} SET deleted_at = ? WHERE id = ? AND deleted_at IS NULL",
-        (datetime.utcnow().isoformat(), id)
+        (datetime.utcnow().isoformat(), id),
     )
     return result > 0
 
 
 async def restore(table: str, id: int) -> bool:
     """Restore soft-deleted record."""
-    result = await execute(
-        f"UPDATE {table} SET deleted_at = NULL WHERE id = ?",
-        (id,)
-    )
+    result = await execute(f"UPDATE {table} SET deleted_at = NULL WHERE id = ?", (id,))
     return result > 0
 
 
@@ -369,8 +404,7 @@ async def purge_deleted(table: str, days: int = 30) -> int:
     """Purge records deleted more than X days ago."""
     cutoff = (datetime.utcnow() - timedelta(days=days)).isoformat()
     return await execute(
-        f"DELETE FROM {table} WHERE deleted_at IS NOT NULL AND deleted_at < ?",
-        (cutoff,)
+        f"DELETE FROM {table} WHERE deleted_at IS NOT NULL AND deleted_at < ?", (cutoff,)
     )
 
 
@@ -378,8 +412,10 @@ async def purge_deleted(table: str, days: int = 30) -> int:
 # Waitlist
 # =============================================================================
 
+
 def waitlist_gate(template: str, **extra_context):
     """Intercept GET requests when WAITLIST_MODE=true and render the waitlist template."""
+
     def decorator(f):
         @wraps(f)
         async def wrapper(*args, **kwargs):
@@ -389,7 +425,9 @@ def waitlist_gate(template: str, **extra_context):
                     ctx[k] = v() if callable(v) else v
                 return await render_template(template, **ctx)
             return await f(*args, **kwargs)
+
         return wrapper
+
     return decorator
 
 
@@ -411,16 +449,19 @@ async def capture_waitlist_email(
 
     if result:
         from .worker import enqueue
+
         await enqueue("send_waitlist_confirmation", {"email": email})
 
         if config.RESEND_AUDIENCE_WAITLIST and config.RESEND_API_KEY:
             try:
                 resend.api_key = config.RESEND_API_KEY
-                resend.Contacts.create({
-                    "email": email,
-                    "audience_id": config.RESEND_AUDIENCE_WAITLIST,
-                    "unsubscribed": False,
-                })
+                resend.Contacts.create(
+                    {
+                        "email": email,
+                        "audience_id": config.RESEND_AUDIENCE_WAITLIST,
+                        "unsubscribed": False,
+                    }
+                )
             except Exception as e:
                 print(f"[WAITLIST] Resend audience error: {e}")
 
@@ -432,8 +473,10 @@ async def capture_waitlist_email(
 # A/B Testing
 # =============================================================================
 
+
 def ab_test(experiment: str, variants: tuple = ("control", "treatment")):
     """Assign visitor to an A/B test variant via cookie, tag Umami pageviews."""
+
     def decorator(f):
         @wraps(f)
         async def wrapper(*args, **kwargs):
@@ -448,7 +491,9 @@ def ab_test(experiment: str, variants: tuple = ("control", "treatment")):
             response = await make_response(await f(*args, **kwargs))
             response.set_cookie(cookie_key, assigned, max_age=30 * 24 * 60 * 60)
             return response
+
         return wrapper
+
     return decorator
 
 
@@ -456,6 +501,7 @@ def ab_test(experiment: str, variants: tuple = ("control", "treatment")):
 # Feature Flags (DB-backed, admin-toggleable)
 # =============================================================================
 
+
 async def is_flag_enabled(name: str, default: bool = False) -> bool:
     """Check if a feature flag is enabled. Falls back to default if not found."""
     row = await fetch_one("SELECT enabled FROM feature_flags WHERE name = ?", (name,))
@@ -482,6 +528,7 @@ async def get_all_flags() -> list[dict]:
 
 def feature_gate(flag_name: str, fallback_template: str, **extra_context):
     """Gate a route behind a feature flag; renders fallback on GET, 403 on POST."""
+
     def decorator(f):
         @wraps(f)
         async def decorated(*args, **kwargs):
@@ -491,5 +538,7 @@ def feature_gate(flag_name: str, fallback_template: str, **extra_context):
                     return await render_template(fallback_template, **ctx)
                 return {"error": "Feature not available"}, 403
             return await f(*args, **kwargs)
+
         return decorated
-    return decorator
+
+    return decorator