Refactor to git-based deployment: simplify CI/CD and supervisor

Addresses GitLab PR comments:
1. Remove hardcoded secrets from Pulumi.prod.yaml, use ESC environment
2. Simplify deployment by using git pull instead of R2 artifacts
3. Add bootstrap script for one-time supervisor setup

Major changes:
- **Pulumi config**: Use ESC environment (beanflows/prod) for all secrets
- **Supervisor script**: Git-based deployment (git pull every 15 min)
  * No more artifact downloads from R2
  * Runs code directly via `uv run materia`
  * Self-updating from master branch
- **Bootstrap script**: New infra/bootstrap_supervisor.sh for initial setup
  * One-time script to clone repo and setup systemd service
  * Idempotent and simple
- **CI/CD simplification**: Remove build and R2 deployment stages
  * Eliminated build:extract, build:transform, build:cli jobs
  * Eliminated deploy:r2 job
  * Simplified deploy:supervisor to just check bootstrap status
  * Reduced from 4 stages to 3 stages (Lint → Test → Deploy)
- **Documentation**: Updated CLAUDE.md with new architecture
  * Git-based deployment flow
  * Bootstrap instructions
  * Simplified execution model

Benefits:
- ✅ No hardcoded secrets in config files
- ✅ Simpler deployment (no artifact builds)
- ✅ Easy to test locally (just git clone + uv sync)
- ✅ Auto-updates every 15 minutes
- ✅ Fewer CI/CD jobs (faster pipelines)
- ✅ Cleaner separation of concerns

Inspired by TigerBeetle's CFO supervisor pattern.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

infra/Pulumi.prod.yaml

@@ -1,5 +1,7 @@
+# Production stack configuration
+# All secrets come from Pulumi ESC environment: beanflows/prod
+environment:
+  - beanflows/prod
+
 config:
-  hcloud:token:
-    secure: AAABAEdhCpoRPhSknCQDgJWRFUjqwyM7TIz60ICRfcpy2GcYeFH098aX/3/rPCJCuetsRma0Wa145Ff3XXIEgUHFJ4Xr9/fZTZtlAtfMROaEhukWL19k96Fh6m8JihMl
-  materia-infrastructure:ssh_public_key:
-    secure: AAABAERKCdqTMBjaxXE+AzlVlCCxUkF1R7+1kFo7c69gqQt1JQuuvzAL/16f099iMP0Ij97U45VBpKUrMtZfHy68d1w1hyCueMHwhoOsfN7bLpj4R/DdCsupXfs8Vx/bJtBjIvsPKbK7f+DygWM1RA==
+  materia-infrastructure:hetzner_location: "nbg1"  # Nuremberg, Germany

infra/bootstrap_supervisor.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Bootstrap script for Materia supervisor instance
+# Run this once on a new supervisor to set it up
+#
+# Usage:
+#   From CI/CD or locally:
+#   ssh root@<supervisor_ip> 'bash -s' < infra/bootstrap_supervisor.sh
+#
+# Or on the supervisor itself:
+#   curl -fsSL <url-to-this-script> | bash
+
+set -euo pipefail
+
+echo "=== Materia Supervisor Bootstrap ==="
+echo "This script will:"
+echo "  1. Install dependencies (git, uv, esc)"
+echo "  2. Clone the materia repository"
+echo "  3. Setup systemd service"
+echo "  4. Start the supervisor"
+echo ""
+
+# Check if we're root
+if [ "$EUID" -ne 0 ]; then
+    echo "ERROR: This script must be run as root"
+    exit 1
+fi
+
+# Configuration
+REPO_URL="${REPO_URL:-https://gitlab.com/YOUR_USERNAME/materia.git}"  # TODO: Update this!
+MATERIA_DIR="/opt/materia"
+REPO_DIR="$MATERIA_DIR/repo"
+
+echo "--- Installing system dependencies ---"
+apt-get update
+apt-get install -y git curl python3-pip
+
+echo "--- Installing uv ---"
+if ! command -v uv &> /dev/null; then
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    export PATH="$HOME/.cargo/bin:$PATH"
+    echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /root/.bashrc
+fi
+
+echo "--- Installing Pulumi ESC ---"
+if ! command -v esc &> /dev/null; then
+    curl -fsSL https://get.pulumi.com/esc/install.sh | sh
+    export PATH="$HOME/.pulumi/bin:$PATH"
+    echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
+fi
+
+echo "--- Setting up Pulumi ESC authentication ---"
+if [ -z "${PULUMI_ACCESS_TOKEN:-}" ]; then
+    echo "ERROR: PULUMI_ACCESS_TOKEN environment variable not set"
+    echo "Please set it before running this script:"
+    echo "  export PULUMI_ACCESS_TOKEN=<your-token>"
+    exit 1
+fi
+
+esc login --token "$PULUMI_ACCESS_TOKEN"
+
+echo "--- Loading secrets from Pulumi ESC ---"
+eval $(esc env open beanflows/prod --format shell)
+
+echo "--- Cloning repository ---"
+mkdir -p "$MATERIA_DIR"
+if [ -d "$REPO_DIR" ]; then
+    echo "Repository already exists, pulling latest..."
+    cd "$REPO_DIR"
+    git pull origin master
+else
+    cd "$MATERIA_DIR"
+    git clone "$REPO_URL" repo
+    cd repo
+fi
+
+echo "--- Installing Python dependencies ---"
+uv sync
+
+echo "--- Creating environment file ---"
+cat > "$MATERIA_DIR/.env" <<EOF
+# Environment variables for supervisor
+# Loaded from Pulumi ESC: beanflows/prod
+PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
+PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
+EOF
+
+echo "--- Setting up systemd service ---"
+cat > /etc/systemd/system/materia-supervisor.service <<'EOF'
+[Unit]
+Description=Materia Supervisor - Pipeline Orchestration
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/materia/repo
+ExecStart=/opt/materia/repo/infra/supervisor/supervisor.sh
+Restart=always
+RestartSec=10
+EnvironmentFile=/opt/materia/.env
+
+# Resource limits
+LimitNOFILE=65536
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=materia-supervisor
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "--- Enabling and starting service ---"
+systemctl daemon-reload
+systemctl enable materia-supervisor
+systemctl start materia-supervisor
+
+echo ""
+echo "=== Bootstrap complete! ==="
+echo ""
+echo "Supervisor is now running. Check status with:"
+echo "  systemctl status materia-supervisor"
+echo ""
+echo "View logs with:"
+echo "  journalctl -u materia-supervisor -f"
+echo ""
+echo "Repository location: $REPO_DIR"
+echo "Current commit: $(cd $REPO_DIR && git rev-parse --short HEAD)"

infra/supervisor/supervisor.sh

@@ -2,24 +2,22 @@
 # Materia Supervisor - Continuous pipeline orchestration
 # Inspired by TigerBeetle's CFO supervisor pattern
 # https://github.com/tigerbeetle/tigerbeetle/blob/main/src/scripts/cfo_supervisor.sh
+#
+# Git-based deployment: pulls latest code from master and runs pipelines via uv
 
 set -euo pipefail
 
 # Configuration
 readonly CHECK_INTERVAL=900  # 15 minutes
-readonly CLI_VERSION_CHECK_INTERVAL=3600  # 1 hour
-readonly MATERIA_DIR="/opt/materia"
-readonly R2_ARTIFACTS_URL="https://${R2_ENDPOINT}/${R2_ARTIFACTS_BUCKET}"
-readonly CLI_ARTIFACT="materia-cli-latest.tar.gz"
+readonly MATERIA_REPO="/opt/materia/repo"
+readonly STATE_DIR="/var/lib/materia"
 
 # Schedules (cron-style times in UTC)
 readonly EXTRACT_SCHEDULE_HOUR=2  # 02:00 UTC
 readonly TRANSFORM_SCHEDULE_HOUR=3  # 03:00 UTC
 
-# State tracking
-last_extract_run=""
-last_transform_run=""
-last_cli_check=0
+# Ensure state directory exists
+mkdir -p "$STATE_DIR"
 
 log() {
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
@@ -29,151 +27,121 @@ log_error() {
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
 }
 
-# Check if CLI needs updating
-check_cli_update() {
-    local now
-    now=$(date +%s)
+# Update code from git
+update_code() {
+    log "Checking for code updates..."
+    cd "$MATERIA_REPO"
 
-    # Only check once per hour
-    if (( now - last_cli_check < CLI_VERSION_CHECK_INTERVAL )); then
-        return 0
-    fi
-
-    last_cli_check=$now
-
-    log "Checking for CLI updates..."
-
-    # Download new version
-    local temp_file="${MATERIA_DIR}/cli-new.tar.gz"
-    if ! curl -fsSL -o "$temp_file" "${R2_ARTIFACTS_URL}/${CLI_ARTIFACT}"; then
-        log_error "Failed to download CLI artifact"
+    # Fetch latest from master
+    if ! git fetch origin master 2>&1 | grep -v "^From"; then
+        log_error "Failed to fetch from git"
         return 1
     fi
 
-    # Compare checksums
-    local old_checksum=""
-    local new_checksum
+    # Check if update available
+    LOCAL=$(git rev-parse HEAD)
+    REMOTE=$(git rev-parse origin/master)
 
-    if [ -f "${MATERIA_DIR}/${CLI_ARTIFACT}" ]; then
-        old_checksum=$(sha256sum "${MATERIA_DIR}/${CLI_ARTIFACT}" | awk '{print $1}')
+    if [ "$LOCAL" != "$REMOTE" ]; then
+        log "New version detected: $LOCAL -> $REMOTE"
+
+        # Pull latest code
+        if git pull origin master; then
+            log "Code updated successfully"
+
+            # Update dependencies
+            log "Updating dependencies with uv sync..."
+            if uv sync; then
+                log "Dependencies updated"
+                return 0
+            else
+                log_error "Failed to update dependencies"
+                return 1
+            fi
+        else
+            log_error "Failed to pull code"
+            return 1
+        fi
     fi
 
-    new_checksum=$(sha256sum "$temp_file" | awk '{print $1}')
+    log "Already up to date at $(git rev-parse --short HEAD)"
+    return 1  # Return 1 to indicate no update (not an error)
+}
 
-    if [ "$old_checksum" = "$new_checksum" ]; then
-        log "CLI is up to date"
-        rm -f "$temp_file"
+# Run pipeline using materia CLI via uv
+run_pipeline() {
+    local pipeline=$1
+    local date=$(date -u +%Y-%m-%d)
+    local state_file="$STATE_DIR/${pipeline}_last_run"
+
+    log "Running $pipeline pipeline..."
+
+    cd "$MATERIA_REPO"
+    if uv run materia pipeline run "$pipeline"; then
+        log "$pipeline completed successfully"
+        echo "$date" > "$state_file"
         return 0
-    fi
-
-    log "New CLI version detected, updating..."
-
-    # Install new version
-    mv "$temp_file" "${MATERIA_DIR}/${CLI_ARTIFACT}"
-
-    cd "$MATERIA_DIR"
-    rm -rf cli && mkdir -p cli
-    tar -xzf "$CLI_ARTIFACT" -C cli/
-
-    if pip3 install --force-reinstall cli/*.whl; then
-        log "CLI updated successfully"
-        materia version
     else
-        log_error "Failed to install CLI"
+        log_error "$pipeline failed"
         return 1
     fi
 }
 
-# Check if we should run extract pipeline (daily at specified hour)
-should_run_extract() {
-    local current_hour
-    local current_date
-
-    current_hour=$(date -u +%H)
-    current_date=$(date -u +%Y-%m-%d)
+# Check if pipeline should run today
+should_run_pipeline() {
+    local pipeline=$1
+    local schedule_hour=$2
+    local current_hour=$(date -u +%H)
+    local current_date=$(date -u +%Y-%m-%d)
+    local state_file="$STATE_DIR/${pipeline}_last_run"
 
     # Only run at the scheduled hour
-    if [ "$current_hour" != "$EXTRACT_SCHEDULE_HOUR" ]; then
+    if [ "$current_hour" -ne "$schedule_hour" ]; then
         return 1
     fi
 
-    # Only run once per day
-    if [ "$last_extract_run" = "$current_date" ]; then
-        return 1
+    # Check if already ran today
+    if [ -f "$state_file" ]; then
+        local last_run=$(cat "$state_file")
+        if [ "$last_run" = "$current_date" ]; then
+            return 1  # Already ran today
+        fi
     fi
 
-    return 0
-}
-
-# Check if we should run transform pipeline (daily at specified hour)
-should_run_transform() {
-    local current_hour
-    local current_date
-
-    current_hour=$(date -u +%H)
-    current_date=$(date -u +%Y-%m-%d)
-
-    # Only run at the scheduled hour
-    if [ "$current_hour" != "$TRANSFORM_SCHEDULE_HOUR" ]; then
-        return 1
-    fi
-
-    # Only run once per day
-    if [ "$last_transform_run" = "$current_date" ]; then
-        return 1
-    fi
-
-    return 0
-}
-
-# Run extract pipeline
-run_extract() {
-    log "Starting extract pipeline..."
-
-    if materia pipeline run extract; then
-        log "Extract pipeline completed successfully"
-        last_extract_run=$(date -u +%Y-%m-%d)
-    else
-        log_error "Extract pipeline failed"
-        return 1
-    fi
-}
-
-# Run transform pipeline
-run_transform() {
-    log "Starting transform pipeline..."
-
-    if materia pipeline run transform; then
-        log "Transform pipeline completed successfully"
-        last_transform_run=$(date -u +%Y-%m-%d)
-    else
-        log_error "Transform pipeline failed"
-        return 1
-    fi
+    return 0  # Should run
 }
 
 # Main supervisor loop
 main() {
     log "Materia supervisor starting..."
+    log "Repository: $MATERIA_REPO"
     log "Extract schedule: daily at ${EXTRACT_SCHEDULE_HOUR}:00 UTC"
     log "Transform schedule: daily at ${TRANSFORM_SCHEDULE_HOUR}:00 UTC"
     log "Check interval: ${CHECK_INTERVAL}s"
 
-    # Initial CLI check
-    check_cli_update || log_error "Initial CLI check failed, continuing anyway"
+    # Ensure repo exists
+    if [ ! -d "$MATERIA_REPO/.git" ]; then
+        log_error "Repository not found at $MATERIA_REPO"
+        log_error "Run bootstrap script first!"
+        exit 1
+    fi
+
+    # Show initial version
+    cd "$MATERIA_REPO"
+    log "Starting at commit: $(git rev-parse --short HEAD)"
 
     while true; do
-        # Check for CLI updates
-        check_cli_update || true
+        # Check for code updates every loop
+        update_code || true
 
-        # Check and run extract pipeline
-        if should_run_extract; then
-            run_extract || true
+        # Check extract schedule
+        if should_run_pipeline "extract" "$EXTRACT_SCHEDULE_HOUR"; then
+            run_pipeline extract || true
         fi
 
-        # Check and run transform pipeline
-        if should_run_transform; then
-            run_transform || true
+        # Check transform schedule
+        if should_run_pipeline "transform" "$TRANSFORM_SCHEDULE_HOUR"; then
+            run_pipeline transform || true
         fi
 
         sleep "$CHECK_INTERVAL"