Refactor to git-based deployment: simplify CI/CD and supervisor

Addresses GitLab PR comments:
1. Remove hardcoded secrets from Pulumi.prod.yaml, use ESC environment
2. Simplify deployment by using git pull instead of R2 artifacts
3. Add bootstrap script for one-time supervisor setup

Major changes:
- **Pulumi config**: Use ESC environment (beanflows/prod) for all secrets
- **Supervisor script**: Git-based deployment (git pull every 15 min)
  * No more artifact downloads from R2
  * Runs code directly via `uv run materia`
  * Self-updating from master branch
- **Bootstrap script**: New infra/bootstrap_supervisor.sh for initial setup
  * One-time script to clone repo and setup systemd service
  * Idempotent and simple
- **CI/CD simplification**: Remove build and R2 deployment stages
  * Eliminated build:extract, build:transform, build:cli jobs
  * Eliminated deploy:r2 job
  * Simplified deploy:supervisor to just check bootstrap status
  * Reduced from 4 stages to 3 stages (Lint → Test → Deploy)
- **Documentation**: Updated CLAUDE.md with new architecture
  * Git-based deployment flow
  * Bootstrap instructions
  * Simplified execution model

Benefits:
- ✅ No hardcoded secrets in config files
- ✅ Simpler deployment (no artifact builds)
- ✅ Easy to test locally (just git clone + uv sync)
- ✅ Auto-updates every 15 minutes
- ✅ Fewer CI/CD jobs (faster pipelines)
- ✅ Cleaner separation of concerns

Inspired by TigerBeetle's CFO supervisor pattern.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

infra/supervisor/supervisor.sh

@@ -2,24 +2,22 @@
 # Materia Supervisor - Continuous pipeline orchestration
 # Inspired by TigerBeetle's CFO supervisor pattern
 # https://github.com/tigerbeetle/tigerbeetle/blob/main/src/scripts/cfo_supervisor.sh
+#
+# Git-based deployment: pulls latest code from master and runs pipelines via uv
 
 set -euo pipefail
 
 # Configuration
 readonly CHECK_INTERVAL=900  # 15 minutes
-readonly CLI_VERSION_CHECK_INTERVAL=3600  # 1 hour
-readonly MATERIA_DIR="/opt/materia"
-readonly R2_ARTIFACTS_URL="https://${R2_ENDPOINT}/${R2_ARTIFACTS_BUCKET}"
-readonly CLI_ARTIFACT="materia-cli-latest.tar.gz"
+readonly MATERIA_REPO="/opt/materia/repo"
+readonly STATE_DIR="/var/lib/materia"
 
 # Schedules (cron-style times in UTC)
 readonly EXTRACT_SCHEDULE_HOUR=2  # 02:00 UTC
 readonly TRANSFORM_SCHEDULE_HOUR=3  # 03:00 UTC
 
-# State tracking
-last_extract_run=""
-last_transform_run=""
-last_cli_check=0
+# Ensure state directory exists
+mkdir -p "$STATE_DIR"
 
 log() {
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
@@ -29,151 +27,121 @@ log_error() {
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
 }
 
-# Check if CLI needs updating
-check_cli_update() {
-    local now
-    now=$(date +%s)
+# Update code from git
+update_code() {
+    log "Checking for code updates..."
+    cd "$MATERIA_REPO"
 
-    # Only check once per hour
-    if (( now - last_cli_check < CLI_VERSION_CHECK_INTERVAL )); then
-        return 0
-    fi
-
-    last_cli_check=$now
-
-    log "Checking for CLI updates..."
-
-    # Download new version
-    local temp_file="${MATERIA_DIR}/cli-new.tar.gz"
-    if ! curl -fsSL -o "$temp_file" "${R2_ARTIFACTS_URL}/${CLI_ARTIFACT}"; then
-        log_error "Failed to download CLI artifact"
+    # Fetch latest from master
+    if ! git fetch origin master 2>&1 | grep -v "^From"; then
+        log_error "Failed to fetch from git"
         return 1
     fi
 
-    # Compare checksums
-    local old_checksum=""
-    local new_checksum
+    # Check if update available
+    LOCAL=$(git rev-parse HEAD)
+    REMOTE=$(git rev-parse origin/master)
 
-    if [ -f "${MATERIA_DIR}/${CLI_ARTIFACT}" ]; then
-        old_checksum=$(sha256sum "${MATERIA_DIR}/${CLI_ARTIFACT}" | awk '{print $1}')
+    if [ "$LOCAL" != "$REMOTE" ]; then
+        log "New version detected: $LOCAL -> $REMOTE"
+
+        # Pull latest code
+        if git pull origin master; then
+            log "Code updated successfully"
+
+            # Update dependencies
+            log "Updating dependencies with uv sync..."
+            if uv sync; then
+                log "Dependencies updated"
+                return 0
+            else
+                log_error "Failed to update dependencies"
+                return 1
+            fi
+        else
+            log_error "Failed to pull code"
+            return 1
+        fi
     fi
 
-    new_checksum=$(sha256sum "$temp_file" | awk '{print $1}')
+    log "Already up to date at $(git rev-parse --short HEAD)"
+    return 1  # Return 1 to indicate no update (not an error)
+}
 
-    if [ "$old_checksum" = "$new_checksum" ]; then
-        log "CLI is up to date"
-        rm -f "$temp_file"
+# Run pipeline using materia CLI via uv
+run_pipeline() {
+    local pipeline=$1
+    local date=$(date -u +%Y-%m-%d)
+    local state_file="$STATE_DIR/${pipeline}_last_run"
+
+    log "Running $pipeline pipeline..."
+
+    cd "$MATERIA_REPO"
+    if uv run materia pipeline run "$pipeline"; then
+        log "$pipeline completed successfully"
+        echo "$date" > "$state_file"
         return 0
-    fi
-
-    log "New CLI version detected, updating..."
-
-    # Install new version
-    mv "$temp_file" "${MATERIA_DIR}/${CLI_ARTIFACT}"
-
-    cd "$MATERIA_DIR"
-    rm -rf cli && mkdir -p cli
-    tar -xzf "$CLI_ARTIFACT" -C cli/
-
-    if pip3 install --force-reinstall cli/*.whl; then
-        log "CLI updated successfully"
-        materia version
     else
-        log_error "Failed to install CLI"
+        log_error "$pipeline failed"
         return 1
     fi
 }
 
-# Check if we should run extract pipeline (daily at specified hour)
-should_run_extract() {
-    local current_hour
-    local current_date
-
-    current_hour=$(date -u +%H)
-    current_date=$(date -u +%Y-%m-%d)
+# Check if pipeline should run today
+should_run_pipeline() {
+    local pipeline=$1
+    local schedule_hour=$2
+    local current_hour=$(date -u +%H)
+    local current_date=$(date -u +%Y-%m-%d)
+    local state_file="$STATE_DIR/${pipeline}_last_run"
 
     # Only run at the scheduled hour
-    if [ "$current_hour" != "$EXTRACT_SCHEDULE_HOUR" ]; then
+    if [ "$current_hour" -ne "$schedule_hour" ]; then
         return 1
     fi
 
-    # Only run once per day
-    if [ "$last_extract_run" = "$current_date" ]; then
-        return 1
+    # Check if already ran today
+    if [ -f "$state_file" ]; then
+        local last_run=$(cat "$state_file")
+        if [ "$last_run" = "$current_date" ]; then
+            return 1  # Already ran today
+        fi
     fi
 
-    return 0
-}
-
-# Check if we should run transform pipeline (daily at specified hour)
-should_run_transform() {
-    local current_hour
-    local current_date
-
-    current_hour=$(date -u +%H)
-    current_date=$(date -u +%Y-%m-%d)
-
-    # Only run at the scheduled hour
-    if [ "$current_hour" != "$TRANSFORM_SCHEDULE_HOUR" ]; then
-        return 1
-    fi
-
-    # Only run once per day
-    if [ "$last_transform_run" = "$current_date" ]; then
-        return 1
-    fi
-
-    return 0
-}
-
-# Run extract pipeline
-run_extract() {
-    log "Starting extract pipeline..."
-
-    if materia pipeline run extract; then
-        log "Extract pipeline completed successfully"
-        last_extract_run=$(date -u +%Y-%m-%d)
-    else
-        log_error "Extract pipeline failed"
-        return 1
-    fi
-}
-
-# Run transform pipeline
-run_transform() {
-    log "Starting transform pipeline..."
-
-    if materia pipeline run transform; then
-        log "Transform pipeline completed successfully"
-        last_transform_run=$(date -u +%Y-%m-%d)
-    else
-        log_error "Transform pipeline failed"
-        return 1
-    fi
+    return 0  # Should run
 }
 
 # Main supervisor loop
 main() {
     log "Materia supervisor starting..."
+    log "Repository: $MATERIA_REPO"
     log "Extract schedule: daily at ${EXTRACT_SCHEDULE_HOUR}:00 UTC"
     log "Transform schedule: daily at ${TRANSFORM_SCHEDULE_HOUR}:00 UTC"
     log "Check interval: ${CHECK_INTERVAL}s"
 
-    # Initial CLI check
-    check_cli_update || log_error "Initial CLI check failed, continuing anyway"
+    # Ensure repo exists
+    if [ ! -d "$MATERIA_REPO/.git" ]; then
+        log_error "Repository not found at $MATERIA_REPO"
+        log_error "Run bootstrap script first!"
+        exit 1
+    fi
+
+    # Show initial version
+    cd "$MATERIA_REPO"
+    log "Starting at commit: $(git rev-parse --short HEAD)"
 
     while true; do
-        # Check for CLI updates
-        check_cli_update || true
+        # Check for code updates every loop
+        update_code || true
 
-        # Check and run extract pipeline
-        if should_run_extract; then
-            run_extract || true
+        # Check extract schedule
+        if should_run_pipeline "extract" "$EXTRACT_SCHEDULE_HOUR"; then
+            run_pipeline extract || true
         fi
 
-        # Check and run transform pipeline
-        if should_run_transform; then
-            run_transform || true
+        # Check transform schedule
+        if should_run_pipeline "transform" "$TRANSFORM_SCHEDULE_HOUR"; then
+            run_pipeline transform || true
         fi
 
         sleep "$CHECK_INTERVAL"