feat(infra): replace Pulumi ESC with SOPS in bootstrap + setup scripts

- bootstrap_supervisor.sh: remove esc CLI + PULUMI_ACCESS_TOKEN; install sops+age; check age keypair exists; decrypt .env.prod.sops → .env; checkout latest release tag; use uv sync --all-packages - setup_server.sh: add age keypair generation at /opt/materia/age-key.txt; install age binary; print public key with .sops.yaml instructions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 12:03:11 +01:00
parent 5d7d53a260
commit 95f881827e
2 changed files with 148 additions and 90 deletions
--- a/infra/bootstrap_supervisor.sh
+++ b/infra/bootstrap_supervisor.sh
@@ -1,117 +1,146 @@
 #!/bin/bash
-# Bootstrap script for Materia supervisor instance
-# Run this once on a new supervisor to set it up
+# Bootstrap script for Materia supervisor instance.
+# Run once on a fresh server after setup_server.sh.
 #
 # Usage:
-#   From CI/CD or locally:
-#   ssh root@<supervisor_ip> 'bash -s' < infra/bootstrap_supervisor.sh
+#   ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
 #
-# Or on the supervisor itself:
-#   curl -fsSL <url-to-this-script> | bash
+# Prerequisites:
+#   - age keypair exists at /opt/materia/age-key.txt
+#     (or SOPS_AGE_KEY_FILE env var pointing elsewhere)
+#   - The server age public key is already in .sops.yaml and .env.prod.sops
+#     (run setup_server.sh first, then add the key and re-commit)
+#   - GITLAB_READ_TOKEN is set (GitLab project access token, read-only)

 set -euo pipefail

 echo "=== Materia Supervisor Bootstrap ==="
 echo "This script will:"
-echo "  1. Install dependencies (git, uv, esc)"
+echo "  1. Install dependencies (git, uv, sops, age)"
 echo "  2. Clone the materia repository"
-echo "  3. Setup systemd service"
-echo "  4. Start the supervisor"
+echo "  3. Decrypt secrets from .env.prod.sops"
+echo "  4. Set up systemd service"
+echo "  5. Start the supervisor"
 echo ""

-# Check if we're root
 if [ "$EUID" -ne 0 ]; then
    echo "ERROR: This script must be run as root"
    exit 1
 fi

-# Configuration
+# ── Configuration ──────────────────────────────────────────
 REPO_DIR="/opt/materia"
 GITLAB_PROJECT="deemanone/materia"
+AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}"

-# GITLAB_READ_TOKEN should be set in Pulumi ESC (beanflows/prod)
 if [ -z "${GITLAB_READ_TOKEN:-}" ]; then
-    echo "ERROR: GITLAB_READ_TOKEN environment variable not set"
-    echo "Please add it to Pulumi ESC (beanflows/prod) first"
+    echo "ERROR: GITLAB_READ_TOKEN not set"
+    echo "  export GITLAB_READ_TOKEN=<gitlab-project-access-token>"
    exit 1
 fi

-REPO_URL="https://gitlab-ci-token:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git"
+REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git"

+# ── System dependencies ────────────────────────────────────
 echo "--- Installing system dependencies ---"
-apt-get update
-apt-get install -y git curl python3-pip
+apt-get update -q
+apt-get install -y -q git curl ca-certificates

+# ── uv ─────────────────────────────────────────────────────
 echo "--- Installing uv ---"
 if ! command -v uv &>/dev/null; then
    curl -LsSf https://astral.sh/uv/install.sh | sh
-    export PATH="$HOME/.cargo/bin:$PATH"
-    echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /root/.bashrc
+    export PATH="$HOME/.local/bin:$PATH"
+    echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc
 fi

-echo "--- Installing Pulumi ESC ---"
-if ! command -v esc &> /dev/null; then
-    curl -fsSL https://get.pulumi.com/esc/install.sh | sh
-    export PATH="$HOME/.pulumi/bin:$PATH"
-    echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
+# ── sops + age ─────────────────────────────────────────────
+echo "--- Installing sops + age ---"
+ARCH=$(uname -m)
+case "$ARCH" in
+    x86_64)  ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
+    aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
+    *) echo "Unsupported architecture: $ARCH"; exit 1 ;;
+esac
+
+if ! command -v age &>/dev/null; then
+    AGE_VERSION="v1.3.1"
+    curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz
+    tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
+    chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
+    rm /tmp/age.tar.gz
 fi

-echo "--- Setting up Pulumi ESC authentication ---"
-if [ -z "${PULUMI_ACCESS_TOKEN:-}" ]; then
-    echo "ERROR: PULUMI_ACCESS_TOKEN environment variable not set"
-    echo "Please set it before running this script:"
-    echo "  export PULUMI_ACCESS_TOKEN=<your-token>"
-    exit 1
+if ! command -v sops &>/dev/null; then
+    SOPS_VERSION="v3.12.1"
+    curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops
+    chmod +x /usr/local/bin/sops
 fi

-esc login --token "$PULUMI_ACCESS_TOKEN"
-
-echo "--- Loading secrets from Pulumi ESC ---"
-eval $(esc env open beanflows/prod --format shell)
-
+# ── Clone repository ───────────────────────────────────────
 echo "--- Cloning repository ---"
-if [ -d "$REPO_DIR" ]; then
-    echo "Repository already exists, pulling latest..."
+if [ -d "$REPO_DIR/.git" ]; then
+    echo "Repository already exists — fetching latest tags..."
    cd "$REPO_DIR"
-    git pull origin master
+    git fetch --tags --prune-tags origin
 else
    git clone "$REPO_URL" "$REPO_DIR"
    cd "$REPO_DIR"
 fi

+# Checkout latest release tag (same logic as supervisor)
+LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1)
+if [ -n "$LATEST_TAG" ]; then
+    echo "Checking out $LATEST_TAG..."
+    git checkout --detach "$LATEST_TAG"
+else
+    echo "No release tags found — staying on current HEAD"
+fi
+
+# ── Check age keypair ──────────────────────────────────────
+echo "--- Checking age keypair ---"
+if [ ! -f "$AGE_KEY_FILE" ]; then
+    echo "ERROR: Age keypair not found at $AGE_KEY_FILE"
+    echo ""
+    echo "Run infra/setup_server.sh first to generate the keypair, then:"
+    echo "  1. Copy the public key from setup_server.sh output"
+    echo "  2. Add it to .sops.yaml on your workstation"
+    echo "  3. Run: sops updatekeys .env.prod.sops"
+    echo "  4. Commit + push and re-run this bootstrap"
+    exit 1
+fi
+export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
+
+# ── Decrypt secrets ────────────────────────────────────────
+echo "--- Decrypting secrets from .env.prod.sops ---"
+sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env"
+chmod 600 "$REPO_DIR/.env"
+echo "Secrets written to $REPO_DIR/.env"
+
+# ── Data directories ───────────────────────────────────────
 echo "--- Creating data directories ---"
-mkdir -p /data/materia/landing/psd
+mkdir -p /data/materia/landing

+# ── Python dependencies ────────────────────────────────────
 echo "--- Installing Python dependencies ---"
-uv sync
-
-echo "--- Creating environment file ---"
-cat > "$REPO_DIR/.env" <<EOF
-# Environment variables for supervisor
-# Loaded from Pulumi ESC: beanflows/prod
-PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
-PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
-LANDING_DIR=/data/materia/landing
-DUCKDB_PATH=/data/materia/lakehouse.duckdb
-SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb
-EOF
+cd "$REPO_DIR"
+uv sync --all-packages

+# ── Systemd service ────────────────────────────────────────
 echo "--- Setting up systemd service ---"
 cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service

 echo "--- Enabling and starting service ---"
 systemctl daemon-reload
 systemctl enable materia-supervisor
-systemctl start materia-supervisor
+systemctl restart materia-supervisor

 echo ""
 echo "=== Bootstrap complete! ==="
 echo ""
-echo "Supervisor is now running. Check status with:"
-echo "  systemctl status materia-supervisor"
+echo "Check status:  systemctl status materia-supervisor"
+echo "View logs:     journalctl -u materia-supervisor -f"
+echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status"
 echo ""
-echo "View logs with:"
-echo "  journalctl -u materia-supervisor -f"
-echo ""
-echo "Repository location: $REPO_DIR"
-echo "Current commit: $(cd $REPO_DIR && git rev-parse --short HEAD)"
+echo "Repo: $REPO_DIR"
+echo "Tag:  $(cd $REPO_DIR && git describe --tags --always)"
--- a/infra/setup_server.sh
+++ b/infra/setup_server.sh
@@ -1,45 +1,74 @@
 #!/bin/bash
-# One-time server setup: create app user, /opt/beanflows, and GitLab deploy key.
-# Run as root on a fresh Hetzner server before bootstrapping the supervisor.
+# One-time server setup: create data directories, generate age keypair.
+# Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh.
 #
 # Usage:
 #   bash infra/setup_server.sh

 set -euo pipefail

-APP_USER="beanflows_service"
-APP_DIR="/opt/beanflows"
-KEY_PATH="/home/$APP_USER/.ssh/gitlab_deploy"
+REPO_DIR="/opt/materia"
+AGE_KEY_FILE="$REPO_DIR/age-key.txt"

-# Create system user with a home dir (needed for .ssh) but no login shell
-if ! id "$APP_USER" &>/dev/null; then
-    useradd --system --create-home --shell /usr/sbin/nologin "$APP_USER"
-    echo "Created user: $APP_USER"
-else
-    echo "User $APP_USER already exists, skipping"
+if [ "$EUID" -ne 0 ]; then
+    echo "ERROR: This script must be run as root"
+    exit 1
 fi

-# Create app directory owned by app user
-mkdir -p "$APP_DIR"
-chown "$APP_USER:$APP_USER" "$APP_DIR"
-chmod 750 "$APP_DIR"
-echo "Created $APP_DIR (owner: $APP_USER)"
+# ── Create data directories ────────────────────────────────
+echo "--- Creating data directories ---"
+mkdir -p /data/materia/landing
+mkdir -p "$REPO_DIR"
+echo "Data dir: /data/materia"

-# Generate deploy key if not already present
-if [ ! -f "$KEY_PATH" ]; then
-    mkdir -p "/home/$APP_USER/.ssh"
-    ssh-keygen -t ed25519 -f "$KEY_PATH" -N "" -C "beanflows-server"
-    chown -R "$APP_USER:$APP_USER" "/home/$APP_USER/.ssh"
-    chmod 700 "/home/$APP_USER/.ssh"
-    chmod 600 "$KEY_PATH"
-    chmod 644 "$KEY_PATH.pub"
-    echo "Generated deploy key: $KEY_PATH"
-else
-    echo "Deploy key already exists, skipping"
+# ── Install age ────────────────────────────────────────────
+echo "--- Installing age ---"
+ARCH=$(uname -m)
+case "$ARCH" in
+    x86_64)  ARCH_AGE="amd64" ;;
+    aarch64) ARCH_AGE="arm64" ;;
+    *) echo "Unsupported architecture: $ARCH"; exit 1 ;;
+esac
+
+if ! command -v age-keygen &>/dev/null; then
+    AGE_VERSION="v1.3.1"
+    curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
+    tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
+    chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
+    rm /tmp/age.tar.gz
+    echo "age installed to /usr/local/bin"
 fi

+# ── Generate age keypair ───────────────────────────────────
+echo "--- Setting up age keypair ---"
+if [ -f "$AGE_KEY_FILE" ]; then
+    echo "Keypair already exists at $AGE_KEY_FILE — skipping generation"
+else
+    age-keygen -o "$AGE_KEY_FILE" 2>/dev/null
+    chmod 600 "$AGE_KEY_FILE"
+    echo "Generated: $AGE_KEY_FILE"
+fi
+
+AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
+
 echo ""
-echo "=== Add this deploy key to GitLab ==="
-echo "GitLab → repo → Settings → Repository → Deploy Keys (read-only)"
+echo "=================================================================="
+echo "  Server age public key:"
 echo ""
-cat "$KEY_PATH.pub"
+echo "    $AGE_PUB"
+echo ""
+echo "  Add this key to .sops.yaml on your workstation:"
+echo ""
+echo "    creation_rules:"
+echo "      - path_regex: \\.env\\.(dev|prod)\\.sops\$"
+echo "        age: >-"
+echo "          <dev-key>"
+echo "          + $AGE_PUB"
+echo ""
+echo "  Then re-encrypt the prod secrets file:"
+echo "    sops updatekeys .env.prod.sops"
+echo "    git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'"
+echo "    git push"
+echo ""
+echo "  Then run infra/bootstrap_supervisor.sh to complete setup."
+echo "=================================================================="