From b27f06d811ef6dff2dbcdff81112252d56bda9d6 Mon Sep 17 00:00:00 2001 From: Deeman Date: Thu, 26 Feb 2026 20:28:31 +0100 Subject: [PATCH 1/3] chore: remove stale ralph-loop session file Co-Authored-By: Claude Opus 4.6 --- .claude/ralph-loop.local.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 .claude/ralph-loop.local.md diff --git a/.claude/ralph-loop.local.md b/.claude/ralph-loop.local.md deleted file mode 100644 index 32800ae..0000000 --- a/.claude/ralph-loop.local.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -active: true -iteration: 1 -max_iterations: 0 -completion_promise: "all phases complete and tagged" -started_at: "2026-02-25T23:17:16Z" ---- - -implement the remaining phases from the current plan (phases 2 through 7) for the quart_saas_boilerplate template and beanflows update From 0317cb885fae9801a1f4d38bd3f0a9f1e12959b9 Mon Sep 17 00:00:00 2001 From: Deeman Date: Thu, 26 Feb 2026 21:33:31 +0100 Subject: [PATCH 2/3] feat(infra): use beanflows_service for supervisor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - materia-supervisor.service: User=root → User=beanflows_service, add PATH so uv (~/.local/bin) is found without a login shell - setup_server.sh: full rewrite — creates beanflows_service (nologin), generates SSH deploy key + age keypair as service user at XDG path (~/.config/sops/age/keys.txt), installs age/sops/rclone as root, prints both public keys + numbered next-step instructions - bootstrap_supervisor.sh: full rewrite — removes GITLAB_READ_TOKEN requirement, clones via SSH as service user, installs uv as service user, decrypts with SOPS auto-discovery, uv sync as service user, systemctl as root - web/deploy.sh: remove self-contained sops/age install + keypair generation; replace with simple sops check (exit if missing) and SOPS auto-discovery decrypt (no explicit key file needed) - infra/readme.md: update architecture diagram for beanflows_service paths, update setup steps to match new scripts Co-Authored-By: Claude Sonnet 4.6 --- infra/bootstrap_supervisor.sh | 146 ++++++++--------- infra/readme.md | 43 ++--- infra/setup_server.sh | 169 +++++++++++++++----- infra/supervisor/materia-supervisor.service | 3 +- web/deploy.sh | 51 +----- 5 files changed, 227 insertions(+), 185 deletions(-) diff --git a/infra/bootstrap_supervisor.sh b/infra/bootstrap_supervisor.sh index a6a018e..a966a9d 100755 --- a/infra/bootstrap_supervisor.sh +++ b/infra/bootstrap_supervisor.sh @@ -6,129 +6,123 @@ # ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh # # Prerequisites: -# - age keypair exists at /opt/materia/age-key.txt -# (or SOPS_AGE_KEY_FILE env var pointing elsewhere) -# - The server age public key is already in .sops.yaml and .env.prod.sops -# (run setup_server.sh first, then add the key and re-commit) -# - GITLAB_READ_TOKEN is set (GitLab project access token, read-only) +# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair) +# - Deploy key added to GitLab (Settings → Repository → Deploy Keys) +# - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed set -euo pipefail -echo "=== Materia Supervisor Bootstrap ===" -echo "This script will:" -echo " 1. Install dependencies (git, uv, sops, age)" -echo " 2. Clone the materia repository" -echo " 3. Decrypt secrets from .env.prod.sops" -echo " 4. Set up systemd service" -echo " 5. Start the supervisor" -echo "" - -if [ "$EUID" -ne 0 ]; then - echo "ERROR: This script must be run as root" - exit 1 -fi - -# ── Configuration ────────────────────────────────────────── +SERVICE_USER="beanflows_service" REPO_DIR="/opt/materia" GITLAB_PROJECT="deemanone/materia" -AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}" +UV="/home/${SERVICE_USER}/.local/bin/uv" -if [ -z "${GITLAB_READ_TOKEN:-}" ]; then - echo "ERROR: GITLAB_READ_TOKEN not set" - echo " export GITLAB_READ_TOKEN=" - exit 1 -fi +echo "=== Materia Supervisor Bootstrap ===" +echo "" -REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git" +[ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; } + +# ── System dependencies ──────────────────────────────────────────────────────── -# ── System dependencies ──────────────────────────────────── echo "--- Installing system dependencies ---" apt-get update -q apt-get install -y -q git curl ca-certificates -# ── uv ───────────────────────────────────────────────────── +# ── uv (installed as service user) ──────────────────────────────────────────── + echo "--- Installing uv ---" -if ! command -v uv &>/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="$HOME/.local/bin:$PATH" - echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc +if [ ! -f "${UV}" ]; then + sudo -u "${SERVICE_USER}" bash -c \ + 'curl -LsSf https://astral.sh/uv/install.sh | sh' fi -# ── sops + age ───────────────────────────────────────────── +# ── sops + age (as root, idempotent — setup_server.sh may have done this) ───── + echo "--- Installing sops + age ---" ARCH=$(uname -m) -case "$ARCH" in - x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;; - aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;; - *) echo "Unsupported architecture: $ARCH"; exit 1 ;; +case "${ARCH}" in + x86_64) ARCH_TAG="amd64" ;; + aarch64) ARCH_TAG="arm64" ;; + *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; esac -if ! command -v age &>/dev/null; then +if ! command -v age >/dev/null 2>&1; then AGE_VERSION="v1.3.1" - curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz + curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen chmod +x /usr/local/bin/age /usr/local/bin/age-keygen rm /tmp/age.tar.gz fi -if ! command -v sops &>/dev/null; then +if ! command -v sops >/dev/null 2>&1; then SOPS_VERSION="v3.12.1" - curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops + curl -fsSL \ + "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \ + -o /usr/local/bin/sops chmod +x /usr/local/bin/sops fi -# ── Clone repository ─────────────────────────────────────── +# ── Clone repository via SSH as service user ────────────────────────────────── + echo "--- Cloning repository ---" -if [ -d "$REPO_DIR/.git" ]; then +if [ -d "${REPO_DIR}/.git" ]; then echo "Repository already exists — fetching latest tags..." - cd "$REPO_DIR" - git fetch --tags --prune-tags origin + sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin else - git clone "$REPO_URL" "$REPO_DIR" - cd "$REPO_DIR" + sudo -u "${SERVICE_USER}" git clone \ + "git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}" fi # Checkout latest release tag (same logic as supervisor) -LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1) -if [ -n "$LATEST_TAG" ]; then - echo "Checking out $LATEST_TAG..." - git checkout --detach "$LATEST_TAG" +LATEST_TAG=$(sudo -u "${SERVICE_USER}" \ + git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1) +if [ -n "${LATEST_TAG}" ]; then + echo "Checking out ${LATEST_TAG}..." + sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}" else echo "No release tags found — staying on current HEAD" fi -# ── Check age keypair ────────────────────────────────────── +# ── Check age keypair ───────────────────────────────────────────────────────── + echo "--- Checking age keypair ---" -if [ ! -f "$AGE_KEY_FILE" ]; then - echo "ERROR: Age keypair not found at $AGE_KEY_FILE" +AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt" +if [ ! -f "${AGE_KEY_FILE}" ]; then echo "" - echo "Run infra/setup_server.sh first to generate the keypair, then:" - echo " 1. Copy the public key from setup_server.sh output" - echo " 2. Add it to .sops.yaml on your workstation" - echo " 3. Run: sops updatekeys .env.prod.sops" - echo " 4. Commit + push and re-run this bootstrap" + echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}" + echo "" + echo "Run infra/setup_server.sh first, then:" + echo " 1. Add the SSH deploy key to GitLab (Settings → Repository → Deploy Keys)" + echo " 2. Add the age public key to .sops.yaml on your workstation" + echo " 3. Run: sops updatekeys .env.prod.sops && git push" + echo " 4. Re-run this bootstrap" exit 1 fi -export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE" -# ── Decrypt secrets ──────────────────────────────────────── +# ── Decrypt secrets (as service user — SOPS auto-discovers age key from XDG) ── + echo "--- Decrypting secrets from .env.prod.sops ---" -sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env" -chmod 600 "$REPO_DIR/.env" -echo "Secrets written to $REPO_DIR/.env" +sudo -u "${SERVICE_USER}" bash -c \ + "sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env" +chmod 600 "${REPO_DIR}/.env" +echo "Secrets written to ${REPO_DIR}/.env" + +# ── Data directories ─────────────────────────────────────────────────────────── -# ── Data directories ─────────────────────────────────────── echo "--- Creating data directories ---" mkdir -p /data/materia/landing +chown -R "${SERVICE_USER}:${SERVICE_USER}" /data/materia + +# ── Python dependencies (as service user) ───────────────────────────────────── -# ── Python dependencies ──────────────────────────────────── echo "--- Installing Python dependencies ---" -cd "$REPO_DIR" -uv sync --all-packages +sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages" + +# ── Systemd service (as root) ────────────────────────────────────────────────── -# ── Systemd service ──────────────────────────────────────── echo "--- Setting up systemd service ---" -cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service +cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" \ + /etc/systemd/system/materia-supervisor.service echo "--- Enabling and starting service ---" systemctl daemon-reload @@ -138,9 +132,9 @@ systemctl restart materia-supervisor echo "" echo "=== Bootstrap complete! ===" echo "" -echo "Check status: systemctl status materia-supervisor" -echo "View logs: journalctl -u materia-supervisor -f" -echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status" +echo "Check status: systemctl status materia-supervisor" +echo "View logs: journalctl -u materia-supervisor -f" +echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status" echo "" -echo "Repo: $REPO_DIR" -echo "Tag: $(cd $REPO_DIR && git describe --tags --always)" +echo "Repo: ${REPO_DIR}" +echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" diff --git a/infra/readme.md b/infra/readme.md index a7319a2..82881b7 100644 --- a/infra/readme.md +++ b/infra/readme.md @@ -6,15 +6,17 @@ Single-server local-first setup for BeanFlows.coffee on Hetzner NVMe. ``` Hetzner Server (NVMe) -├── /opt/materia/ # Git repo (checked out at latest release tag) -├── /opt/materia/age-key.txt # Server age keypair (chmod 600, gitignored) -├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time -├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed) -├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write -├── /data/materia/analytics.duckdb # Read-only serving copy for web app +├── beanflows_service (system user, nologin) +│ ├── ~/.ssh/materia_deploy # ed25519 deploy key for GitLab read access +│ └── ~/.config/sops/age/keys.txt # age keypair (auto-discovered by SOPS) +├── /opt/materia/ # Git repo (owned by beanflows_service, latest release tag) +├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time +├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed) +├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write +├── /data/materia/analytics.duckdb # Read-only serving copy for web app └── systemd services: - ├── materia-supervisor # Python supervisor: extract → transform → export → deploy - └── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours + ├── materia-supervisor # Python supervisor: extract → transform → export → deploy + └── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours ``` ## Data Flow @@ -33,15 +35,16 @@ Hetzner Server (NVMe) bash infra/setup_server.sh ``` -This creates data directories, installs age, and generates the server age keypair at `/opt/materia/age-key.txt`. It prints the server's age public key. +This creates the `beanflows_service` user, data directories, installs age + sops + rclone, generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys. -### 2. Add the server key to SOPS - -On your workstation: +### 2. Add keys to GitLab and SOPS ```bash -# Add the server public key to .sops.yaml -# Then re-encrypt prod secrets to include the server key: +# Add the SSH deploy key to GitLab: +# → Repository Settings → Deploy Keys → Add key (read-only) + +# Add the server age public key to .sops.yaml on your workstation, +# then re-encrypt prod secrets to include the server key: sops updatekeys .env.prod.sops git add .sops.yaml .env.prod.sops git commit -m "chore: add server age key" @@ -51,18 +54,19 @@ git push ### 3. Bootstrap the supervisor ```bash -# Requires GITLAB_READ_TOKEN (GitLab project access token, read-only) -export GITLAB_READ_TOKEN= ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh ``` -This installs uv + sops + age, clones the repo, decrypts secrets, installs Python dependencies, and starts the supervisor service. +This installs uv (as service user), clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. ### 4. Set up R2 backup ```bash apt install rclone -cp infra/backup/rclone.conf.example /root/.config/rclone/rclone.conf +# Configure rclone as the service user (used by the backup timer): +sudo -u beanflows_service mkdir -p /home/beanflows_service/.config/rclone +sudo -u beanflows_service cp infra/backup/rclone.conf.example \ + /home/beanflows_service/.config/rclone/rclone.conf # Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint) cp infra/backup/materia-backup.service /etc/systemd/system/ cp infra/backup/materia-backup.timer /etc/systemd/system/ @@ -90,6 +94,7 @@ make secrets-edit-prod `bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup. `web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically). +SOPS auto-discovers the service user's age key at `~/.config/sops/age/keys.txt` (XDG default). ## Deploy model (pull-based) @@ -109,7 +114,7 @@ systemctl status materia-supervisor journalctl -u materia-supervisor -f # Workflow status table -cd /opt/materia && uv run python src/materia/supervisor.py status +cd /opt/materia && sudo -u beanflows_service uv run python src/materia/supervisor.py status # Backup timer status systemctl list-timers materia-backup.timer diff --git a/infra/setup_server.sh b/infra/setup_server.sh index a5a0048..b8f46f1 100644 --- a/infra/setup_server.sh +++ b/infra/setup_server.sh @@ -1,74 +1,161 @@ #!/bin/bash -# One-time server setup: create data directories, generate age keypair. +# One-time server setup: create service user, SSH deploy key, age keypair. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh. # # Usage: # bash infra/setup_server.sh +# +# What it does: +# 1. Creates beanflows_service user (nologin) + adds to docker group +# 2. Creates /opt/materia + /data/materia/landing with correct ownership +# 3. Generates ed25519 SSH deploy key for GitLab read access +# 4. Installs age + sops + rclone to /usr/local/bin (as root) +# 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user) +# 6. Prints both public keys + numbered next-step instructions set -euo pipefail -REPO_DIR="/opt/materia" -AGE_KEY_FILE="$REPO_DIR/age-key.txt" +SERVICE_USER="beanflows_service" +APP_DIR="/opt/materia" +DATA_DIR="/data/materia" +SSH_DIR="/home/${SERVICE_USER}/.ssh" +DEPLOY_KEY="${SSH_DIR}/materia_deploy" +SOPS_AGE_DIR="/home/${SERVICE_USER}/.config/sops/age" -if [ "$EUID" -ne 0 ]; then - echo "ERROR: This script must be run as root" - exit 1 +[ "$(id -u)" = "0" ] || { echo "ERROR: Run as root: sudo bash infra/setup_server.sh"; exit 1; } + +log() { echo "$(date '+%H:%M:%S') ==> $*"; } + +# ── Service user ────────────────────────────────────────────────────────────── + +log "Creating service user ${SERVICE_USER}..." +if ! id "${SERVICE_USER}" >/dev/null 2>&1; then + useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}" +fi +usermod -aG docker "${SERVICE_USER}" +log "User OK." + +# ── Directories ─────────────────────────────────────────────────────────────── + +log "Creating directories..." +mkdir -p "${APP_DIR}" "${DATA_DIR}/landing" +chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}" +chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}" +log "Directories OK." + +# ── SSH deploy key ──────────────────────────────────────────────────────────── + +log "Setting up SSH deploy key..." +sudo -u "${SERVICE_USER}" mkdir -p "${SSH_DIR}" +chmod 700 "${SSH_DIR}" + +if [ ! -f "${DEPLOY_KEY}" ]; then + sudo -u "${SERVICE_USER}" ssh-keygen -t ed25519 \ + -f "${DEPLOY_KEY}" -N "" -C "materia-deploy" fi -# ── Create data directories ──────────────────────────────── -echo "--- Creating data directories ---" -mkdir -p /data/materia/landing -mkdir -p "$REPO_DIR" -echo "Data dir: /data/materia" +sudo -u "${SERVICE_USER}" bash -c "cat > ${SSH_DIR}/config" <> ${SSH_DIR}/known_hosts 2>/dev/null; \ + sort -u ${SSH_DIR}/known_hosts -o ${SSH_DIR}/known_hosts" +chmod 644 "${SSH_DIR}/known_hosts" +log "SSH deploy key OK." + +# ── age ─────────────────────────────────────────────────────────────────────── -# ── Install age ──────────────────────────────────────────── -echo "--- Installing age ---" ARCH=$(uname -m) -case "$ARCH" in - x86_64) ARCH_AGE="amd64" ;; - aarch64) ARCH_AGE="arm64" ;; - *) echo "Unsupported architecture: $ARCH"; exit 1 ;; +case "${ARCH}" in + x86_64) ARCH_TAG="amd64" ;; + aarch64) ARCH_TAG="arm64" ;; + *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; esac -if ! command -v age-keygen &>/dev/null; then +if ! command -v age-keygen >/dev/null 2>&1; then + log "Installing age..." AGE_VERSION="v1.3.1" - curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz + curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen chmod +x /usr/local/bin/age /usr/local/bin/age-keygen rm /tmp/age.tar.gz - echo "age installed to /usr/local/bin" + log "age installed." fi -# ── Generate age keypair ─────────────────────────────────── -echo "--- Setting up age keypair ---" -if [ -f "$AGE_KEY_FILE" ]; then - echo "Keypair already exists at $AGE_KEY_FILE — skipping generation" +# ── sops ────────────────────────────────────────────────────────────────────── + +if ! command -v sops >/dev/null 2>&1; then + log "Installing sops..." + SOPS_VERSION="v3.12.1" + curl -fsSL \ + "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \ + -o /usr/local/bin/sops + chmod +x /usr/local/bin/sops + log "sops installed." +fi + +# ── rclone ──────────────────────────────────────────────────────────────────── + +if ! command -v rclone >/dev/null 2>&1; then + log "Installing rclone..." + curl -fsSL https://rclone.org/install.sh | bash + log "rclone installed." +fi + +# ── age keypair (as service user, XDG path auto-discovered by SOPS) ─────────── + +log "Generating age keypair for ${SERVICE_USER}..." +AGE_KEY_FILE="${SOPS_AGE_DIR}/keys.txt" +if [ ! -f "${AGE_KEY_FILE}" ]; then + sudo -u "${SERVICE_USER}" mkdir -p "${SOPS_AGE_DIR}" + sudo -u "${SERVICE_USER}" age-keygen -o "${AGE_KEY_FILE}" + chmod 600 "${AGE_KEY_FILE}" + log "Age keypair generated at ${AGE_KEY_FILE}" else - age-keygen -o "$AGE_KEY_FILE" 2>/dev/null - chmod 600 "$AGE_KEY_FILE" - echo "Generated: $AGE_KEY_FILE" + log "Age keypair already exists — skipping." fi -AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}') +# ── Summary ─────────────────────────────────────────────────────────────────── + +DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub") +AGE_PUB=$(grep "public key:" "${AGE_KEY_FILE}" | awk '{print $NF}') echo "" echo "==================================================================" -echo " Server age public key:" echo "" -echo " $AGE_PUB" +echo " SSH deploy key (add to GitLab → Settings → Deploy Keys):" echo "" -echo " Add this key to .sops.yaml on your workstation:" +echo " ${DEPLOY_PUB}" echo "" -echo " creation_rules:" -echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$" -echo " age: >-" -echo " " -echo " + $AGE_PUB" +echo " Server age public key (add to .sops.yaml):" echo "" -echo " Then re-encrypt the prod secrets file:" -echo " sops updatekeys .env.prod.sops" -echo " git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'" -echo " git push" +echo " ${AGE_PUB}" +echo "" +echo "==================================================================" +echo "" +echo " Next steps:" +echo "" +echo " 1. Add the SSH deploy key to GitLab:" +echo " → Repository Settings → Deploy Keys → Add key (read-only)" +echo "" +echo " 2. Add the age public key to .sops.yaml on your workstation:" +echo " creation_rules:" +echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$" +echo " age: >-" +echo " " +echo " + ${AGE_PUB}" +echo "" +echo " 3. Re-encrypt prod secrets to include the server key:" +echo " sops updatekeys .env.prod.sops" +echo " git add .sops.yaml .env.prod.sops" +echo " git commit -m 'chore: add server age key'" +echo " git push" +echo "" +echo " 4. Run bootstrap:" +echo " bash infra/bootstrap_supervisor.sh" echo "" -echo " Then run infra/bootstrap_supervisor.sh to complete setup." echo "==================================================================" diff --git a/infra/supervisor/materia-supervisor.service b/infra/supervisor/materia-supervisor.service index 16c0427..d9dff68 100644 --- a/infra/supervisor/materia-supervisor.service +++ b/infra/supervisor/materia-supervisor.service @@ -5,12 +5,13 @@ Wants=network-online.target [Service] Type=simple -User=root +User=beanflows_service WorkingDirectory=/opt/materia ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py' Restart=always RestartSec=10 EnvironmentFile=/opt/materia/.env +Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin Environment=LANDING_DIR=/data/materia/landing Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb diff --git a/web/deploy.sh b/web/deploy.sh index 4426b69..8797862 100644 --- a/web/deploy.sh +++ b/web/deploy.sh @@ -1,60 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -# ── Ensure sops + age are installed ─────────────────────── APP_DIR="$(cd "$(dirname "$0")" && pwd)" -BIN_DIR="$APP_DIR/bin" -mkdir -p "$BIN_DIR" -export PATH="$BIN_DIR:$PATH" - -ARCH=$(uname -m) -case "$ARCH" in - x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;; - aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;; - *) echo "Unsupported architecture: $ARCH"; exit 1 ;; -esac - -if ! command -v age &>/dev/null; then - echo "==> Installing age to $BIN_DIR..." - AGE_VERSION="v1.3.1" - curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz - tar -xzf /tmp/age.tar.gz -C "$BIN_DIR" --strip-components=1 age/age age/age-keygen - chmod +x "$BIN_DIR/age" "$BIN_DIR/age-keygen" - rm /tmp/age.tar.gz -fi +# ── Verify sops is installed (setup_server.sh installs it to /usr/local/bin) ── if ! command -v sops &>/dev/null; then - echo "==> Installing sops to $BIN_DIR..." - SOPS_VERSION="v3.12.1" - curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o "$BIN_DIR/sops" - chmod +x "$BIN_DIR/sops" -fi - -# ── Ensure age keypair exists ───────────────────────────── -# Key file lives at repo root (one level up from web/) -AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$APP_DIR/../age-key.txt}" -AGE_KEY_FILE="$(realpath "$AGE_KEY_FILE")" -export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE" - -if [ ! -f "$AGE_KEY_FILE" ]; then - echo "==> Generating age keypair at $AGE_KEY_FILE..." - age-keygen -o "$AGE_KEY_FILE" 2>&1 - chmod 600 "$AGE_KEY_FILE" - AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}') - echo "" - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "!! NEW SERVER — add this public key to .sops.yaml: !!" - echo "!! !!" - echo "!! $AGE_PUB !!" - echo "!! !!" - echo "!! Then run: sops updatekeys .env.prod.sops !!" - echo "!! Commit, push, and re-deploy. !!" - echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" - echo "" + echo "ERROR: sops not found — run infra/setup_server.sh first" exit 1 fi -# ── Decrypt secrets ─────────────────────────────────────── +# ── Decrypt secrets (SOPS auto-discovers age key from ~/.config/sops/age/) ──── echo "==> Decrypting secrets from .env.prod.sops..." sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env" chmod 600 "$APP_DIR/.env" From cf65fa16b6084ebd3cdbf13c4258d40e159d7e73 Mon Sep 17 00:00:00 2001 From: Deeman Date: Thu, 26 Feb 2026 22:25:31 +0100 Subject: [PATCH 3/3] refactor(infra): consolidate tool installs in setup, strip bootstrap to essentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setup_server.sh: add git/curl/ca-certificates apt install, add uv install as service user, fix SSH config write (root + chown vs sudo heredoc), remove noise log lines after set -e makes them redundant - bootstrap_supervisor.sh: remove all tool installs (apt, uv, sops, age) — setup_server.sh is now the single source of truth; strip to ~45 lines: age-key check, clone/fetch, tag checkout, decrypt, uv sync, systemd enable - readme.md: update step 1 and step 3 descriptions Co-Authored-By: Claude Sonnet 4.6 --- infra/bootstrap_supervisor.sh | 105 ++++++---------------------------- infra/readme.md | 4 +- infra/setup_server.sh | 31 ++++++---- 3 files changed, 40 insertions(+), 100 deletions(-) diff --git a/infra/bootstrap_supervisor.sh b/infra/bootstrap_supervisor.sh index a966a9d..4e372ae 100755 --- a/infra/bootstrap_supervisor.sh +++ b/infra/bootstrap_supervisor.sh @@ -1,12 +1,13 @@ #!/bin/bash -# Bootstrap script for Materia supervisor instance. -# Run once on a fresh server after setup_server.sh. +# Bootstrap Materia supervisor after setup_server.sh + adding keys. +# Run once on the server after SSH deploy key is added to GitLab +# and the server age key is committed to .env.prod.sops. # # Usage: # ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh # # Prerequisites: -# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair) +# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair, uv) # - Deploy key added to GitLab (Settings → Repository → Deploy Keys) # - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed @@ -17,117 +18,47 @@ REPO_DIR="/opt/materia" GITLAB_PROJECT="deemanone/materia" UV="/home/${SERVICE_USER}/.local/bin/uv" -echo "=== Materia Supervisor Bootstrap ===" -echo "" - [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; } -# ── System dependencies ──────────────────────────────────────────────────────── +# ── Check age keypair ───────────────────────────────────────────────────────── -echo "--- Installing system dependencies ---" -apt-get update -q -apt-get install -y -q git curl ca-certificates - -# ── uv (installed as service user) ──────────────────────────────────────────── - -echo "--- Installing uv ---" -if [ ! -f "${UV}" ]; then - sudo -u "${SERVICE_USER}" bash -c \ - 'curl -LsSf https://astral.sh/uv/install.sh | sh' +AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt" +if [ ! -f "${AGE_KEY_FILE}" ]; then + echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}" + echo "Run infra/setup_server.sh first, then add the printed keys, then re-run." + exit 1 fi -# ── sops + age (as root, idempotent — setup_server.sh may have done this) ───── +# ── Clone or update repository ──────────────────────────────────────────────── -echo "--- Installing sops + age ---" -ARCH=$(uname -m) -case "${ARCH}" in - x86_64) ARCH_TAG="amd64" ;; - aarch64) ARCH_TAG="arm64" ;; - *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; -esac - -if ! command -v age >/dev/null 2>&1; then - AGE_VERSION="v1.3.1" - curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz - tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen - chmod +x /usr/local/bin/age /usr/local/bin/age-keygen - rm /tmp/age.tar.gz -fi - -if ! command -v sops >/dev/null 2>&1; then - SOPS_VERSION="v3.12.1" - curl -fsSL \ - "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \ - -o /usr/local/bin/sops - chmod +x /usr/local/bin/sops -fi - -# ── Clone repository via SSH as service user ────────────────────────────────── - -echo "--- Cloning repository ---" if [ -d "${REPO_DIR}/.git" ]; then - echo "Repository already exists — fetching latest tags..." sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin else sudo -u "${SERVICE_USER}" git clone \ "git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}" fi -# Checkout latest release tag (same logic as supervisor) LATEST_TAG=$(sudo -u "${SERVICE_USER}" \ git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1) if [ -n "${LATEST_TAG}" ]; then - echo "Checking out ${LATEST_TAG}..." sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}" -else - echo "No release tags found — staying on current HEAD" fi -# ── Check age keypair ───────────────────────────────────────────────────────── +# ── Decrypt secrets ─────────────────────────────────────────────────────────── -echo "--- Checking age keypair ---" -AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt" -if [ ! -f "${AGE_KEY_FILE}" ]; then - echo "" - echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}" - echo "" - echo "Run infra/setup_server.sh first, then:" - echo " 1. Add the SSH deploy key to GitLab (Settings → Repository → Deploy Keys)" - echo " 2. Add the age public key to .sops.yaml on your workstation" - echo " 3. Run: sops updatekeys .env.prod.sops && git push" - echo " 4. Re-run this bootstrap" - exit 1 -fi - -# ── Decrypt secrets (as service user — SOPS auto-discovers age key from XDG) ── - -echo "--- Decrypting secrets from .env.prod.sops ---" sudo -u "${SERVICE_USER}" bash -c \ "sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env" chmod 600 "${REPO_DIR}/.env" -echo "Secrets written to ${REPO_DIR}/.env" -# ── Data directories ─────────────────────────────────────────────────────────── +# ── Python dependencies ─────────────────────────────────────────────────────── -echo "--- Creating data directories ---" -mkdir -p /data/materia/landing -chown -R "${SERVICE_USER}:${SERVICE_USER}" /data/materia - -# ── Python dependencies (as service user) ───────────────────────────────────── - -echo "--- Installing Python dependencies ---" sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages" -# ── Systemd service (as root) ────────────────────────────────────────────────── +# ── Systemd service ─────────────────────────────────────────────────────────── -echo "--- Setting up systemd service ---" -cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" \ - /etc/systemd/system/materia-supervisor.service - -echo "--- Enabling and starting service ---" +cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" /etc/systemd/system/ systemctl daemon-reload -systemctl enable materia-supervisor -systemctl restart materia-supervisor +systemctl enable --now materia-supervisor echo "" echo "=== Bootstrap complete! ===" @@ -135,6 +66,4 @@ echo "" echo "Check status: systemctl status materia-supervisor" echo "View logs: journalctl -u materia-supervisor -f" echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status" -echo "" -echo "Repo: ${REPO_DIR}" -echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" +echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" diff --git a/infra/readme.md b/infra/readme.md index 82881b7..ff91664 100644 --- a/infra/readme.md +++ b/infra/readme.md @@ -35,7 +35,7 @@ Hetzner Server (NVMe) bash infra/setup_server.sh ``` -This creates the `beanflows_service` user, data directories, installs age + sops + rclone, generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys. +This creates the `beanflows_service` user, data directories, installs all tools (git, curl, age, sops, rclone, uv), generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys. ### 2. Add keys to GitLab and SOPS @@ -57,7 +57,7 @@ git push ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh ``` -This installs uv (as service user), clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. +This clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. (All tools must already be installed by setup_server.sh.) ### 4. Set up R2 backup diff --git a/infra/setup_server.sh b/infra/setup_server.sh index b8f46f1..31e141e 100644 --- a/infra/setup_server.sh +++ b/infra/setup_server.sh @@ -1,5 +1,5 @@ #!/bin/bash -# One-time server setup: create service user, SSH deploy key, age keypair. +# One-time server setup: create service user, install tools, SSH deploy key, age keypair. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh. # # Usage: @@ -8,8 +8,8 @@ # What it does: # 1. Creates beanflows_service user (nologin) + adds to docker group # 2. Creates /opt/materia + /data/materia/landing with correct ownership -# 3. Generates ed25519 SSH deploy key for GitLab read access -# 4. Installs age + sops + rclone to /usr/local/bin (as root) +# 3. Installs git, curl, age, sops, rclone, uv +# 4. Generates ed25519 SSH deploy key for GitLab read access # 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user) # 6. Prints both public keys + numbered next-step instructions @@ -33,7 +33,6 @@ if ! id "${SERVICE_USER}" >/dev/null 2>&1; then useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}" fi usermod -aG docker "${SERVICE_USER}" -log "User OK." # ── Directories ─────────────────────────────────────────────────────────────── @@ -41,7 +40,12 @@ log "Creating directories..." mkdir -p "${APP_DIR}" "${DATA_DIR}/landing" chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}" chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}" -log "Directories OK." + +# ── System tools ────────────────────────────────────────────────────────────── + +log "Installing system tools..." +apt-get update -q +apt-get install -y -q git curl ca-certificates # ── SSH deploy key ──────────────────────────────────────────────────────────── @@ -54,18 +58,18 @@ if [ ! -f "${DEPLOY_KEY}" ]; then -f "${DEPLOY_KEY}" -N "" -C "materia-deploy" fi -sudo -u "${SERVICE_USER}" bash -c "cat > ${SSH_DIR}/config" < "${SSH_DIR}/config" <> ${SSH_DIR}/known_hosts 2>/dev/null; \ - sort -u ${SSH_DIR}/known_hosts -o ${SSH_DIR}/known_hosts" +ssh-keyscan -H gitlab.com >> "${SSH_DIR}/known_hosts" 2>/dev/null +sort -u "${SSH_DIR}/known_hosts" -o "${SSH_DIR}/known_hosts" +chown "${SERVICE_USER}:${SERVICE_USER}" "${SSH_DIR}/known_hosts" chmod 644 "${SSH_DIR}/known_hosts" -log "SSH deploy key OK." # ── age ─────────────────────────────────────────────────────────────────────── @@ -119,6 +123,13 @@ else log "Age keypair already exists — skipping." fi +# ── uv (installed as service user) ──────────────────────────────────────────── + +if [ ! -f "/home/${SERVICE_USER}/.local/bin/uv" ]; then + log "Installing uv..." + sudo -u "${SERVICE_USER}" bash -c 'curl -LsSf https://astral.sh/uv/install.sh | sh' +fi + # ── Summary ─────────────────────────────────────────────────────────────────── DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub")