deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(infra): replace Pulumi ESC with SOPS in bootstrap + setup scripts

Browse Source 
- bootstrap_supervisor.sh: remove esc CLI + PULUMI_ACCESS_TOKEN; install
  sops+age; check age keypair exists; decrypt .env.prod.sops → .env;
  checkout latest release tag; use uv sync --all-packages
- setup_server.sh: add age keypair generation at /opt/materia/age-key.txt;
  install age binary; print public key with .sops.yaml instructions

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
2026-02-26 12:03:11 +01:00
parent 5d7d53a260
commit 95f881827e
2 changed files with 148 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
infra/bootstrap_supervisor.sh
View File

@@ -1,117 +1,146 @@
#!/bin/bash #!/bin/bash
# Bootstrap script for Materia supervisor instance # Bootstrap script for Materia supervisor instance.
# Run this once on a new supervisor to set it up # Run once on a fresh server after setup_server.sh.
# #
# Usage: # Usage:
# From CI/CD or locally: # ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
# ssh root@<supervisor_ip> 'bash -s' < infra/bootstrap_supervisor.sh
# #
# Or on the supervisor itself: # Prerequisites:
# curl -fsSL <url-to-this-script> | bash # - age keypair exists at /opt/materia/age-key.txt
# (or SOPS_AGE_KEY_FILE env var pointing elsewhere)
# - The server age public key is already in .sops.yaml and .env.prod.sops
# (run setup_server.sh first, then add the key and re-commit)
# - GITLAB_READ_TOKEN is set (GitLab project access token, read-only)
set -euo pipefail set -euo pipefail
echo "=== Materia Supervisor Bootstrap ===" echo "=== Materia Supervisor Bootstrap ==="
echo "This script will:" echo "This script will:"
echo " 1. Install dependencies (git, uv, esc)" echo " 1. Install dependencies (git, uv, sops, age)"
echo " 2. Clone the materia repository" echo " 2. Clone the materia repository"
echo " 3. Setup systemd service" echo " 3. Decrypt secrets from .env.prod.sops"
echo " 4. Start the supervisor" echo " 4. Set up systemd service"
echo " 5. Start the supervisor"
echo "" echo ""
# Check if we're root
if [ "$EUID" -ne 0 ]; then if [ "$EUID" -ne 0 ]; then
echo "ERROR: This script must be run as root" echo "ERROR: This script must be run as root"
exit 1 exit 1
fi fi
# Configuration # ── Configuration ──────────────────────────────────────────
REPO_DIR="/opt/materia" REPO_DIR="/opt/materia"
GITLAB_PROJECT="deemanone/materia" GITLAB_PROJECT="deemanone/materia"
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}"
# GITLAB_READ_TOKEN should be set in Pulumi ESC (beanflows/prod)
if [ -z "${GITLAB_READ_TOKEN:-}" ]; then if [ -z "${GITLAB_READ_TOKEN:-}" ]; then
echo "ERROR: GITLAB_READ_TOKEN environment variable not set" echo "ERROR: GITLAB_READ_TOKEN not set"
echo "Please add it to Pulumi ESC (beanflows/prod) first" echo " export GITLAB_READ_TOKEN=<gitlab-project-access-token>"
exit 1 exit 1
fi fi
REPO_URL="https://gitlab-ci-token:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git" REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git"
# ── System dependencies ────────────────────────────────────
echo "--- Installing system dependencies ---" echo "--- Installing system dependencies ---"
apt-get update apt-get update -q
apt-get install -y git curl python3-pip apt-get install -y -q git curl ca-certificates
# ── uv ─────────────────────────────────────────────────────
echo "--- Installing uv ---" echo "--- Installing uv ---"
if ! command -v uv &> /dev/null; then if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh curl -LsSf https://astral.sh/uv/install.sh | sh
export PATH="$HOME/.cargo/bin:$PATH" export PATH="$HOME/.local/bin:$PATH"
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /root/.bashrc echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc
fi fi
echo "--- Installing Pulumi ESC ---" # ── sops + age ─────────────────────────────────────────────
if ! command -v esc &> /dev/null; then echo "--- Installing sops + age ---"
curl -fsSL https://get.pulumi.com/esc/install.sh | sh ARCH=$(uname -m)
export PATH="$HOME/.pulumi/bin:$PATH" case "$ARCH" in
echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
esac
if ! command -v age &>/dev/null; then
AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz
fi fi
echo "--- Setting up Pulumi ESC authentication ---" if ! command -v sops &>/dev/null; then
if [ -z "${PULUMI_ACCESS_TOKEN:-}" ]; then SOPS_VERSION="v3.12.1"
echo "ERROR: PULUMI_ACCESS_TOKEN environment variable not set" curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops
echo "Please set it before running this script:" chmod +x /usr/local/bin/sops
echo " export PULUMI_ACCESS_TOKEN=<your-token>"
exit 1
fi fi
esc login --token "$PULUMI_ACCESS_TOKEN" # ── Clone repository ───────────────────────────────────────
echo "--- Loading secrets from Pulumi ESC ---"
eval $(esc env open beanflows/prod --format shell)
echo "--- Cloning repository ---" echo "--- Cloning repository ---"
if [ -d "$REPO_DIR" ]; then if [ -d "$REPO_DIR/.git" ]; then
echo "Repository already exists, pulling latest..." echo "Repository already exists — fetching latest tags..."
cd "$REPO_DIR" cd "$REPO_DIR"
git pull origin master git fetch --tags --prune-tags origin
else else
git clone "$REPO_URL" "$REPO_DIR" git clone "$REPO_URL" "$REPO_DIR"
cd "$REPO_DIR" cd "$REPO_DIR"
fi fi
# Checkout latest release tag (same logic as supervisor)
LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1)
if [ -n "$LATEST_TAG" ]; then
echo "Checking out $LATEST_TAG..."
git checkout --detach "$LATEST_TAG"
else
echo "No release tags found — staying on current HEAD"
fi
# ── Check age keypair ──────────────────────────────────────
echo "--- Checking age keypair ---"
if [ ! -f "$AGE_KEY_FILE" ]; then
echo "ERROR: Age keypair not found at $AGE_KEY_FILE"
echo ""
echo "Run infra/setup_server.sh first to generate the keypair, then:"
echo " 1. Copy the public key from setup_server.sh output"
echo " 2. Add it to .sops.yaml on your workstation"
echo " 3. Run: sops updatekeys .env.prod.sops"
echo " 4. Commit + push and re-run this bootstrap"
exit 1
fi
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
# ── Decrypt secrets ────────────────────────────────────────
echo "--- Decrypting secrets from .env.prod.sops ---"
sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env"
chmod 600 "$REPO_DIR/.env"
echo "Secrets written to $REPO_DIR/.env"
# ── Data directories ───────────────────────────────────────
echo "--- Creating data directories ---" echo "--- Creating data directories ---"
mkdir -p /data/materia/landing/psd mkdir -p /data/materia/landing
# ── Python dependencies ────────────────────────────────────
echo "--- Installing Python dependencies ---" echo "--- Installing Python dependencies ---"
uv sync cd "$REPO_DIR"
uv sync --all-packages
echo "--- Creating environment file ---"
cat > "$REPO_DIR/.env" <<EOF
# Environment variables for supervisor
# Loaded from Pulumi ESC: beanflows/prod
PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
LANDING_DIR=/data/materia/landing
DUCKDB_PATH=/data/materia/lakehouse.duckdb
SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb
EOF
# ── Systemd service ────────────────────────────────────────
echo "--- Setting up systemd service ---" echo "--- Setting up systemd service ---"
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service
echo "--- Enabling and starting service ---" echo "--- Enabling and starting service ---"
systemctl daemon-reload systemctl daemon-reload
systemctl enable materia-supervisor systemctl enable materia-supervisor
systemctl start materia-supervisor systemctl restart materia-supervisor
echo "" echo ""
echo "=== Bootstrap complete! ===" echo "=== Bootstrap complete! ==="
echo "" echo ""
echo "Supervisor is now running. Check status with:" echo "Check status: systemctl status materia-supervisor"
echo " systemctl status materia-supervisor" echo "View logs: journalctl -u materia-supervisor -f"
echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status"
echo "" echo ""
echo "View logs with:" echo "Repo: $REPO_DIR"
echo " journalctl -u materia-supervisor -f" echo "Tag: $(cd $REPO_DIR && git describe --tags --always)"
echo ""
echo "Repository location: $REPO_DIR"
echo "Current commit: $(cd $REPO_DIR && git rev-parse --short HEAD)"

89
infra/setup_server.sh
View File

@@ -1,45 +1,74 @@
#!/bin/bash #!/bin/bash
# One-time server setup: create app user, /opt/beanflows, and GitLab deploy key. # One-time server setup: create data directories, generate age keypair.
# Run as root on a fresh Hetzner server before bootstrapping the supervisor. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh.
# #
# Usage: # Usage:
# bash infra/setup_server.sh # bash infra/setup_server.sh
set -euo pipefail set -euo pipefail
APP_USER="beanflows_service" REPO_DIR="/opt/materia"
APP_DIR="/opt/beanflows" AGE_KEY_FILE="$REPO_DIR/age-key.txt"
KEY_PATH="/home/$APP_USER/.ssh/gitlab_deploy"
# Create system user with a home dir (needed for .ssh) but no login shell if [ "$EUID" -ne 0 ]; then
if ! id "$APP_USER" &>/dev/null; then echo "ERROR: This script must be run as root"
useradd --system --create-home --shell /usr/sbin/nologin "$APP_USER" exit 1
echo "Created user: $APP_USER"
else
echo "User $APP_USER already exists, skipping"
fi fi
# Create app directory owned by app user # ── Create data directories ────────────────────────────────
mkdir -p "$APP_DIR" echo "--- Creating data directories ---"
chown "$APP_USER:$APP_USER" "$APP_DIR" mkdir -p /data/materia/landing
chmod 750 "$APP_DIR" mkdir -p "$REPO_DIR"
echo "Created $APP_DIR (owner: $APP_USER)" echo "Data dir: /data/materia"
# Generate deploy key if not already present # ── Install age ────────────────────────────────────────────
if [ ! -f "$KEY_PATH" ]; then echo "--- Installing age ---"
mkdir -p "/home/$APP_USER/.ssh" ARCH=$(uname -m)
ssh-keygen -t ed25519 -f "$KEY_PATH" -N "" -C "beanflows-server" case "$ARCH" in
chown -R "$APP_USER:$APP_USER" "/home/$APP_USER/.ssh" x86_64) ARCH_AGE="amd64" ;;
chmod 700 "/home/$APP_USER/.ssh" aarch64) ARCH_AGE="arm64" ;;
chmod 600 "$KEY_PATH" *) echo "Unsupported architecture: $ARCH"; exit 1 ;;
chmod 644 "$KEY_PATH.pub" esac
echo "Generated deploy key: $KEY_PATH"
else if ! command -v age-keygen &>/dev/null; then
echo "Deploy key already exists, skipping" AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz
echo "age installed to /usr/local/bin"
fi fi
# ── Generate age keypair ───────────────────────────────────
echo "--- Setting up age keypair ---"
if [ -f "$AGE_KEY_FILE" ]; then
echo "Keypair already exists at $AGE_KEY_FILE — skipping generation"
else
age-keygen -o "$AGE_KEY_FILE" 2>/dev/null
chmod 600 "$AGE_KEY_FILE"
echo "Generated: $AGE_KEY_FILE"
fi
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
echo "" echo ""
echo "=== Add this deploy key to GitLab ===" echo "=================================================================="
echo "GitLab → repo → Settings → Repository → Deploy Keys (read-only)" echo " Server age public key:"
echo "" echo ""
cat "$KEY_PATH.pub" echo " $AGE_PUB"
echo ""
echo " Add this key to .sops.yaml on your workstation:"
echo ""
echo " creation_rules:"
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
echo " age: >-"
echo " <dev-key>"
echo " + $AGE_PUB"
echo ""
echo " Then re-encrypt the prod secrets file:"
echo " sops updatekeys .env.prod.sops"
echo " git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'"
echo " git push"
echo ""
echo " Then run infra/bootstrap_supervisor.sh to complete setup."
echo "=================================================================="