deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge(infra): consolidate tool installs in setup, strip bootstrap to essentials

Browse Source 
Merges worktree-sops-supervisor-docs → master.

Summary of changes:
- setup_server.sh: now installs all tools (git, curl, age, sops, rclone, uv) —
  single source of truth for server provisioning
- bootstrap_supervisor.sh: stripped to ~45 lines — zero tool installs, only
  clone/fetch + decrypt + uv sync + systemd enable
- readme.md: updated descriptions to reflect new responsibilities

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
2026-02-27 06:57:09 +01:00
parent ba3994a63b cf65fa16b6
commit c778903264
6 changed files with 206 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.claude/ralph-loop.local.md
View File

@@ -1,9 +0,0 @@
---
active: true
iteration: 1
max_iterations: 0
completion_promise: "all phases complete and tagged"
started_at: "2026-02-25T23:17:16Z"
---
implement the remaining phases from the current plan (phases 2 through 7) for the quart_saas_boilerplate template and beanflows update

153
infra/bootstrap_supervisor.sh
View File

@@ -1,146 +1,69 @@
#!/bin/bash #!/bin/bash
# Bootstrap script for Materia supervisor instance. # Bootstrap Materia supervisor after setup_server.sh + adding keys.
# Run once on a fresh server after setup_server.sh. # Run once on the server after SSH deploy key is added to GitLab
# and the server age key is committed to .env.prod.sops.
# #
# Usage: # Usage:
# ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh # ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
# #
# Prerequisites: # Prerequisites:
# - age keypair exists at /opt/materia/age-key.txt # - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair, uv)
# (or SOPS_AGE_KEY_FILE env var pointing elsewhere) # - Deploy key added to GitLab (Settings → Repository → Deploy Keys)
# - The server age public key is already in .sops.yaml and .env.prod.sops # - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed
# (run setup_server.sh first, then add the key and re-commit)
# - GITLAB_READ_TOKEN is set (GitLab project access token, read-only)
set -euo pipefail set -euo pipefail
echo "=== Materia Supervisor Bootstrap ===" SERVICE_USER="beanflows_service"
echo "This script will:"
echo " 1. Install dependencies (git, uv, sops, age)"
echo " 2. Clone the materia repository"
echo " 3. Decrypt secrets from .env.prod.sops"
echo " 4. Set up systemd service"
echo " 5. Start the supervisor"
echo ""
if [ "$EUID" -ne 0 ]; then
echo "ERROR: This script must be run as root"
exit 1
fi
# ── Configuration ──────────────────────────────────────────
REPO_DIR="/opt/materia" REPO_DIR="/opt/materia"
GITLAB_PROJECT="deemanone/materia" GITLAB_PROJECT="deemanone/materia"
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}" UV="/home/${SERVICE_USER}/.local/bin/uv"
if [ -z "${GITLAB_READ_TOKEN:-}" ]; then [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; }
echo "ERROR: GITLAB_READ_TOKEN not set"
echo " export GITLAB_READ_TOKEN=<gitlab-project-access-token>" # ── Check age keypair ─────────────────────────────────────────────────────────
AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt"
if [ ! -f "${AGE_KEY_FILE}" ]; then
echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}"
echo "Run infra/setup_server.sh first, then add the printed keys, then re-run."
exit 1 exit 1
fi fi
REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git" # ── Clone or update repository ────────────────────────────────────────────────
# ── System dependencies ──────────────────────────────────── if [ -d "${REPO_DIR}/.git" ]; then
echo "--- Installing system dependencies ---" sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin
apt-get update -q
apt-get install -y -q git curl ca-certificates
# ── uv ─────────────────────────────────────────────────────
echo "--- Installing uv ---"
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh
export PATH="$HOME/.local/bin:$PATH"
echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc
fi
# ── sops + age ─────────────────────────────────────────────
echo "--- Installing sops + age ---"
ARCH=$(uname -m)
case "$ARCH" in
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
esac
if ! command -v age &>/dev/null; then
AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz
fi
if ! command -v sops &>/dev/null; then
SOPS_VERSION="v3.12.1"
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops
chmod +x /usr/local/bin/sops
fi
# ── Clone repository ───────────────────────────────────────
echo "--- Cloning repository ---"
if [ -d "$REPO_DIR/.git" ]; then
echo "Repository already exists — fetching latest tags..."
cd "$REPO_DIR"
git fetch --tags --prune-tags origin
else else
git clone "$REPO_URL" "$REPO_DIR" sudo -u "${SERVICE_USER}" git clone \
cd "$REPO_DIR" "git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}"
fi fi
# Checkout latest release tag (same logic as supervisor) LATEST_TAG=$(sudo -u "${SERVICE_USER}" \
LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1) git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1)
if [ -n "$LATEST_TAG" ]; then if [ -n "${LATEST_TAG}" ]; then
echo "Checking out $LATEST_TAG..." sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}"
git checkout --detach "$LATEST_TAG"
else
echo "No release tags found — staying on current HEAD"
fi fi
# ── Check age keypair ────────────────────────────────────── # ── Decrypt secrets ───────────────────────────────────────────────────────────
echo "--- Checking age keypair ---"
if [ ! -f "$AGE_KEY_FILE" ]; then
echo "ERROR: Age keypair not found at $AGE_KEY_FILE"
echo ""
echo "Run infra/setup_server.sh first to generate the keypair, then:"
echo " 1. Copy the public key from setup_server.sh output"
echo " 2. Add it to .sops.yaml on your workstation"
echo " 3. Run: sops updatekeys .env.prod.sops"
echo " 4. Commit + push and re-run this bootstrap"
exit 1
fi
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
# ── Decrypt secrets ──────────────────────────────────────── sudo -u "${SERVICE_USER}" bash -c \
echo "--- Decrypting secrets from .env.prod.sops ---" "sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env"
sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env" chmod 600 "${REPO_DIR}/.env"
chmod 600 "$REPO_DIR/.env"
echo "Secrets written to $REPO_DIR/.env"
# ── Data directories ─────────────────────────────────────── # ── Python dependencies ───────────────────────────────────────────────────────
echo "--- Creating data directories ---"
mkdir -p /data/materia/landing
# ── Python dependencies ──────────────────────────────────── sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages"
echo "--- Installing Python dependencies ---"
cd "$REPO_DIR"
uv sync --all-packages
# ── Systemd service ──────────────────────────────────────── # ── Systemd service ───────────────────────────────────────────────────────────
echo "--- Setting up systemd service ---"
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service
echo "--- Enabling and starting service ---" cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" /etc/systemd/system/
systemctl daemon-reload systemctl daemon-reload
systemctl enable materia-supervisor systemctl enable --now materia-supervisor
systemctl restart materia-supervisor
echo "" echo ""
echo "=== Bootstrap complete! ===" echo "=== Bootstrap complete! ==="
echo "" echo ""
echo "Check status: systemctl status materia-supervisor" echo "Check status: systemctl status materia-supervisor"
echo "View logs: journalctl -u materia-supervisor -f" echo "View logs: journalctl -u materia-supervisor -f"
echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status" echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status"
echo "" echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)"
echo "Repo: $REPO_DIR"
echo "Tag: $(cd $REPO_DIR && git describe --tags --always)"

43
infra/readme.md
View File

@@ -6,15 +6,17 @@ Single-server local-first setup for BeanFlows.coffee on Hetzner NVMe.
``` ```
Hetzner Server (NVMe) Hetzner Server (NVMe)
├── /opt/materia/ # Git repo (checked out at latest release tag) ├── beanflows_service (system user, nologin)
├── /opt/materia/age-key.txt # Server age keypair (chmod 600, gitignored) │ ├── ~/.ssh/materia_deploy # ed25519 deploy key for GitLab read access
├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time │ └── ~/.config/sops/age/keys.txt # age keypair (auto-discovered by SOPS)
├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed) ├── /opt/materia/ # Git repo (owned by beanflows_service, latest release tag)
├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write ├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time
├── /data/materia/analytics.duckdb # Read-only serving copy for web app ├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed)
├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write
├── /data/materia/analytics.duckdb # Read-only serving copy for web app
└── systemd services: └── systemd services:
├── materia-supervisor # Python supervisor: extract → transform → export → deploy ├── materia-supervisor # Python supervisor: extract → transform → export → deploy
└── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours └── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours
``` ```
## Data Flow ## Data Flow
@@ -33,15 +35,16 @@ Hetzner Server (NVMe)
bash infra/setup_server.sh bash infra/setup_server.sh
``` ```
This creates data directories, installs age, and generates the server age keypair at `/opt/materia/age-key.txt`. It prints the server's age public key. This creates the `beanflows_service` user, data directories, installs all tools (git, curl, age, sops, rclone, uv), generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys.
### 2. Add the server key to SOPS ### 2. Add keys to GitLab and SOPS
On your workstation:
```bash ```bash
# Add the server public key to .sops.yaml # Add the SSH deploy key to GitLab:
# Then re-encrypt prod secrets to include the server key: # → Repository Settings → Deploy Keys → Add key (read-only)
# Add the server age public key to .sops.yaml on your workstation,
# then re-encrypt prod secrets to include the server key:
sops updatekeys .env.prod.sops sops updatekeys .env.prod.sops
git add .sops.yaml .env.prod.sops git add .sops.yaml .env.prod.sops
git commit -m "chore: add server age key" git commit -m "chore: add server age key"
@@ -51,18 +54,19 @@ git push
### 3. Bootstrap the supervisor ### 3. Bootstrap the supervisor
```bash ```bash
# Requires GITLAB_READ_TOKEN (GitLab project access token, read-only)
export GITLAB_READ_TOKEN=<token>
ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
``` ```
This installs uv + sops + age, clones the repo, decrypts secrets, installs Python dependencies, and starts the supervisor service. This clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. (All tools must already be installed by setup_server.sh.)
### 4. Set up R2 backup ### 4. Set up R2 backup
```bash ```bash
apt install rclone apt install rclone
cp infra/backup/rclone.conf.example /root/.config/rclone/rclone.conf # Configure rclone as the service user (used by the backup timer):
sudo -u beanflows_service mkdir -p /home/beanflows_service/.config/rclone
sudo -u beanflows_service cp infra/backup/rclone.conf.example \
/home/beanflows_service/.config/rclone/rclone.conf
# Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint) # Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint)
cp infra/backup/materia-backup.service /etc/systemd/system/ cp infra/backup/materia-backup.service /etc/systemd/system/
cp infra/backup/materia-backup.timer /etc/systemd/system/ cp infra/backup/materia-backup.timer /etc/systemd/system/
@@ -90,6 +94,7 @@ make secrets-edit-prod
`bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup. `bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup.
`web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically). `web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically).
SOPS auto-discovers the service user's age key at `~/.config/sops/age/keys.txt` (XDG default).
## Deploy model (pull-based) ## Deploy model (pull-based)
@@ -109,7 +114,7 @@ systemctl status materia-supervisor
journalctl -u materia-supervisor -f journalctl -u materia-supervisor -f
# Workflow status table # Workflow status table
cd /opt/materia && uv run python src/materia/supervisor.py status cd /opt/materia && sudo -u beanflows_service uv run python src/materia/supervisor.py status
# Backup timer status # Backup timer status
systemctl list-timers materia-backup.timer systemctl list-timers materia-backup.timer

180
infra/setup_server.sh
View File

@@ -1,74 +1,172 @@
#!/bin/bash #!/bin/bash
# One-time server setup: create data directories, generate age keypair. # One-time server setup: create service user, install tools, SSH deploy key, age keypair.
# Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh.
# #
# Usage: # Usage:
# bash infra/setup_server.sh # bash infra/setup_server.sh
#
# What it does:
# 1. Creates beanflows_service user (nologin) + adds to docker group
# 2. Creates /opt/materia + /data/materia/landing with correct ownership
# 3. Installs git, curl, age, sops, rclone, uv
# 4. Generates ed25519 SSH deploy key for GitLab read access
# 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user)
# 6. Prints both public keys + numbered next-step instructions
set -euo pipefail set -euo pipefail
REPO_DIR="/opt/materia" SERVICE_USER="beanflows_service"
AGE_KEY_FILE="$REPO_DIR/age-key.txt" APP_DIR="/opt/materia"
DATA_DIR="/data/materia"
SSH_DIR="/home/${SERVICE_USER}/.ssh"
DEPLOY_KEY="${SSH_DIR}/materia_deploy"
SOPS_AGE_DIR="/home/${SERVICE_USER}/.config/sops/age"
if [ "$EUID" -ne 0 ]; then [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root: sudo bash infra/setup_server.sh"; exit 1; }
echo "ERROR: This script must be run as root"
exit 1 log() { echo "$(date '+%H:%M:%S') ==> $*"; }
# ── Service user ──────────────────────────────────────────────────────────────
log "Creating service user ${SERVICE_USER}..."
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}"
fi
usermod -aG docker "${SERVICE_USER}"
# ── Directories ───────────────────────────────────────────────────────────────
log "Creating directories..."
mkdir -p "${APP_DIR}" "${DATA_DIR}/landing"
chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}"
chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}"
# ── System tools ──────────────────────────────────────────────────────────────
log "Installing system tools..."
apt-get update -q
apt-get install -y -q git curl ca-certificates
# ── SSH deploy key ────────────────────────────────────────────────────────────
log "Setting up SSH deploy key..."
sudo -u "${SERVICE_USER}" mkdir -p "${SSH_DIR}"
chmod 700 "${SSH_DIR}"
if [ ! -f "${DEPLOY_KEY}" ]; then
sudo -u "${SERVICE_USER}" ssh-keygen -t ed25519 \
-f "${DEPLOY_KEY}" -N "" -C "materia-deploy"
fi fi
# ── Create data directories ──────────────────────────────── cat > "${SSH_DIR}/config" <<EOF
echo "--- Creating data directories ---" Host gitlab.com
mkdir -p /data/materia/landing IdentityFile ${DEPLOY_KEY}
mkdir -p "$REPO_DIR" IdentitiesOnly yes
echo "Data dir: /data/materia" EOF
chown "${SERVICE_USER}:${SERVICE_USER}" "${SSH_DIR}/config"
chmod 600 "${SSH_DIR}/config"
ssh-keyscan -H gitlab.com >> "${SSH_DIR}/known_hosts" 2>/dev/null
sort -u "${SSH_DIR}/known_hosts" -o "${SSH_DIR}/known_hosts"
chown "${SERVICE_USER}:${SERVICE_USER}" "${SSH_DIR}/known_hosts"
chmod 644 "${SSH_DIR}/known_hosts"
# ── age ───────────────────────────────────────────────────────────────────────
# ── Install age ────────────────────────────────────────────
echo "--- Installing age ---"
ARCH=$(uname -m) ARCH=$(uname -m)
case "$ARCH" in case "${ARCH}" in
x86_64) ARCH_AGE="amd64" ;; x86_64) ARCH_TAG="amd64" ;;
aarch64) ARCH_AGE="arm64" ;; aarch64) ARCH_TAG="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;; *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;;
esac esac
if ! command -v age-keygen &>/dev/null; then if ! command -v age-keygen >/dev/null 2>&1; then
log "Installing age..."
AGE_VERSION="v1.3.1" AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz rm /tmp/age.tar.gz
echo "age installed to /usr/local/bin" log "age installed."
fi fi
# ── Generate age keypair ─────────────────────────────────── # ── sops ──────────────────────────────────────────────────────────────────────
echo "--- Setting up age keypair ---"
if [ -f "$AGE_KEY_FILE" ]; then if ! command -v sops >/dev/null 2>&1; then
echo "Keypair already exists at $AGE_KEY_FILE — skipping generation" log "Installing sops..."
SOPS_VERSION="v3.12.1"
curl -fsSL \
"https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \
-o /usr/local/bin/sops
chmod +x /usr/local/bin/sops
log "sops installed."
fi
# ── rclone ────────────────────────────────────────────────────────────────────
if ! command -v rclone >/dev/null 2>&1; then
log "Installing rclone..."
curl -fsSL https://rclone.org/install.sh | bash
log "rclone installed."
fi
# ── age keypair (as service user, XDG path auto-discovered by SOPS) ───────────
log "Generating age keypair for ${SERVICE_USER}..."
AGE_KEY_FILE="${SOPS_AGE_DIR}/keys.txt"
if [ ! -f "${AGE_KEY_FILE}" ]; then
sudo -u "${SERVICE_USER}" mkdir -p "${SOPS_AGE_DIR}"
sudo -u "${SERVICE_USER}" age-keygen -o "${AGE_KEY_FILE}"
chmod 600 "${AGE_KEY_FILE}"
log "Age keypair generated at ${AGE_KEY_FILE}"
else else
age-keygen -o "$AGE_KEY_FILE" 2>/dev/null log "Age keypair already exists — skipping."
chmod 600 "$AGE_KEY_FILE"
echo "Generated: $AGE_KEY_FILE"
fi fi
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}') # ── uv (installed as service user) ────────────────────────────────────────────
if [ ! -f "/home/${SERVICE_USER}/.local/bin/uv" ]; then
log "Installing uv..."
sudo -u "${SERVICE_USER}" bash -c 'curl -LsSf https://astral.sh/uv/install.sh | sh'
fi
# ── Summary ───────────────────────────────────────────────────────────────────
DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub")
AGE_PUB=$(grep "public key:" "${AGE_KEY_FILE}" | awk '{print $NF}')
echo "" echo ""
echo "==================================================================" echo "=================================================================="
echo " Server age public key:"
echo "" echo ""
echo " $AGE_PUB" echo " SSH deploy key (add to GitLab → Settings → Deploy Keys):"
echo "" echo ""
echo " Add this key to .sops.yaml on your workstation:" echo " ${DEPLOY_PUB}"
echo "" echo ""
echo " creation_rules:" echo " Server age public key (add to .sops.yaml):"
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
echo " age: >-"
echo " <dev-key>"
echo " + $AGE_PUB"
echo "" echo ""
echo " Then re-encrypt the prod secrets file:" echo " ${AGE_PUB}"
echo " sops updatekeys .env.prod.sops" echo ""
echo " git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'" echo "=================================================================="
echo " git push" echo ""
echo " Next steps:"
echo ""
echo " 1. Add the SSH deploy key to GitLab:"
echo " → Repository Settings → Deploy Keys → Add key (read-only)"
echo ""
echo " 2. Add the age public key to .sops.yaml on your workstation:"
echo " creation_rules:"
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
echo " age: >-"
echo " <dev-key>"
echo " + ${AGE_PUB}"
echo ""
echo " 3. Re-encrypt prod secrets to include the server key:"
echo " sops updatekeys .env.prod.sops"
echo " git add .sops.yaml .env.prod.sops"
echo " git commit -m 'chore: add server age key'"
echo " git push"
echo ""
echo " 4. Run bootstrap:"
echo " bash infra/bootstrap_supervisor.sh"
echo "" echo ""
echo " Then run infra/bootstrap_supervisor.sh to complete setup."
echo "==================================================================" echo "=================================================================="

3
infra/supervisor/materia-supervisor.service
View File

@@ -5,12 +5,13 @@ Wants=network-online.target
[Service] [Service]
Type=simple Type=simple
User=root User=beanflows_service
WorkingDirectory=/opt/materia WorkingDirectory=/opt/materia
ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py' ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py'
Restart=always Restart=always
RestartSec=10 RestartSec=10
EnvironmentFile=/opt/materia/.env EnvironmentFile=/opt/materia/.env
Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin
Environment=LANDING_DIR=/data/materia/landing Environment=LANDING_DIR=/data/materia/landing
Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb
Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb

51
web/deploy.sh
View File

@@ -1,60 +1,15 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
# ── Ensure sops + age are installed ───────────────────────
APP_DIR="$(cd "$(dirname "$0")" && pwd)" APP_DIR="$(cd "$(dirname "$0")" && pwd)"
BIN_DIR="$APP_DIR/bin"
mkdir -p "$BIN_DIR"
export PATH="$BIN_DIR:$PATH"
ARCH=$(uname -m)
case "$ARCH" in
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
esac
if ! command -v age &>/dev/null; then
echo "==> Installing age to $BIN_DIR..."
AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C "$BIN_DIR" --strip-components=1 age/age age/age-keygen
chmod +x "$BIN_DIR/age" "$BIN_DIR/age-keygen"
rm /tmp/age.tar.gz
fi
# ── Verify sops is installed (setup_server.sh installs it to /usr/local/bin) ──
if ! command -v sops &>/dev/null; then if ! command -v sops &>/dev/null; then
echo "==> Installing sops to $BIN_DIR..." echo "ERROR: sops not found — run infra/setup_server.sh first"
SOPS_VERSION="v3.12.1"
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o "$BIN_DIR/sops"
chmod +x "$BIN_DIR/sops"
fi
# ── Ensure age keypair exists ─────────────────────────────
# Key file lives at repo root (one level up from web/)
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$APP_DIR/../age-key.txt}"
AGE_KEY_FILE="$(realpath "$AGE_KEY_FILE")"
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
if [ ! -f "$AGE_KEY_FILE" ]; then
echo "==> Generating age keypair at $AGE_KEY_FILE..."
age-keygen -o "$AGE_KEY_FILE" 2>&1
chmod 600 "$AGE_KEY_FILE"
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
echo ""
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!! NEW SERVER — add this public key to .sops.yaml: !!"
echo "!! !!"
echo "!! $AGE_PUB !!"
echo "!! !!"
echo "!! Then run: sops updatekeys .env.prod.sops !!"
echo "!! Commit, push, and re-deploy. !!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo ""
exit 1 exit 1
fi fi
# ── Decrypt secrets ─────────────────────────────────────── # ── Decrypt secrets (SOPS auto-discovers age key from ~/.config/sops/age/) ────
echo "==> Decrypting secrets from .env.prod.sops..." echo "==> Decrypting secrets from .env.prod.sops..."
sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env" sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env"
chmod 600 "$APP_DIR/.env" chmod 600 "$APP_DIR/.env"