deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(infra): use beanflows_service for supervisor

Browse Source 
- materia-supervisor.service: User=root → User=beanflows_service,
  add PATH so uv (~/.local/bin) is found without a login shell
- setup_server.sh: full rewrite — creates beanflows_service (nologin),
  generates SSH deploy key + age keypair as service user at XDG path
  (~/.config/sops/age/keys.txt), installs age/sops/rclone as root,
  prints both public keys + numbered next-step instructions
- bootstrap_supervisor.sh: full rewrite — removes GITLAB_READ_TOKEN
  requirement, clones via SSH as service user, installs uv as service
  user, decrypts with SOPS auto-discovery, uv sync as service user,
  systemctl as root
- web/deploy.sh: remove self-contained sops/age install + keypair
  generation; replace with simple sops check (exit if missing) and
  SOPS auto-discovery decrypt (no explicit key file needed)
- infra/readme.md: update architecture diagram for beanflows_service
  paths, update setup steps to match new scripts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
2026-02-26 21:33:31 +01:00
parent b27f06d811
commit 0317cb885f
5 changed files with 227 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
infra/bootstrap_supervisor.sh
View File

@@ -6,129 +6,123 @@
# ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh # ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
# #
# Prerequisites: # Prerequisites:
# - age keypair exists at /opt/materia/age-key.txt # - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair)
# (or SOPS_AGE_KEY_FILE env var pointing elsewhere) # - Deploy key added to GitLab (Settings → Repository → Deploy Keys)
# - The server age public key is already in .sops.yaml and .env.prod.sops # - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed
# (run setup_server.sh first, then add the key and re-commit)
# - GITLAB_READ_TOKEN is set (GitLab project access token, read-only)
set -euo pipefail set -euo pipefail
echo "=== Materia Supervisor Bootstrap ===" SERVICE_USER="beanflows_service"
echo "This script will:"
echo " 1. Install dependencies (git, uv, sops, age)"
echo " 2. Clone the materia repository"
echo " 3. Decrypt secrets from .env.prod.sops"
echo " 4. Set up systemd service"
echo " 5. Start the supervisor"
echo ""
if [ "$EUID" -ne 0 ]; then
echo "ERROR: This script must be run as root"
exit 1
fi
# ── Configuration ──────────────────────────────────────────
REPO_DIR="/opt/materia" REPO_DIR="/opt/materia"
GITLAB_PROJECT="deemanone/materia" GITLAB_PROJECT="deemanone/materia"
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}" UV="/home/${SERVICE_USER}/.local/bin/uv"
if [ -z "${GITLAB_READ_TOKEN:-}" ]; then echo "=== Materia Supervisor Bootstrap ==="
echo "ERROR: GITLAB_READ_TOKEN not set" echo ""
echo " export GITLAB_READ_TOKEN=<gitlab-project-access-token>"
exit 1
fi
REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git" [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; }
# ── System dependencies ────────────────────────────────────────────────────────
# ── System dependencies ────────────────────────────────────
echo "--- Installing system dependencies ---" echo "--- Installing system dependencies ---"
apt-get update -q apt-get update -q
apt-get install -y -q git curl ca-certificates apt-get install -y -q git curl ca-certificates
# ── uv ───────────────────────────────────────────────────── # ── uv (installed as service user) ────────────────────────────────────────────
echo "--- Installing uv ---" echo "--- Installing uv ---"
if ! command -v uv &>/dev/null; then if [ ! -f "${UV}" ]; then
curl -LsSf https://astral.sh/uv/install.sh | sh sudo -u "${SERVICE_USER}" bash -c \
export PATH="$HOME/.local/bin:$PATH" 'curl -LsSf https://astral.sh/uv/install.sh | sh'
echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc
fi fi
# ── sops + age ───────────────────────────────────────────── # ── sops + age (as root, idempotent — setup_server.sh may have done this) ─────
echo "--- Installing sops + age ---" echo "--- Installing sops + age ---"
ARCH=$(uname -m) ARCH=$(uname -m)
case "$ARCH" in case "${ARCH}" in
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;; x86_64) ARCH_TAG="amd64" ;;
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;; aarch64) ARCH_TAG="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;; *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;;
esac esac
if ! command -v age &>/dev/null; then if ! command -v age >/dev/null 2>&1; then
AGE_VERSION="v1.3.1" AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz rm /tmp/age.tar.gz
fi fi
if ! command -v sops &>/dev/null; then if ! command -v sops >/dev/null 2>&1; then
SOPS_VERSION="v3.12.1" SOPS_VERSION="v3.12.1"
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops curl -fsSL \
"https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \
-o /usr/local/bin/sops
chmod +x /usr/local/bin/sops chmod +x /usr/local/bin/sops
fi fi
# ── Clone repository ─────────────────────────────────────── # ── Clone repository via SSH as service user ──────────────────────────────────
echo "--- Cloning repository ---" echo "--- Cloning repository ---"
if [ -d "$REPO_DIR/.git" ]; then if [ -d "${REPO_DIR}/.git" ]; then
echo "Repository already exists — fetching latest tags..." echo "Repository already exists — fetching latest tags..."
cd "$REPO_DIR" sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin
git fetch --tags --prune-tags origin
else else
git clone "$REPO_URL" "$REPO_DIR" sudo -u "${SERVICE_USER}" git clone \
cd "$REPO_DIR" "git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}"
fi fi
# Checkout latest release tag (same logic as supervisor) # Checkout latest release tag (same logic as supervisor)
LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1) LATEST_TAG=$(sudo -u "${SERVICE_USER}" \
if [ -n "$LATEST_TAG" ]; then git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1)
echo "Checking out $LATEST_TAG..." if [ -n "${LATEST_TAG}" ]; then
git checkout --detach "$LATEST_TAG" echo "Checking out ${LATEST_TAG}..."
sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}"
else else
echo "No release tags found — staying on current HEAD" echo "No release tags found — staying on current HEAD"
fi fi
# ── Check age keypair ────────────────────────────────────── # ── Check age keypair ─────────────────────────────────────────────────────────
echo "--- Checking age keypair ---" echo "--- Checking age keypair ---"
if [ ! -f "$AGE_KEY_FILE" ]; then AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt"
echo "ERROR: Age keypair not found at $AGE_KEY_FILE" if [ ! -f "${AGE_KEY_FILE}" ]; then
echo "" echo ""
echo "Run infra/setup_server.sh first to generate the keypair, then:" echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}"
echo " 1. Copy the public key from setup_server.sh output" echo ""
echo " 2. Add it to .sops.yaml on your workstation" echo "Run infra/setup_server.sh first, then:"
echo " 3. Run: sops updatekeys .env.prod.sops" echo " 1. Add the SSH deploy key to GitLab (Settings → Repository → Deploy Keys)"
echo " 4. Commit + push and re-run this bootstrap" echo " 2. Add the age public key to .sops.yaml on your workstation"
echo " 3. Run: sops updatekeys .env.prod.sops && git push"
echo " 4. Re-run this bootstrap"
exit 1 exit 1
fi fi
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
# ── Decrypt secrets ──────────────────────────────────────── # ── Decrypt secrets (as service user — SOPS auto-discovers age key from XDG) ──
echo "--- Decrypting secrets from .env.prod.sops ---" echo "--- Decrypting secrets from .env.prod.sops ---"
sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env" sudo -u "${SERVICE_USER}" bash -c \
chmod 600 "$REPO_DIR/.env" "sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env"
echo "Secrets written to $REPO_DIR/.env" chmod 600 "${REPO_DIR}/.env"
echo "Secrets written to ${REPO_DIR}/.env"
# ── Data directories ───────────────────────────────────────────────────────────
# ── Data directories ───────────────────────────────────────
echo "--- Creating data directories ---" echo "--- Creating data directories ---"
mkdir -p /data/materia/landing mkdir -p /data/materia/landing
chown -R "${SERVICE_USER}:${SERVICE_USER}" /data/materia
# ── Python dependencies (as service user) ─────────────────────────────────────
# ── Python dependencies ────────────────────────────────────
echo "--- Installing Python dependencies ---" echo "--- Installing Python dependencies ---"
cd "$REPO_DIR" sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages"
uv sync --all-packages
# ── Systemd service (as root) ──────────────────────────────────────────────────
# ── Systemd service ────────────────────────────────────────
echo "--- Setting up systemd service ---" echo "--- Setting up systemd service ---"
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" \
/etc/systemd/system/materia-supervisor.service
echo "--- Enabling and starting service ---" echo "--- Enabling and starting service ---"
systemctl daemon-reload systemctl daemon-reload
@@ -138,9 +132,9 @@ systemctl restart materia-supervisor
echo "" echo ""
echo "=== Bootstrap complete! ===" echo "=== Bootstrap complete! ==="
echo "" echo ""
echo "Check status: systemctl status materia-supervisor" echo "Check status: systemctl status materia-supervisor"
echo "View logs: journalctl -u materia-supervisor -f" echo "View logs: journalctl -u materia-supervisor -f"
echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status" echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status"
echo "" echo ""
echo "Repo: $REPO_DIR" echo "Repo: ${REPO_DIR}"
echo "Tag: $(cd $REPO_DIR && git describe --tags --always)" echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)"

43
infra/readme.md
View File

@@ -6,15 +6,17 @@ Single-server local-first setup for BeanFlows.coffee on Hetzner NVMe.
``` ```
Hetzner Server (NVMe) Hetzner Server (NVMe)
├── /opt/materia/ # Git repo (checked out at latest release tag) ├── beanflows_service (system user, nologin)
├── /opt/materia/age-key.txt # Server age keypair (chmod 600, gitignored) │ ├── ~/.ssh/materia_deploy # ed25519 deploy key for GitLab read access
├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time │ └── ~/.config/sops/age/keys.txt # age keypair (auto-discovered by SOPS)
├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed) ├── /opt/materia/ # Git repo (owned by beanflows_service, latest release tag)
├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write ├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time
├── /data/materia/analytics.duckdb # Read-only serving copy for web app ├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed)
├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write
├── /data/materia/analytics.duckdb # Read-only serving copy for web app
└── systemd services: └── systemd services:
├── materia-supervisor # Python supervisor: extract → transform → export → deploy ├── materia-supervisor # Python supervisor: extract → transform → export → deploy
└── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours └── materia-backup.timer # rclone: syncs landing/ to R2 every 6 hours
``` ```
## Data Flow ## Data Flow
@@ -33,15 +35,16 @@ Hetzner Server (NVMe)
bash infra/setup_server.sh bash infra/setup_server.sh
``` ```
This creates data directories, installs age, and generates the server age keypair at `/opt/materia/age-key.txt`. It prints the server's age public key. This creates the `beanflows_service` user, data directories, installs age + sops + rclone, generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys.
### 2. Add the server key to SOPS ### 2. Add keys to GitLab and SOPS
On your workstation:
```bash ```bash
# Add the server public key to .sops.yaml # Add the SSH deploy key to GitLab:
# Then re-encrypt prod secrets to include the server key: # → Repository Settings → Deploy Keys → Add key (read-only)
# Add the server age public key to .sops.yaml on your workstation,
# then re-encrypt prod secrets to include the server key:
sops updatekeys .env.prod.sops sops updatekeys .env.prod.sops
git add .sops.yaml .env.prod.sops git add .sops.yaml .env.prod.sops
git commit -m "chore: add server age key" git commit -m "chore: add server age key"
@@ -51,18 +54,19 @@ git push
### 3. Bootstrap the supervisor ### 3. Bootstrap the supervisor
```bash ```bash
# Requires GITLAB_READ_TOKEN (GitLab project access token, read-only)
export GITLAB_READ_TOKEN=<token>
ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
``` ```
This installs uv + sops + age, clones the repo, decrypts secrets, installs Python dependencies, and starts the supervisor service. This installs uv (as service user), clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key.
### 4. Set up R2 backup ### 4. Set up R2 backup
```bash ```bash
apt install rclone apt install rclone
cp infra/backup/rclone.conf.example /root/.config/rclone/rclone.conf # Configure rclone as the service user (used by the backup timer):
sudo -u beanflows_service mkdir -p /home/beanflows_service/.config/rclone
sudo -u beanflows_service cp infra/backup/rclone.conf.example \
/home/beanflows_service/.config/rclone/rclone.conf
# Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint) # Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint)
cp infra/backup/materia-backup.service /etc/systemd/system/ cp infra/backup/materia-backup.service /etc/systemd/system/
cp infra/backup/materia-backup.timer /etc/systemd/system/ cp infra/backup/materia-backup.timer /etc/systemd/system/
@@ -90,6 +94,7 @@ make secrets-edit-prod
`bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup. `bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup.
`web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically). `web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically).
SOPS auto-discovers the service user's age key at `~/.config/sops/age/keys.txt` (XDG default).
## Deploy model (pull-based) ## Deploy model (pull-based)
@@ -109,7 +114,7 @@ systemctl status materia-supervisor
journalctl -u materia-supervisor -f journalctl -u materia-supervisor -f
# Workflow status table # Workflow status table
cd /opt/materia && uv run python src/materia/supervisor.py status cd /opt/materia && sudo -u beanflows_service uv run python src/materia/supervisor.py status
# Backup timer status # Backup timer status
systemctl list-timers materia-backup.timer systemctl list-timers materia-backup.timer

169
infra/setup_server.sh
View File

@@ -1,74 +1,161 @@
#!/bin/bash #!/bin/bash
# One-time server setup: create data directories, generate age keypair. # One-time server setup: create service user, SSH deploy key, age keypair.
# Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh.
# #
# Usage: # Usage:
# bash infra/setup_server.sh # bash infra/setup_server.sh
#
# What it does:
# 1. Creates beanflows_service user (nologin) + adds to docker group
# 2. Creates /opt/materia + /data/materia/landing with correct ownership
# 3. Generates ed25519 SSH deploy key for GitLab read access
# 4. Installs age + sops + rclone to /usr/local/bin (as root)
# 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user)
# 6. Prints both public keys + numbered next-step instructions
set -euo pipefail set -euo pipefail
REPO_DIR="/opt/materia" SERVICE_USER="beanflows_service"
AGE_KEY_FILE="$REPO_DIR/age-key.txt" APP_DIR="/opt/materia"
DATA_DIR="/data/materia"
SSH_DIR="/home/${SERVICE_USER}/.ssh"
DEPLOY_KEY="${SSH_DIR}/materia_deploy"
SOPS_AGE_DIR="/home/${SERVICE_USER}/.config/sops/age"
if [ "$EUID" -ne 0 ]; then [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root: sudo bash infra/setup_server.sh"; exit 1; }
echo "ERROR: This script must be run as root"
exit 1 log() { echo "$(date '+%H:%M:%S') ==> $*"; }
# ── Service user ──────────────────────────────────────────────────────────────
log "Creating service user ${SERVICE_USER}..."
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}"
fi
usermod -aG docker "${SERVICE_USER}"
log "User OK."
# ── Directories ───────────────────────────────────────────────────────────────
log "Creating directories..."
mkdir -p "${APP_DIR}" "${DATA_DIR}/landing"
chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}"
chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}"
log "Directories OK."
# ── SSH deploy key ────────────────────────────────────────────────────────────
log "Setting up SSH deploy key..."
sudo -u "${SERVICE_USER}" mkdir -p "${SSH_DIR}"
chmod 700 "${SSH_DIR}"
if [ ! -f "${DEPLOY_KEY}" ]; then
sudo -u "${SERVICE_USER}" ssh-keygen -t ed25519 \
-f "${DEPLOY_KEY}" -N "" -C "materia-deploy"
fi fi
# ── Create data directories ──────────────────────────────── sudo -u "${SERVICE_USER}" bash -c "cat > ${SSH_DIR}/config" <<EOF
echo "--- Creating data directories ---" Host gitlab.com
mkdir -p /data/materia/landing IdentityFile ${DEPLOY_KEY}
mkdir -p "$REPO_DIR" IdentitiesOnly yes
echo "Data dir: /data/materia" EOF
chmod 600 "${SSH_DIR}/config"
sudo -u "${SERVICE_USER}" bash -c \
"ssh-keyscan -H gitlab.com >> ${SSH_DIR}/known_hosts 2>/dev/null; \
sort -u ${SSH_DIR}/known_hosts -o ${SSH_DIR}/known_hosts"
chmod 644 "${SSH_DIR}/known_hosts"
log "SSH deploy key OK."
# ── age ───────────────────────────────────────────────────────────────────────
# ── Install age ────────────────────────────────────────────
echo "--- Installing age ---"
ARCH=$(uname -m) ARCH=$(uname -m)
case "$ARCH" in case "${ARCH}" in
x86_64) ARCH_AGE="amd64" ;; x86_64) ARCH_TAG="amd64" ;;
aarch64) ARCH_AGE="arm64" ;; aarch64) ARCH_TAG="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;; *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;;
esac esac
if ! command -v age-keygen &>/dev/null; then if ! command -v age-keygen >/dev/null 2>&1; then
log "Installing age..."
AGE_VERSION="v1.3.1" AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
rm /tmp/age.tar.gz rm /tmp/age.tar.gz
echo "age installed to /usr/local/bin" log "age installed."
fi fi
# ── Generate age keypair ─────────────────────────────────── # ── sops ──────────────────────────────────────────────────────────────────────
echo "--- Setting up age keypair ---"
if [ -f "$AGE_KEY_FILE" ]; then if ! command -v sops >/dev/null 2>&1; then
echo "Keypair already exists at $AGE_KEY_FILE — skipping generation" log "Installing sops..."
SOPS_VERSION="v3.12.1"
curl -fsSL \
"https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \
-o /usr/local/bin/sops
chmod +x /usr/local/bin/sops
log "sops installed."
fi
# ── rclone ────────────────────────────────────────────────────────────────────
if ! command -v rclone >/dev/null 2>&1; then
log "Installing rclone..."
curl -fsSL https://rclone.org/install.sh | bash
log "rclone installed."
fi
# ── age keypair (as service user, XDG path auto-discovered by SOPS) ───────────
log "Generating age keypair for ${SERVICE_USER}..."
AGE_KEY_FILE="${SOPS_AGE_DIR}/keys.txt"
if [ ! -f "${AGE_KEY_FILE}" ]; then
sudo -u "${SERVICE_USER}" mkdir -p "${SOPS_AGE_DIR}"
sudo -u "${SERVICE_USER}" age-keygen -o "${AGE_KEY_FILE}"
chmod 600 "${AGE_KEY_FILE}"
log "Age keypair generated at ${AGE_KEY_FILE}"
else else
age-keygen -o "$AGE_KEY_FILE" 2>/dev/null log "Age keypair already exists — skipping."
chmod 600 "$AGE_KEY_FILE"
echo "Generated: $AGE_KEY_FILE"
fi fi
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}') # ── Summary ───────────────────────────────────────────────────────────────────
DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub")
AGE_PUB=$(grep "public key:" "${AGE_KEY_FILE}" | awk '{print $NF}')
echo "" echo ""
echo "==================================================================" echo "=================================================================="
echo " Server age public key:"
echo "" echo ""
echo " $AGE_PUB" echo " SSH deploy key (add to GitLab → Settings → Deploy Keys):"
echo "" echo ""
echo " Add this key to .sops.yaml on your workstation:" echo " ${DEPLOY_PUB}"
echo "" echo ""
echo " creation_rules:" echo " Server age public key (add to .sops.yaml):"
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
echo " age: >-"
echo " <dev-key>"
echo " + $AGE_PUB"
echo "" echo ""
echo " Then re-encrypt the prod secrets file:" echo " ${AGE_PUB}"
echo " sops updatekeys .env.prod.sops" echo ""
echo " git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'" echo "=================================================================="
echo " git push" echo ""
echo " Next steps:"
echo ""
echo " 1. Add the SSH deploy key to GitLab:"
echo " → Repository Settings → Deploy Keys → Add key (read-only)"
echo ""
echo " 2. Add the age public key to .sops.yaml on your workstation:"
echo " creation_rules:"
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
echo " age: >-"
echo " <dev-key>"
echo " + ${AGE_PUB}"
echo ""
echo " 3. Re-encrypt prod secrets to include the server key:"
echo " sops updatekeys .env.prod.sops"
echo " git add .sops.yaml .env.prod.sops"
echo " git commit -m 'chore: add server age key'"
echo " git push"
echo ""
echo " 4. Run bootstrap:"
echo " bash infra/bootstrap_supervisor.sh"
echo "" echo ""
echo " Then run infra/bootstrap_supervisor.sh to complete setup."
echo "==================================================================" echo "=================================================================="

3
infra/supervisor/materia-supervisor.service
View File

@@ -5,12 +5,13 @@ Wants=network-online.target
[Service] [Service]
Type=simple Type=simple
User=root User=beanflows_service
WorkingDirectory=/opt/materia WorkingDirectory=/opt/materia
ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py' ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py'
Restart=always Restart=always
RestartSec=10 RestartSec=10
EnvironmentFile=/opt/materia/.env EnvironmentFile=/opt/materia/.env
Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin
Environment=LANDING_DIR=/data/materia/landing Environment=LANDING_DIR=/data/materia/landing
Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb
Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb

51
web/deploy.sh
View File

@@ -1,60 +1,15 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
# ── Ensure sops + age are installed ───────────────────────
APP_DIR="$(cd "$(dirname "$0")" && pwd)" APP_DIR="$(cd "$(dirname "$0")" && pwd)"
BIN_DIR="$APP_DIR/bin"
mkdir -p "$BIN_DIR"
export PATH="$BIN_DIR:$PATH"
ARCH=$(uname -m)
case "$ARCH" in
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
esac
if ! command -v age &>/dev/null; then
echo "==> Installing age to $BIN_DIR..."
AGE_VERSION="v1.3.1"
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
tar -xzf /tmp/age.tar.gz -C "$BIN_DIR" --strip-components=1 age/age age/age-keygen
chmod +x "$BIN_DIR/age" "$BIN_DIR/age-keygen"
rm /tmp/age.tar.gz
fi
# ── Verify sops is installed (setup_server.sh installs it to /usr/local/bin) ──
if ! command -v sops &>/dev/null; then if ! command -v sops &>/dev/null; then
echo "==> Installing sops to $BIN_DIR..." echo "ERROR: sops not found — run infra/setup_server.sh first"
SOPS_VERSION="v3.12.1"
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o "$BIN_DIR/sops"
chmod +x "$BIN_DIR/sops"
fi
# ── Ensure age keypair exists ─────────────────────────────
# Key file lives at repo root (one level up from web/)
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$APP_DIR/../age-key.txt}"
AGE_KEY_FILE="$(realpath "$AGE_KEY_FILE")"
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
if [ ! -f "$AGE_KEY_FILE" ]; then
echo "==> Generating age keypair at $AGE_KEY_FILE..."
age-keygen -o "$AGE_KEY_FILE" 2>&1
chmod 600 "$AGE_KEY_FILE"
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
echo ""
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "!! NEW SERVER — add this public key to .sops.yaml: !!"
echo "!! !!"
echo "!! $AGE_PUB !!"
echo "!! !!"
echo "!! Then run: sops updatekeys .env.prod.sops !!"
echo "!! Commit, push, and re-deploy. !!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo ""
exit 1 exit 1
fi fi
# ── Decrypt secrets ─────────────────────────────────────── # ── Decrypt secrets (SOPS auto-discovers age key from ~/.config/sops/age/) ────
echo "==> Decrypting secrets from .env.prod.sops..." echo "==> Decrypting secrets from .env.prod.sops..."
sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env" sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env"
chmod 600 "$APP_DIR/.env" chmod 600 "$APP_DIR/.env"