feat(infra): use beanflows_service for supervisor
- materia-supervisor.service: User=root → User=beanflows_service, add PATH so uv (~/.local/bin) is found without a login shell - setup_server.sh: full rewrite — creates beanflows_service (nologin), generates SSH deploy key + age keypair as service user at XDG path (~/.config/sops/age/keys.txt), installs age/sops/rclone as root, prints both public keys + numbered next-step instructions - bootstrap_supervisor.sh: full rewrite — removes GITLAB_READ_TOKEN requirement, clones via SSH as service user, installs uv as service user, decrypts with SOPS auto-discovery, uv sync as service user, systemctl as root - web/deploy.sh: remove self-contained sops/age install + keypair generation; replace with simple sops check (exit if missing) and SOPS auto-discovery decrypt (no explicit key file needed) - infra/readme.md: update architecture diagram for beanflows_service paths, update setup steps to match new scripts Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -6,129 +6,123 @@
|
||||
# ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - age keypair exists at /opt/materia/age-key.txt
|
||||
# (or SOPS_AGE_KEY_FILE env var pointing elsewhere)
|
||||
# - The server age public key is already in .sops.yaml and .env.prod.sops
|
||||
# (run setup_server.sh first, then add the key and re-commit)
|
||||
# - GITLAB_READ_TOKEN is set (GitLab project access token, read-only)
|
||||
# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair)
|
||||
# - Deploy key added to GitLab (Settings → Repository → Deploy Keys)
|
||||
# - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== Materia Supervisor Bootstrap ==="
|
||||
echo "This script will:"
|
||||
echo " 1. Install dependencies (git, uv, sops, age)"
|
||||
echo " 2. Clone the materia repository"
|
||||
echo " 3. Decrypt secrets from .env.prod.sops"
|
||||
echo " 4. Set up systemd service"
|
||||
echo " 5. Start the supervisor"
|
||||
echo ""
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "ERROR: This script must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────
|
||||
SERVICE_USER="beanflows_service"
|
||||
REPO_DIR="/opt/materia"
|
||||
GITLAB_PROJECT="deemanone/materia"
|
||||
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$REPO_DIR/age-key.txt}"
|
||||
UV="/home/${SERVICE_USER}/.local/bin/uv"
|
||||
|
||||
if [ -z "${GITLAB_READ_TOKEN:-}" ]; then
|
||||
echo "ERROR: GITLAB_READ_TOKEN not set"
|
||||
echo " export GITLAB_READ_TOKEN=<gitlab-project-access-token>"
|
||||
exit 1
|
||||
fi
|
||||
echo "=== Materia Supervisor Bootstrap ==="
|
||||
echo ""
|
||||
|
||||
REPO_URL="https://oauth2:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git"
|
||||
[ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; }
|
||||
|
||||
# ── System dependencies ────────────────────────────────────────────────────────
|
||||
|
||||
# ── System dependencies ────────────────────────────────────
|
||||
echo "--- Installing system dependencies ---"
|
||||
apt-get update -q
|
||||
apt-get install -y -q git curl ca-certificates
|
||||
|
||||
# ── uv ─────────────────────────────────────────────────────
|
||||
# ── uv (installed as service user) ────────────────────────────────────────────
|
||||
|
||||
echo "--- Installing uv ---"
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> /root/.bashrc
|
||||
if [ ! -f "${UV}" ]; then
|
||||
sudo -u "${SERVICE_USER}" bash -c \
|
||||
'curl -LsSf https://astral.sh/uv/install.sh | sh'
|
||||
fi
|
||||
|
||||
# ── sops + age ─────────────────────────────────────────────
|
||||
# ── sops + age (as root, idempotent — setup_server.sh may have done this) ─────
|
||||
|
||||
echo "--- Installing sops + age ---"
|
||||
ARCH=$(uname -m)
|
||||
case "$ARCH" in
|
||||
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
|
||||
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
|
||||
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||
case "${ARCH}" in
|
||||
x86_64) ARCH_TAG="amd64" ;;
|
||||
aarch64) ARCH_TAG="arm64" ;;
|
||||
*) echo "Unsupported architecture: ${ARCH}"; exit 1 ;;
|
||||
esac
|
||||
|
||||
if ! command -v age &>/dev/null; then
|
||||
if ! command -v age >/dev/null 2>&1; then
|
||||
AGE_VERSION="v1.3.1"
|
||||
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${AGE_ARCH}" -o /tmp/age.tar.gz
|
||||
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz
|
||||
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
|
||||
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
|
||||
rm /tmp/age.tar.gz
|
||||
fi
|
||||
|
||||
if ! command -v sops &>/dev/null; then
|
||||
if ! command -v sops >/dev/null 2>&1; then
|
||||
SOPS_VERSION="v3.12.1"
|
||||
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o /usr/local/bin/sops
|
||||
curl -fsSL \
|
||||
"https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \
|
||||
-o /usr/local/bin/sops
|
||||
chmod +x /usr/local/bin/sops
|
||||
fi
|
||||
|
||||
# ── Clone repository ───────────────────────────────────────
|
||||
# ── Clone repository via SSH as service user ──────────────────────────────────
|
||||
|
||||
echo "--- Cloning repository ---"
|
||||
if [ -d "$REPO_DIR/.git" ]; then
|
||||
if [ -d "${REPO_DIR}/.git" ]; then
|
||||
echo "Repository already exists — fetching latest tags..."
|
||||
cd "$REPO_DIR"
|
||||
git fetch --tags --prune-tags origin
|
||||
sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin
|
||||
else
|
||||
git clone "$REPO_URL" "$REPO_DIR"
|
||||
cd "$REPO_DIR"
|
||||
sudo -u "${SERVICE_USER}" git clone \
|
||||
"git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}"
|
||||
fi
|
||||
|
||||
# Checkout latest release tag (same logic as supervisor)
|
||||
LATEST_TAG=$(git tag --list --sort=-version:refname "v*" | head -1)
|
||||
if [ -n "$LATEST_TAG" ]; then
|
||||
echo "Checking out $LATEST_TAG..."
|
||||
git checkout --detach "$LATEST_TAG"
|
||||
LATEST_TAG=$(sudo -u "${SERVICE_USER}" \
|
||||
git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1)
|
||||
if [ -n "${LATEST_TAG}" ]; then
|
||||
echo "Checking out ${LATEST_TAG}..."
|
||||
sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}"
|
||||
else
|
||||
echo "No release tags found — staying on current HEAD"
|
||||
fi
|
||||
|
||||
# ── Check age keypair ──────────────────────────────────────
|
||||
# ── Check age keypair ─────────────────────────────────────────────────────────
|
||||
|
||||
echo "--- Checking age keypair ---"
|
||||
if [ ! -f "$AGE_KEY_FILE" ]; then
|
||||
echo "ERROR: Age keypair not found at $AGE_KEY_FILE"
|
||||
AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt"
|
||||
if [ ! -f "${AGE_KEY_FILE}" ]; then
|
||||
echo ""
|
||||
echo "Run infra/setup_server.sh first to generate the keypair, then:"
|
||||
echo " 1. Copy the public key from setup_server.sh output"
|
||||
echo " 2. Add it to .sops.yaml on your workstation"
|
||||
echo " 3. Run: sops updatekeys .env.prod.sops"
|
||||
echo " 4. Commit + push and re-run this bootstrap"
|
||||
echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}"
|
||||
echo ""
|
||||
echo "Run infra/setup_server.sh first, then:"
|
||||
echo " 1. Add the SSH deploy key to GitLab (Settings → Repository → Deploy Keys)"
|
||||
echo " 2. Add the age public key to .sops.yaml on your workstation"
|
||||
echo " 3. Run: sops updatekeys .env.prod.sops && git push"
|
||||
echo " 4. Re-run this bootstrap"
|
||||
exit 1
|
||||
fi
|
||||
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
|
||||
|
||||
# ── Decrypt secrets ────────────────────────────────────────
|
||||
# ── Decrypt secrets (as service user — SOPS auto-discovers age key from XDG) ──
|
||||
|
||||
echo "--- Decrypting secrets from .env.prod.sops ---"
|
||||
sops --input-type dotenv --output-type dotenv -d "$REPO_DIR/.env.prod.sops" > "$REPO_DIR/.env"
|
||||
chmod 600 "$REPO_DIR/.env"
|
||||
echo "Secrets written to $REPO_DIR/.env"
|
||||
sudo -u "${SERVICE_USER}" bash -c \
|
||||
"sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env"
|
||||
chmod 600 "${REPO_DIR}/.env"
|
||||
echo "Secrets written to ${REPO_DIR}/.env"
|
||||
|
||||
# ── Data directories ───────────────────────────────────────────────────────────
|
||||
|
||||
# ── Data directories ───────────────────────────────────────
|
||||
echo "--- Creating data directories ---"
|
||||
mkdir -p /data/materia/landing
|
||||
chown -R "${SERVICE_USER}:${SERVICE_USER}" /data/materia
|
||||
|
||||
# ── Python dependencies (as service user) ─────────────────────────────────────
|
||||
|
||||
# ── Python dependencies ────────────────────────────────────
|
||||
echo "--- Installing Python dependencies ---"
|
||||
cd "$REPO_DIR"
|
||||
uv sync --all-packages
|
||||
sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages"
|
||||
|
||||
# ── Systemd service (as root) ──────────────────────────────────────────────────
|
||||
|
||||
# ── Systemd service ────────────────────────────────────────
|
||||
echo "--- Setting up systemd service ---"
|
||||
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service
|
||||
cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" \
|
||||
/etc/systemd/system/materia-supervisor.service
|
||||
|
||||
echo "--- Enabling and starting service ---"
|
||||
systemctl daemon-reload
|
||||
@@ -140,7 +134,7 @@ echo "=== Bootstrap complete! ==="
|
||||
echo ""
|
||||
echo "Check status: systemctl status materia-supervisor"
|
||||
echo "View logs: journalctl -u materia-supervisor -f"
|
||||
echo "Workflow status: cd $REPO_DIR && uv run python src/materia/supervisor.py status"
|
||||
echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status"
|
||||
echo ""
|
||||
echo "Repo: $REPO_DIR"
|
||||
echo "Tag: $(cd $REPO_DIR && git describe --tags --always)"
|
||||
echo "Repo: ${REPO_DIR}"
|
||||
echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)"
|
||||
|
||||
@@ -6,8 +6,10 @@ Single-server local-first setup for BeanFlows.coffee on Hetzner NVMe.
|
||||
|
||||
```
|
||||
Hetzner Server (NVMe)
|
||||
├── /opt/materia/ # Git repo (checked out at latest release tag)
|
||||
├── /opt/materia/age-key.txt # Server age keypair (chmod 600, gitignored)
|
||||
├── beanflows_service (system user, nologin)
|
||||
│ ├── ~/.ssh/materia_deploy # ed25519 deploy key for GitLab read access
|
||||
│ └── ~/.config/sops/age/keys.txt # age keypair (auto-discovered by SOPS)
|
||||
├── /opt/materia/ # Git repo (owned by beanflows_service, latest release tag)
|
||||
├── /opt/materia/.env # Decrypted from .env.prod.sops at deploy time
|
||||
├── /data/materia/landing/ # Extracted raw data (immutable, content-addressed)
|
||||
├── /data/materia/lakehouse.duckdb # SQLMesh exclusive write
|
||||
@@ -33,15 +35,16 @@ Hetzner Server (NVMe)
|
||||
bash infra/setup_server.sh
|
||||
```
|
||||
|
||||
This creates data directories, installs age, and generates the server age keypair at `/opt/materia/age-key.txt`. It prints the server's age public key.
|
||||
This creates the `beanflows_service` user, data directories, installs age + sops + rclone, generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys.
|
||||
|
||||
### 2. Add the server key to SOPS
|
||||
|
||||
On your workstation:
|
||||
### 2. Add keys to GitLab and SOPS
|
||||
|
||||
```bash
|
||||
# Add the server public key to .sops.yaml
|
||||
# Then re-encrypt prod secrets to include the server key:
|
||||
# Add the SSH deploy key to GitLab:
|
||||
# → Repository Settings → Deploy Keys → Add key (read-only)
|
||||
|
||||
# Add the server age public key to .sops.yaml on your workstation,
|
||||
# then re-encrypt prod secrets to include the server key:
|
||||
sops updatekeys .env.prod.sops
|
||||
git add .sops.yaml .env.prod.sops
|
||||
git commit -m "chore: add server age key"
|
||||
@@ -51,18 +54,19 @@ git push
|
||||
### 3. Bootstrap the supervisor
|
||||
|
||||
```bash
|
||||
# Requires GITLAB_READ_TOKEN (GitLab project access token, read-only)
|
||||
export GITLAB_READ_TOKEN=<token>
|
||||
ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
|
||||
```
|
||||
|
||||
This installs uv + sops + age, clones the repo, decrypts secrets, installs Python dependencies, and starts the supervisor service.
|
||||
This installs uv (as service user), clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key.
|
||||
|
||||
### 4. Set up R2 backup
|
||||
|
||||
```bash
|
||||
apt install rclone
|
||||
cp infra/backup/rclone.conf.example /root/.config/rclone/rclone.conf
|
||||
# Configure rclone as the service user (used by the backup timer):
|
||||
sudo -u beanflows_service mkdir -p /home/beanflows_service/.config/rclone
|
||||
sudo -u beanflows_service cp infra/backup/rclone.conf.example \
|
||||
/home/beanflows_service/.config/rclone/rclone.conf
|
||||
# Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint)
|
||||
cp infra/backup/materia-backup.service /etc/systemd/system/
|
||||
cp infra/backup/materia-backup.timer /etc/systemd/system/
|
||||
@@ -90,6 +94,7 @@ make secrets-edit-prod
|
||||
|
||||
`bootstrap_supervisor.sh` decrypts `.env.prod.sops` → `/opt/materia/.env` during setup.
|
||||
`web/deploy.sh` re-decrypts on every deploy (so secret rotations take effect automatically).
|
||||
SOPS auto-discovers the service user's age key at `~/.config/sops/age/keys.txt` (XDG default).
|
||||
|
||||
## Deploy model (pull-based)
|
||||
|
||||
@@ -109,7 +114,7 @@ systemctl status materia-supervisor
|
||||
journalctl -u materia-supervisor -f
|
||||
|
||||
# Workflow status table
|
||||
cd /opt/materia && uv run python src/materia/supervisor.py status
|
||||
cd /opt/materia && sudo -u beanflows_service uv run python src/materia/supervisor.py status
|
||||
|
||||
# Backup timer status
|
||||
systemctl list-timers materia-backup.timer
|
||||
|
||||
@@ -1,74 +1,161 @@
|
||||
#!/bin/bash
|
||||
# One-time server setup: create data directories, generate age keypair.
|
||||
# One-time server setup: create service user, SSH deploy key, age keypair.
|
||||
# Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh.
|
||||
#
|
||||
# Usage:
|
||||
# bash infra/setup_server.sh
|
||||
#
|
||||
# What it does:
|
||||
# 1. Creates beanflows_service user (nologin) + adds to docker group
|
||||
# 2. Creates /opt/materia + /data/materia/landing with correct ownership
|
||||
# 3. Generates ed25519 SSH deploy key for GitLab read access
|
||||
# 4. Installs age + sops + rclone to /usr/local/bin (as root)
|
||||
# 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user)
|
||||
# 6. Prints both public keys + numbered next-step instructions
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_DIR="/opt/materia"
|
||||
AGE_KEY_FILE="$REPO_DIR/age-key.txt"
|
||||
SERVICE_USER="beanflows_service"
|
||||
APP_DIR="/opt/materia"
|
||||
DATA_DIR="/data/materia"
|
||||
SSH_DIR="/home/${SERVICE_USER}/.ssh"
|
||||
DEPLOY_KEY="${SSH_DIR}/materia_deploy"
|
||||
SOPS_AGE_DIR="/home/${SERVICE_USER}/.config/sops/age"
|
||||
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "ERROR: This script must be run as root"
|
||||
exit 1
|
||||
[ "$(id -u)" = "0" ] || { echo "ERROR: Run as root: sudo bash infra/setup_server.sh"; exit 1; }
|
||||
|
||||
log() { echo "$(date '+%H:%M:%S') ==> $*"; }
|
||||
|
||||
# ── Service user ──────────────────────────────────────────────────────────────
|
||||
|
||||
log "Creating service user ${SERVICE_USER}..."
|
||||
if ! id "${SERVICE_USER}" >/dev/null 2>&1; then
|
||||
useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}"
|
||||
fi
|
||||
usermod -aG docker "${SERVICE_USER}"
|
||||
log "User OK."
|
||||
|
||||
# ── Directories ───────────────────────────────────────────────────────────────
|
||||
|
||||
log "Creating directories..."
|
||||
mkdir -p "${APP_DIR}" "${DATA_DIR}/landing"
|
||||
chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}"
|
||||
chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}"
|
||||
log "Directories OK."
|
||||
|
||||
# ── SSH deploy key ────────────────────────────────────────────────────────────
|
||||
|
||||
log "Setting up SSH deploy key..."
|
||||
sudo -u "${SERVICE_USER}" mkdir -p "${SSH_DIR}"
|
||||
chmod 700 "${SSH_DIR}"
|
||||
|
||||
if [ ! -f "${DEPLOY_KEY}" ]; then
|
||||
sudo -u "${SERVICE_USER}" ssh-keygen -t ed25519 \
|
||||
-f "${DEPLOY_KEY}" -N "" -C "materia-deploy"
|
||||
fi
|
||||
|
||||
# ── Create data directories ────────────────────────────────
|
||||
echo "--- Creating data directories ---"
|
||||
mkdir -p /data/materia/landing
|
||||
mkdir -p "$REPO_DIR"
|
||||
echo "Data dir: /data/materia"
|
||||
sudo -u "${SERVICE_USER}" bash -c "cat > ${SSH_DIR}/config" <<EOF
|
||||
Host gitlab.com
|
||||
IdentityFile ${DEPLOY_KEY}
|
||||
IdentitiesOnly yes
|
||||
EOF
|
||||
chmod 600 "${SSH_DIR}/config"
|
||||
|
||||
sudo -u "${SERVICE_USER}" bash -c \
|
||||
"ssh-keyscan -H gitlab.com >> ${SSH_DIR}/known_hosts 2>/dev/null; \
|
||||
sort -u ${SSH_DIR}/known_hosts -o ${SSH_DIR}/known_hosts"
|
||||
chmod 644 "${SSH_DIR}/known_hosts"
|
||||
log "SSH deploy key OK."
|
||||
|
||||
# ── age ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Install age ────────────────────────────────────────────
|
||||
echo "--- Installing age ---"
|
||||
ARCH=$(uname -m)
|
||||
case "$ARCH" in
|
||||
x86_64) ARCH_AGE="amd64" ;;
|
||||
aarch64) ARCH_AGE="arm64" ;;
|
||||
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||
case "${ARCH}" in
|
||||
x86_64) ARCH_TAG="amd64" ;;
|
||||
aarch64) ARCH_TAG="arm64" ;;
|
||||
*) echo "Unsupported architecture: ${ARCH}"; exit 1 ;;
|
||||
esac
|
||||
|
||||
if ! command -v age-keygen &>/dev/null; then
|
||||
if ! command -v age-keygen >/dev/null 2>&1; then
|
||||
log "Installing age..."
|
||||
AGE_VERSION="v1.3.1"
|
||||
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
|
||||
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz
|
||||
tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen
|
||||
chmod +x /usr/local/bin/age /usr/local/bin/age-keygen
|
||||
rm /tmp/age.tar.gz
|
||||
echo "age installed to /usr/local/bin"
|
||||
log "age installed."
|
||||
fi
|
||||
|
||||
# ── Generate age keypair ───────────────────────────────────
|
||||
echo "--- Setting up age keypair ---"
|
||||
if [ -f "$AGE_KEY_FILE" ]; then
|
||||
echo "Keypair already exists at $AGE_KEY_FILE — skipping generation"
|
||||
# ── sops ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
if ! command -v sops >/dev/null 2>&1; then
|
||||
log "Installing sops..."
|
||||
SOPS_VERSION="v3.12.1"
|
||||
curl -fsSL \
|
||||
"https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \
|
||||
-o /usr/local/bin/sops
|
||||
chmod +x /usr/local/bin/sops
|
||||
log "sops installed."
|
||||
fi
|
||||
|
||||
# ── rclone ────────────────────────────────────────────────────────────────────
|
||||
|
||||
if ! command -v rclone >/dev/null 2>&1; then
|
||||
log "Installing rclone..."
|
||||
curl -fsSL https://rclone.org/install.sh | bash
|
||||
log "rclone installed."
|
||||
fi
|
||||
|
||||
# ── age keypair (as service user, XDG path auto-discovered by SOPS) ───────────
|
||||
|
||||
log "Generating age keypair for ${SERVICE_USER}..."
|
||||
AGE_KEY_FILE="${SOPS_AGE_DIR}/keys.txt"
|
||||
if [ ! -f "${AGE_KEY_FILE}" ]; then
|
||||
sudo -u "${SERVICE_USER}" mkdir -p "${SOPS_AGE_DIR}"
|
||||
sudo -u "${SERVICE_USER}" age-keygen -o "${AGE_KEY_FILE}"
|
||||
chmod 600 "${AGE_KEY_FILE}"
|
||||
log "Age keypair generated at ${AGE_KEY_FILE}"
|
||||
else
|
||||
age-keygen -o "$AGE_KEY_FILE" 2>/dev/null
|
||||
chmod 600 "$AGE_KEY_FILE"
|
||||
echo "Generated: $AGE_KEY_FILE"
|
||||
log "Age keypair already exists — skipping."
|
||||
fi
|
||||
|
||||
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
|
||||
DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub")
|
||||
AGE_PUB=$(grep "public key:" "${AGE_KEY_FILE}" | awk '{print $NF}')
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Server age public key:"
|
||||
echo ""
|
||||
echo " $AGE_PUB"
|
||||
echo " SSH deploy key (add to GitLab → Settings → Deploy Keys):"
|
||||
echo ""
|
||||
echo " Add this key to .sops.yaml on your workstation:"
|
||||
echo " ${DEPLOY_PUB}"
|
||||
echo ""
|
||||
echo " Server age public key (add to .sops.yaml):"
|
||||
echo ""
|
||||
echo " ${AGE_PUB}"
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo ""
|
||||
echo " Next steps:"
|
||||
echo ""
|
||||
echo " 1. Add the SSH deploy key to GitLab:"
|
||||
echo " → Repository Settings → Deploy Keys → Add key (read-only)"
|
||||
echo ""
|
||||
echo " 2. Add the age public key to .sops.yaml on your workstation:"
|
||||
echo " creation_rules:"
|
||||
echo " - path_regex: \\.env\\.(dev|prod)\\.sops\$"
|
||||
echo " age: >-"
|
||||
echo " <dev-key>"
|
||||
echo " + $AGE_PUB"
|
||||
echo " + ${AGE_PUB}"
|
||||
echo ""
|
||||
echo " Then re-encrypt the prod secrets file:"
|
||||
echo " 3. Re-encrypt prod secrets to include the server key:"
|
||||
echo " sops updatekeys .env.prod.sops"
|
||||
echo " git add .sops.yaml .env.prod.sops && git commit -m 'chore: add server age key'"
|
||||
echo " git add .sops.yaml .env.prod.sops"
|
||||
echo " git commit -m 'chore: add server age key'"
|
||||
echo " git push"
|
||||
echo ""
|
||||
echo " Then run infra/bootstrap_supervisor.sh to complete setup."
|
||||
echo " 4. Run bootstrap:"
|
||||
echo " bash infra/bootstrap_supervisor.sh"
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
|
||||
@@ -5,12 +5,13 @@ Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
User=beanflows_service
|
||||
WorkingDirectory=/opt/materia
|
||||
ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py'
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
EnvironmentFile=/opt/materia/.env
|
||||
Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
Environment=LANDING_DIR=/data/materia/landing
|
||||
Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb
|
||||
Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb
|
||||
|
||||
@@ -1,60 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ── Ensure sops + age are installed ───────────────────────
|
||||
APP_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
BIN_DIR="$APP_DIR/bin"
|
||||
mkdir -p "$BIN_DIR"
|
||||
export PATH="$BIN_DIR:$PATH"
|
||||
|
||||
ARCH=$(uname -m)
|
||||
case "$ARCH" in
|
||||
x86_64) ARCH_SOPS="amd64"; ARCH_AGE="amd64" ;;
|
||||
aarch64) ARCH_SOPS="arm64"; ARCH_AGE="arm64" ;;
|
||||
*) echo "Unsupported architecture: $ARCH"; exit 1 ;;
|
||||
esac
|
||||
|
||||
if ! command -v age &>/dev/null; then
|
||||
echo "==> Installing age to $BIN_DIR..."
|
||||
AGE_VERSION="v1.3.1"
|
||||
curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_AGE}" -o /tmp/age.tar.gz
|
||||
tar -xzf /tmp/age.tar.gz -C "$BIN_DIR" --strip-components=1 age/age age/age-keygen
|
||||
chmod +x "$BIN_DIR/age" "$BIN_DIR/age-keygen"
|
||||
rm /tmp/age.tar.gz
|
||||
fi
|
||||
|
||||
# ── Verify sops is installed (setup_server.sh installs it to /usr/local/bin) ──
|
||||
if ! command -v sops &>/dev/null; then
|
||||
echo "==> Installing sops to $BIN_DIR..."
|
||||
SOPS_VERSION="v3.12.1"
|
||||
curl -fsSL "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_SOPS}" -o "$BIN_DIR/sops"
|
||||
chmod +x "$BIN_DIR/sops"
|
||||
fi
|
||||
|
||||
# ── Ensure age keypair exists ─────────────────────────────
|
||||
# Key file lives at repo root (one level up from web/)
|
||||
AGE_KEY_FILE="${SOPS_AGE_KEY_FILE:-$APP_DIR/../age-key.txt}"
|
||||
AGE_KEY_FILE="$(realpath "$AGE_KEY_FILE")"
|
||||
export SOPS_AGE_KEY_FILE="$AGE_KEY_FILE"
|
||||
|
||||
if [ ! -f "$AGE_KEY_FILE" ]; then
|
||||
echo "==> Generating age keypair at $AGE_KEY_FILE..."
|
||||
age-keygen -o "$AGE_KEY_FILE" 2>&1
|
||||
chmod 600 "$AGE_KEY_FILE"
|
||||
AGE_PUB=$(grep "public key:" "$AGE_KEY_FILE" | awk '{print $NF}')
|
||||
echo ""
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo "!! NEW SERVER — add this public key to .sops.yaml: !!"
|
||||
echo "!! !!"
|
||||
echo "!! $AGE_PUB !!"
|
||||
echo "!! !!"
|
||||
echo "!! Then run: sops updatekeys .env.prod.sops !!"
|
||||
echo "!! Commit, push, and re-deploy. !!"
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo ""
|
||||
echo "ERROR: sops not found — run infra/setup_server.sh first"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Decrypt secrets ───────────────────────────────────────
|
||||
# ── Decrypt secrets (SOPS auto-discovers age key from ~/.config/sops/age/) ────
|
||||
echo "==> Decrypting secrets from .env.prod.sops..."
|
||||
sops --input-type dotenv --output-type dotenv -d "$APP_DIR/../.env.prod.sops" > "$APP_DIR/.env"
|
||||
chmod 600 "$APP_DIR/.env"
|
||||
|
||||
Reference in New Issue
Block a user