From cf65fa16b6084ebd3cdbf13c4258d40e159d7e73 Mon Sep 17 00:00:00 2001 From: Deeman Date: Thu, 26 Feb 2026 22:25:31 +0100 Subject: [PATCH] refactor(infra): consolidate tool installs in setup, strip bootstrap to essentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setup_server.sh: add git/curl/ca-certificates apt install, add uv install as service user, fix SSH config write (root + chown vs sudo heredoc), remove noise log lines after set -e makes them redundant - bootstrap_supervisor.sh: remove all tool installs (apt, uv, sops, age) — setup_server.sh is now the single source of truth; strip to ~45 lines: age-key check, clone/fetch, tag checkout, decrypt, uv sync, systemd enable - readme.md: update step 1 and step 3 descriptions Co-Authored-By: Claude Sonnet 4.6 --- infra/bootstrap_supervisor.sh | 105 ++++++---------------------------- infra/readme.md | 4 +- infra/setup_server.sh | 31 ++++++---- 3 files changed, 40 insertions(+), 100 deletions(-) diff --git a/infra/bootstrap_supervisor.sh b/infra/bootstrap_supervisor.sh index a966a9d..4e372ae 100755 --- a/infra/bootstrap_supervisor.sh +++ b/infra/bootstrap_supervisor.sh @@ -1,12 +1,13 @@ #!/bin/bash -# Bootstrap script for Materia supervisor instance. -# Run once on a fresh server after setup_server.sh. +# Bootstrap Materia supervisor after setup_server.sh + adding keys. +# Run once on the server after SSH deploy key is added to GitLab +# and the server age key is committed to .env.prod.sops. # # Usage: # ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh # # Prerequisites: -# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair) +# - setup_server.sh already run (beanflows_service user, SSH deploy key, age keypair, uv) # - Deploy key added to GitLab (Settings → Repository → Deploy Keys) # - Server age public key added to .sops.yaml + .env.prod.sops committed + pushed @@ -17,117 +18,47 @@ REPO_DIR="/opt/materia" GITLAB_PROJECT="deemanone/materia" UV="/home/${SERVICE_USER}/.local/bin/uv" -echo "=== Materia Supervisor Bootstrap ===" -echo "" - [ "$(id -u)" = "0" ] || { echo "ERROR: Run as root"; exit 1; } -# ── System dependencies ──────────────────────────────────────────────────────── +# ── Check age keypair ───────────────────────────────────────────────────────── -echo "--- Installing system dependencies ---" -apt-get update -q -apt-get install -y -q git curl ca-certificates - -# ── uv (installed as service user) ──────────────────────────────────────────── - -echo "--- Installing uv ---" -if [ ! -f "${UV}" ]; then - sudo -u "${SERVICE_USER}" bash -c \ - 'curl -LsSf https://astral.sh/uv/install.sh | sh' +AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt" +if [ ! -f "${AGE_KEY_FILE}" ]; then + echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}" + echo "Run infra/setup_server.sh first, then add the printed keys, then re-run." + exit 1 fi -# ── sops + age (as root, idempotent — setup_server.sh may have done this) ───── +# ── Clone or update repository ──────────────────────────────────────────────── -echo "--- Installing sops + age ---" -ARCH=$(uname -m) -case "${ARCH}" in - x86_64) ARCH_TAG="amd64" ;; - aarch64) ARCH_TAG="arm64" ;; - *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; -esac - -if ! command -v age >/dev/null 2>&1; then - AGE_VERSION="v1.3.1" - curl -fsSL "https://dl.filippo.io/age/${AGE_VERSION}?for=linux/${ARCH_TAG}" -o /tmp/age.tar.gz - tar -xzf /tmp/age.tar.gz -C /usr/local/bin --strip-components=1 age/age age/age-keygen - chmod +x /usr/local/bin/age /usr/local/bin/age-keygen - rm /tmp/age.tar.gz -fi - -if ! command -v sops >/dev/null 2>&1; then - SOPS_VERSION="v3.12.1" - curl -fsSL \ - "https://github.com/getsops/sops/releases/download/${SOPS_VERSION}/sops-${SOPS_VERSION}.linux.${ARCH_TAG}" \ - -o /usr/local/bin/sops - chmod +x /usr/local/bin/sops -fi - -# ── Clone repository via SSH as service user ────────────────────────────────── - -echo "--- Cloning repository ---" if [ -d "${REPO_DIR}/.git" ]; then - echo "Repository already exists — fetching latest tags..." sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" fetch --tags --prune-tags origin else sudo -u "${SERVICE_USER}" git clone \ "git@gitlab.com:${GITLAB_PROJECT}.git" "${REPO_DIR}" fi -# Checkout latest release tag (same logic as supervisor) LATEST_TAG=$(sudo -u "${SERVICE_USER}" \ git -C "${REPO_DIR}" tag --list --sort=-version:refname "v*" | head -1) if [ -n "${LATEST_TAG}" ]; then - echo "Checking out ${LATEST_TAG}..." sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" checkout --detach "${LATEST_TAG}" -else - echo "No release tags found — staying on current HEAD" fi -# ── Check age keypair ───────────────────────────────────────────────────────── +# ── Decrypt secrets ─────────────────────────────────────────────────────────── -echo "--- Checking age keypair ---" -AGE_KEY_FILE="/home/${SERVICE_USER}/.config/sops/age/keys.txt" -if [ ! -f "${AGE_KEY_FILE}" ]; then - echo "" - echo "ERROR: Age keypair not found at ${AGE_KEY_FILE}" - echo "" - echo "Run infra/setup_server.sh first, then:" - echo " 1. Add the SSH deploy key to GitLab (Settings → Repository → Deploy Keys)" - echo " 2. Add the age public key to .sops.yaml on your workstation" - echo " 3. Run: sops updatekeys .env.prod.sops && git push" - echo " 4. Re-run this bootstrap" - exit 1 -fi - -# ── Decrypt secrets (as service user — SOPS auto-discovers age key from XDG) ── - -echo "--- Decrypting secrets from .env.prod.sops ---" sudo -u "${SERVICE_USER}" bash -c \ "sops --input-type dotenv --output-type dotenv -d ${REPO_DIR}/.env.prod.sops > ${REPO_DIR}/.env" chmod 600 "${REPO_DIR}/.env" -echo "Secrets written to ${REPO_DIR}/.env" -# ── Data directories ─────────────────────────────────────────────────────────── +# ── Python dependencies ─────────────────────────────────────────────────────── -echo "--- Creating data directories ---" -mkdir -p /data/materia/landing -chown -R "${SERVICE_USER}:${SERVICE_USER}" /data/materia - -# ── Python dependencies (as service user) ───────────────────────────────────── - -echo "--- Installing Python dependencies ---" sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages" -# ── Systemd service (as root) ────────────────────────────────────────────────── +# ── Systemd service ─────────────────────────────────────────────────────────── -echo "--- Setting up systemd service ---" -cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" \ - /etc/systemd/system/materia-supervisor.service - -echo "--- Enabling and starting service ---" +cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" /etc/systemd/system/ systemctl daemon-reload -systemctl enable materia-supervisor -systemctl restart materia-supervisor +systemctl enable --now materia-supervisor echo "" echo "=== Bootstrap complete! ===" @@ -135,6 +66,4 @@ echo "" echo "Check status: systemctl status materia-supervisor" echo "View logs: journalctl -u materia-supervisor -f" echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status" -echo "" -echo "Repo: ${REPO_DIR}" -echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" +echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" diff --git a/infra/readme.md b/infra/readme.md index 82881b7..ff91664 100644 --- a/infra/readme.md +++ b/infra/readme.md @@ -35,7 +35,7 @@ Hetzner Server (NVMe) bash infra/setup_server.sh ``` -This creates the `beanflows_service` user, data directories, installs age + sops + rclone, generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys. +This creates the `beanflows_service` user, data directories, installs all tools (git, curl, age, sops, rclone, uv), generates an ed25519 SSH deploy key and an age keypair (both as the service user). It prints both public keys. ### 2. Add keys to GitLab and SOPS @@ -57,7 +57,7 @@ git push ssh root@ 'bash -s' < infra/bootstrap_supervisor.sh ``` -This installs uv (as service user), clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. +This clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. (All tools must already be installed by setup_server.sh.) ### 4. Set up R2 backup diff --git a/infra/setup_server.sh b/infra/setup_server.sh index b8f46f1..31e141e 100644 --- a/infra/setup_server.sh +++ b/infra/setup_server.sh @@ -1,5 +1,5 @@ #!/bin/bash -# One-time server setup: create service user, SSH deploy key, age keypair. +# One-time server setup: create service user, install tools, SSH deploy key, age keypair. # Run as root on a fresh Hetzner server before running bootstrap_supervisor.sh. # # Usage: @@ -8,8 +8,8 @@ # What it does: # 1. Creates beanflows_service user (nologin) + adds to docker group # 2. Creates /opt/materia + /data/materia/landing with correct ownership -# 3. Generates ed25519 SSH deploy key for GitLab read access -# 4. Installs age + sops + rclone to /usr/local/bin (as root) +# 3. Installs git, curl, age, sops, rclone, uv +# 4. Generates ed25519 SSH deploy key for GitLab read access # 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user) # 6. Prints both public keys + numbered next-step instructions @@ -33,7 +33,6 @@ if ! id "${SERVICE_USER}" >/dev/null 2>&1; then useradd --system --create-home --shell /usr/sbin/nologin "${SERVICE_USER}" fi usermod -aG docker "${SERVICE_USER}" -log "User OK." # ── Directories ─────────────────────────────────────────────────────────────── @@ -41,7 +40,12 @@ log "Creating directories..." mkdir -p "${APP_DIR}" "${DATA_DIR}/landing" chown "${SERVICE_USER}:${SERVICE_USER}" "${APP_DIR}" chown -R "${SERVICE_USER}:${SERVICE_USER}" "${DATA_DIR}" -log "Directories OK." + +# ── System tools ────────────────────────────────────────────────────────────── + +log "Installing system tools..." +apt-get update -q +apt-get install -y -q git curl ca-certificates # ── SSH deploy key ──────────────────────────────────────────────────────────── @@ -54,18 +58,18 @@ if [ ! -f "${DEPLOY_KEY}" ]; then -f "${DEPLOY_KEY}" -N "" -C "materia-deploy" fi -sudo -u "${SERVICE_USER}" bash -c "cat > ${SSH_DIR}/config" < "${SSH_DIR}/config" <> ${SSH_DIR}/known_hosts 2>/dev/null; \ - sort -u ${SSH_DIR}/known_hosts -o ${SSH_DIR}/known_hosts" +ssh-keyscan -H gitlab.com >> "${SSH_DIR}/known_hosts" 2>/dev/null +sort -u "${SSH_DIR}/known_hosts" -o "${SSH_DIR}/known_hosts" +chown "${SERVICE_USER}:${SERVICE_USER}" "${SSH_DIR}/known_hosts" chmod 644 "${SSH_DIR}/known_hosts" -log "SSH deploy key OK." # ── age ─────────────────────────────────────────────────────────────────────── @@ -119,6 +123,13 @@ else log "Age keypair already exists — skipping." fi +# ── uv (installed as service user) ──────────────────────────────────────────── + +if [ ! -f "/home/${SERVICE_USER}/.local/bin/uv" ]; then + log "Installing uv..." + sudo -u "${SERVICE_USER}" bash -c 'curl -LsSf https://astral.sh/uv/install.sh | sh' +fi + # ── Summary ─────────────────────────────────────────────────────────────────── DEPLOY_PUB=$(cat "${DEPLOY_KEY}.pub")