update secrets

infra/backup/materia-backup.service

@@ -5,5 +5,5 @@ Wants=network-online.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/rclone sync /data/materia/landing/ r2:materia-raw/landing/ --log-level INFO
+ExecStart=/usr/bin/rclone sync /data/materia/landing/ r2:backup/materia/landing/ --log-level INFO
 TimeoutStartSec=1800

infra/backup/rclone.conf.example

@@ -3,12 +3,14 @@
 #
 # Get credentials from: Cloudflare Dashboard → R2 → Manage R2 API Tokens
 # Or from Pulumi ESC: esc env open beanflows/prod --format shell
+#
+# Bucket: backup  (syncs to backup/materia/landing/)
 
 [r2]
 type = s3
 provider = Cloudflare
 access_key_id = <R2_ACCESS_KEY_ID>
 secret_access_key = <R2_SECRET_ACCESS_KEY>
-endpoint = https://<CLOUDFLARE_ACCOUNT_ID>.r2.cloudflarestorage.com
+endpoint = <R2_ENDPOINT>
 acl = private
 no_check_bucket = true

infra/bootstrap_supervisor.sh

@@ -54,16 +54,66 @@ chmod 600 "${REPO_DIR}/.env"
 
 sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages"
 
-# ── Systemd service ───────────────────────────────────────────────────────────
+# ── Systemd supervisor service ────────────────────────────────────────────────
 
 cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" /etc/systemd/system/
 systemctl daemon-reload
 systemctl enable --now materia-supervisor
 
+# ── R2 backup timer (optional) ────────────────────────────────────────────────
+# Enabled only when R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ENDPOINT
+# are present in .env. Idempotent: rclone.conf is always regenerated from env
+# (so credential rotations take effect on re-run); systemd units are only copied
+# when they differ from what's already installed.
+
+_env_get() { grep -E "^${1}=" "${REPO_DIR}/.env" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"'"'" || true; }
+
+R2_KEY=$(_env_get R2_ACCESS_KEY_ID)
+R2_SECRET=$(_env_get R2_SECRET_ACCESS_KEY)
+R2_ENDPOINT=$(_env_get R2_ENDPOINT)
+
+if [ -n "${R2_KEY}" ] && [ -n "${R2_SECRET}" ] && [ -n "${R2_ENDPOINT}" ]; then
+    echo "$(date '+%H:%M:%S') ==> Configuring R2 backup..."
+
+    RCLONE_CONF_DIR="/home/${SERVICE_USER}/.config/rclone"
+    RCLONE_CONF="${RCLONE_CONF_DIR}/rclone.conf"
+
+    sudo -u "${SERVICE_USER}" mkdir -p "${RCLONE_CONF_DIR}"
+
+    # Always write from env so credential rotations take effect automatically.
+    cat > "${RCLONE_CONF}" <<EOF
+[r2]
+type = s3
+provider = Cloudflare
+access_key_id = ${R2_KEY}
+secret_access_key = ${R2_SECRET}
+endpoint = ${R2_ENDPOINT}
+acl = private
+no_check_bucket = true
+EOF
+    chown "${SERVICE_USER}:${SERVICE_USER}" "${RCLONE_CONF}"
+    chmod 600 "${RCLONE_CONF}"
+
+    UNITS_CHANGED=0
+    for unit in materia-backup.service materia-backup.timer; do
+        if ! diff -q "${REPO_DIR}/infra/backup/${unit}" "/etc/systemd/system/${unit}" >/dev/null 2>&1; then
+            cp "${REPO_DIR}/infra/backup/${unit}" /etc/systemd/system/
+            UNITS_CHANGED=1
+        fi
+    done
+    [ "${UNITS_CHANGED}" = "1" ] && systemctl daemon-reload
+
+    systemctl enable --now materia-backup.timer
+    echo "$(date '+%H:%M:%S') ==> R2 backup timer enabled."
+else
+    echo "$(date '+%H:%M:%S') ==> R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_ENDPOINT not set — skipping backup timer."
+fi
+
 echo ""
 echo "=== Bootstrap complete! ==="
 echo ""
 echo "Check status:    systemctl status materia-supervisor"
 echo "View logs:       journalctl -u materia-supervisor -f"
 echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status"
+echo "Backup timer:    systemctl list-timers materia-backup.timer"
 echo "Tag:             $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)"

infra/readme.md

@@ -24,7 +24,7 @@ Hetzner Server (NVMe)
 1. **Extract** — Supervisor runs due extractors per `infra/supervisor/workflows.toml`
 2. **Transform** — SQLMesh reads landing → writes `lakehouse.duckdb`
 3. **Export** — `export_serving` copies `serving.*` → `analytics.duckdb` (atomic rename)
-4. **Backup** — rclone syncs `/data/materia/landing/` → R2 `materia-raw/landing/`
+4. **Backup** — rclone syncs `/data/materia/landing/` → R2 `backup/materia/landing/`
 5. **Web** — Web app reads `analytics.duckdb` read-only (per-thread connections)
 
 ## Setup (new server)
@@ -59,20 +59,7 @@ ssh root@<server_ip> 'bash -s' < infra/bootstrap_supervisor.sh
 
 This clones the repo via SSH, decrypts secrets, installs Python dependencies, and starts the supervisor service. No access tokens required — access is via the SSH deploy key. (All tools must already be installed by setup_server.sh.)
 
-### 4. Set up R2 backup
-
-```bash
-apt install rclone
-# Configure rclone as the service user (used by the backup timer):
-sudo -u beanflows_service mkdir -p /home/beanflows_service/.config/rclone
-sudo -u beanflows_service cp infra/backup/rclone.conf.example \
-    /home/beanflows_service/.config/rclone/rclone.conf
-# Fill in R2 credentials from .env.prod.sops (ACCESS_KEY_ID, SECRET_ACCESS_KEY, bucket endpoint)
-cp infra/backup/materia-backup.service /etc/systemd/system/
-cp infra/backup/materia-backup.timer /etc/systemd/system/
-systemctl daemon-reload
-systemctl enable --now materia-backup.timer
-```
+If `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, and `R2_ENDPOINT` are present in `.env.prod.sops`, bootstrap also generates `rclone.conf` and enables `materia-backup.timer` automatically. No manual R2 setup step needed.
 
 ## Secrets management