deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify supervisor architecture and automate bootstrap

Browse Source 
- Simplify supervisor.sh following TigerBeetle pattern
  - Remove complex functions, use simple while loop
  - Add || sleep 600 for resilience against crashes
  - Use git switch --discard-changes for clean updates
  - Run pipelines every hour (SQLMesh handles scheduling)
  - Use POSIX sh instead of bash

- Remove /repo subdirectory nesting
  - Repository clones directly to /opt/materia
  - Simpler paths throughout

- Move systemd service to repo
  - Bootstrap copies from repo instead of hardcoding
  - Service can be updated via git pull

- Automate bootstrap in CI/CD
  - deploy:supervisor now auto-bootstraps on first deploy
  - Waits for SSH to be ready (retry loop)
  - Injects secrets via SSH environment
  - Idempotent: detects if already bootstrapped

Result: Push to master and supervisor "just works"

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Deeman
2025-10-13 21:17:12 +02:00
parent 21f99767bf
commit 2fff895a73
5 changed files with 53 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
infra/bootstrap_supervisor.sh
View File

@@ -26,8 +26,7 @@ if [ "$EUID" -ne 0 ]; then
fi
# Configuration
MATERIA_DIR="/opt/materia"
REPO_DIR="$MATERIA_DIR/repo"
REPO_DIR="/opt/materia"
GITLAB_PROJECT="deemanone/materia"
# GITLAB_READ_TOKEN should be set in Pulumi ESC (beanflows/prod)
@@ -71,22 +70,20 @@ echo "--- Loading secrets from Pulumi ESC ---"
eval $(esc env open beanflows/prod --format shell)
echo "--- Cloning repository ---"
mkdir -p "$MATERIA_DIR"
if [ -d "$REPO_DIR" ]; then
echo "Repository already exists, pulling latest..."
cd "$REPO_DIR"
git pull origin master
else
cd "$MATERIA_DIR"
git clone "$REPO_URL" repo
cd repo
git clone "$REPO_URL" "$REPO_DIR"
cd "$REPO_DIR"
fi
echo "--- Installing Python dependencies ---"
uv sync
echo "--- Creating environment file ---"
cat > "$MATERIA_DIR/.env" <<EOF
cat > "$REPO_DIR/.env" <<EOF
# Environment variables for supervisor
# Loaded from Pulumi ESC: beanflows/prod
PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
@@ -94,32 +91,7 @@ PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
EOF
echo "--- Setting up systemd service ---"
cat > /etc/systemd/system/materia-supervisor.service <<'EOF'
[Unit]
Description=Materia Supervisor - Pipeline Orchestration
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=root
WorkingDirectory=/opt/materia/repo
ExecStart=/opt/materia/repo/infra/supervisor/supervisor.sh
Restart=always
RestartSec=10
EnvironmentFile=/opt/materia/.env
# Resource limits
LimitNOFILE=65536
# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=materia-supervisor
[Install]
WantedBy=multi-user.target
EOF
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service
echo "--- Enabling and starting service ---"
systemctl daemon-reload

9
infra/supervisor/materia-supervisor.service
View File

@@ -7,12 +7,10 @@ Wants=network-online.target
Type=simple
User=root
WorkingDirectory=/opt/materia
Environment="PATH=/usr/local/bin:/usr/bin:/bin:/root/.pulumi/bin"
EnvironmentFile=/opt/materia/.env
# Restart policy
ExecStart=/opt/materia/infra/supervisor/supervisor.sh
Restart=always
RestartSec=10
EnvironmentFile=/opt/materia/.env
# Resource limits
LimitNOFILE=65536
@@ -22,8 +20,5 @@ StandardOutput=journal
StandardError=journal
SyslogIdentifier=materia-supervisor
# Execute supervisor script
ExecStart=/opt/materia/supervisor.sh
[Install]
WantedBy=multi-user.target

168
infra/supervisor/supervisor.sh
View File

@@ -1,152 +1,34 @@
#!/bin/bash
#!/bin/sh
# Materia Supervisor - Continuous pipeline orchestration
# Inspired by TigerBeetle's CFO supervisor pattern
# Inspired by TigerBeetle's CFO supervisor: simple, resilient, easy to understand
# https://github.com/tigerbeetle/tigerbeetle/blob/main/src/scripts/cfo_supervisor.sh
#
# Git-based deployment: pulls latest code from master and runs pipelines via uv
set -euo pipefail
set -eu
# Configuration
readonly CHECK_INTERVAL=900 # 15 minutes
readonly MATERIA_REPO="/opt/materia/repo"
readonly STATE_DIR="/var/lib/materia"
readonly REPO_DIR="/opt/materia"
# Schedules (cron-style times in UTC)
readonly EXTRACT_SCHEDULE_HOUR=2 # 02:00 UTC
readonly TRANSFORM_SCHEDULE_HOUR=3 # 03:00 UTC
# Ensure state directory exists
mkdir -p "$STATE_DIR"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}
log_error() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
}
# Update code from git
update_code() {
log "Checking for code updates..."
cd "$MATERIA_REPO"
# Fetch latest from master
if ! git fetch origin master 2>&1 | grep -v "^From"; then
log_error "Failed to fetch from git"
return 1
fi
# Check if update available
LOCAL=$(git rev-parse HEAD)
REMOTE=$(git rev-parse origin/master)
if [ "$LOCAL" != "$REMOTE" ]; then
log "New version detected: $LOCAL -> $REMOTE"
# Pull latest code
if git pull origin master; then
log "Code updated successfully"
# Update dependencies
log "Updating dependencies with uv sync..."
if uv sync; then
log "Dependencies updated"
return 0
else
log_error "Failed to update dependencies"
return 1
fi
else
log_error "Failed to pull code"
return 1
fi
fi
log "Already up to date at $(git rev-parse --short HEAD)"
return 1 # Return 1 to indicate no update (not an error)
}
# Run pipeline using materia CLI via uv
run_pipeline() {
local pipeline=$1
local date=$(date -u +%Y-%m-%d)
local state_file="$STATE_DIR/${pipeline}_last_run"
log "Running $pipeline pipeline..."
cd "$MATERIA_REPO"
if uv run materia pipeline run "$pipeline"; then
log "$pipeline completed successfully"
echo "$date" > "$state_file"
return 0
else
log_error "$pipeline failed"
return 1
fi
}
# Check if pipeline should run today
should_run_pipeline() {
local pipeline=$1
local schedule_hour=$2
local current_hour=$(date -u +%H)
local current_date=$(date -u +%Y-%m-%d)
local state_file="$STATE_DIR/${pipeline}_last_run"
# Only run at the scheduled hour
if [ "$current_hour" -ne "$schedule_hour" ]; then
return 1
fi
# Check if already ran today
if [ -f "$state_file" ]; then
local last_run=$(cat "$state_file")
if [ "$last_run" = "$current_date" ]; then
return 1 # Already ran today
fi
fi
return 0 # Should run
}
# Main supervisor loop
main() {
log "Materia supervisor starting..."
log "Repository: $MATERIA_REPO"
log "Extract schedule: daily at ${EXTRACT_SCHEDULE_HOUR}:00 UTC"
log "Transform schedule: daily at ${TRANSFORM_SCHEDULE_HOUR}:00 UTC"
log "Check interval: ${CHECK_INTERVAL}s"
# Ensure repo exists
if [ ! -d "$MATERIA_REPO/.git" ]; then
log_error "Repository not found at $MATERIA_REPO"
log_error "Run bootstrap script first!"
exit 1
fi
# Show initial version
cd "$MATERIA_REPO"
log "Starting at commit: $(git rev-parse --short HEAD)"
while true; do
# Check for code updates every loop
update_code || true
# Check extract schedule
if should_run_pipeline "extract" "$EXTRACT_SCHEDULE_HOUR"; then
run_pipeline extract || true
while true
do
(
# Clone repo if missing
if ! [ -d "$REPO_DIR/.git" ]
then
echo "Repository not found, bootstrap required!"
exit 1
fi
# Check transform schedule
if should_run_pipeline "transform" "$TRANSFORM_SCHEDULE_HOUR"; then
run_pipeline transform || true
fi
cd "$REPO_DIR"
sleep "$CHECK_INTERVAL"
done
}
# Update code from git
git fetch origin master
git switch --discard-changes --detach origin/master
uv sync
# Run main loop
main
# Run pipelines (SQLMesh handles scheduling)
uv run materia pipeline run extract
uv run materia pipeline run transform
) || sleep 600 # Sleep 10 min on failure to avoid busy-loop retries
sleep 3600 # Run pipelines every hour
done