558829f70b
Addresses GitLab PR comments: 1. Remove hardcoded secrets from Pulumi.prod.yaml, use ESC environment 2. Simplify deployment by using git pull instead of R2 artifacts 3. Add bootstrap script for one-time supervisor setup Major changes: - **Pulumi config**: Use ESC environment (beanflows/prod) for all secrets - **Supervisor script**: Git-based deployment (git pull every 15 min) * No more artifact downloads from R2 * Runs code directly via `uv run materia` * Self-updating from master branch - **Bootstrap script**: New infra/bootstrap_supervisor.sh for initial setup * One-time script to clone repo and setup systemd service * Idempotent and simple - **CI/CD simplification**: Remove build and R2 deployment stages * Eliminated build:extract, build:transform, build:cli jobs * Eliminated deploy:r2 job * Simplified deploy:supervisor to just check bootstrap status * Reduced from 4 stages to 3 stages (Lint → Test → Deploy) - **Documentation**: Updated CLAUDE.md with new architecture * Git-based deployment flow * Bootstrap instructions * Simplified execution model Benefits: - ✅ No hardcoded secrets in config files - ✅ Simpler deployment (no artifact builds) - ✅ Easy to test locally (just git clone + uv sync) - ✅ Auto-updates every 15 minutes - ✅ Fewer CI/CD jobs (faster pipelines) - ✅ Cleaner separation of concerns Inspired by TigerBeetle's CFO supervisor pattern. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
131 lines
3.4 KiB
Bash
Executable File
131 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Bootstrap script for Materia supervisor instance
|
|
# Run this once on a new supervisor to set it up
|
|
#
|
|
# Usage:
|
|
# From CI/CD or locally:
|
|
# ssh root@<supervisor_ip> 'bash -s' < infra/bootstrap_supervisor.sh
|
|
#
|
|
# Or on the supervisor itself:
|
|
# curl -fsSL <url-to-this-script> | bash
|
|
|
|
set -euo pipefail
|
|
|
|
echo "=== Materia Supervisor Bootstrap ==="
|
|
echo "This script will:"
|
|
echo " 1. Install dependencies (git, uv, esc)"
|
|
echo " 2. Clone the materia repository"
|
|
echo " 3. Setup systemd service"
|
|
echo " 4. Start the supervisor"
|
|
echo ""
|
|
|
|
# Check if we're root
|
|
if [ "$EUID" -ne 0 ]; then
|
|
echo "ERROR: This script must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Configuration
|
|
REPO_URL="${REPO_URL:-https://gitlab.com/YOUR_USERNAME/materia.git}" # TODO: Update this!
|
|
MATERIA_DIR="/opt/materia"
|
|
REPO_DIR="$MATERIA_DIR/repo"
|
|
|
|
echo "--- Installing system dependencies ---"
|
|
apt-get update
|
|
apt-get install -y git curl python3-pip
|
|
|
|
echo "--- Installing uv ---"
|
|
if ! command -v uv &> /dev/null; then
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
export PATH="$HOME/.cargo/bin:$PATH"
|
|
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /root/.bashrc
|
|
fi
|
|
|
|
echo "--- Installing Pulumi ESC ---"
|
|
if ! command -v esc &> /dev/null; then
|
|
curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
|
export PATH="$HOME/.pulumi/bin:$PATH"
|
|
echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
|
|
fi
|
|
|
|
echo "--- Setting up Pulumi ESC authentication ---"
|
|
if [ -z "${PULUMI_ACCESS_TOKEN:-}" ]; then
|
|
echo "ERROR: PULUMI_ACCESS_TOKEN environment variable not set"
|
|
echo "Please set it before running this script:"
|
|
echo " export PULUMI_ACCESS_TOKEN=<your-token>"
|
|
exit 1
|
|
fi
|
|
|
|
esc login --token "$PULUMI_ACCESS_TOKEN"
|
|
|
|
echo "--- Loading secrets from Pulumi ESC ---"
|
|
eval $(esc env open beanflows/prod --format shell)
|
|
|
|
echo "--- Cloning repository ---"
|
|
mkdir -p "$MATERIA_DIR"
|
|
if [ -d "$REPO_DIR" ]; then
|
|
echo "Repository already exists, pulling latest..."
|
|
cd "$REPO_DIR"
|
|
git pull origin master
|
|
else
|
|
cd "$MATERIA_DIR"
|
|
git clone "$REPO_URL" repo
|
|
cd repo
|
|
fi
|
|
|
|
echo "--- Installing Python dependencies ---"
|
|
uv sync
|
|
|
|
echo "--- Creating environment file ---"
|
|
cat > "$MATERIA_DIR/.env" <<EOF
|
|
# Environment variables for supervisor
|
|
# Loaded from Pulumi ESC: beanflows/prod
|
|
PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
|
|
PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
|
|
EOF
|
|
|
|
echo "--- Setting up systemd service ---"
|
|
cat > /etc/systemd/system/materia-supervisor.service <<'EOF'
|
|
[Unit]
|
|
Description=Materia Supervisor - Pipeline Orchestration
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=root
|
|
WorkingDirectory=/opt/materia/repo
|
|
ExecStart=/opt/materia/repo/infra/supervisor/supervisor.sh
|
|
Restart=always
|
|
RestartSec=10
|
|
EnvironmentFile=/opt/materia/.env
|
|
|
|
# Resource limits
|
|
LimitNOFILE=65536
|
|
|
|
# Logging
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=materia-supervisor
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
|
|
echo "--- Enabling and starting service ---"
|
|
systemctl daemon-reload
|
|
systemctl enable materia-supervisor
|
|
systemctl start materia-supervisor
|
|
|
|
echo ""
|
|
echo "=== Bootstrap complete! ==="
|
|
echo ""
|
|
echo "Supervisor is now running. Check status with:"
|
|
echo " systemctl status materia-supervisor"
|
|
echo ""
|
|
echo "View logs with:"
|
|
echo " journalctl -u materia-supervisor -f"
|
|
echo ""
|
|
echo "Repository location: $REPO_DIR"
|
|
echo "Current commit: $(cd $REPO_DIR && git rev-parse --short HEAD)"
|