c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
117 lines
3.3 KiB
Bash
Executable File
117 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Bootstrap script for Materia supervisor instance
|
|
# Run this once on a new supervisor to set it up
|
|
#
|
|
# Usage:
|
|
# From CI/CD or locally:
|
|
# ssh root@<supervisor_ip> 'bash -s' < infra/bootstrap_supervisor.sh
|
|
#
|
|
# Or on the supervisor itself:
|
|
# curl -fsSL <url-to-this-script> | bash
|
|
|
|
set -euo pipefail
|
|
|
|
echo "=== Materia Supervisor Bootstrap ==="
|
|
echo "This script will:"
|
|
echo " 1. Install dependencies (git, uv, esc)"
|
|
echo " 2. Clone the materia repository"
|
|
echo " 3. Setup systemd service"
|
|
echo " 4. Start the supervisor"
|
|
echo ""
|
|
|
|
# Check if we're root
|
|
if [ "$EUID" -ne 0 ]; then
|
|
echo "ERROR: This script must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Configuration
|
|
REPO_DIR="/opt/materia"
|
|
GITLAB_PROJECT="deemanone/materia"
|
|
|
|
# GITLAB_READ_TOKEN should be set in Pulumi ESC (beanflows/prod)
|
|
if [ -z "${GITLAB_READ_TOKEN:-}" ]; then
|
|
echo "ERROR: GITLAB_READ_TOKEN environment variable not set"
|
|
echo "Please add it to Pulumi ESC (beanflows/prod) first"
|
|
exit 1
|
|
fi
|
|
|
|
REPO_URL="https://gitlab-ci-token:${GITLAB_READ_TOKEN}@gitlab.com/${GITLAB_PROJECT}.git"
|
|
|
|
echo "--- Installing system dependencies ---"
|
|
apt-get update
|
|
apt-get install -y git curl python3-pip
|
|
|
|
echo "--- Installing uv ---"
|
|
if ! command -v uv &> /dev/null; then
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
export PATH="$HOME/.cargo/bin:$PATH"
|
|
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> /root/.bashrc
|
|
fi
|
|
|
|
echo "--- Installing Pulumi ESC ---"
|
|
if ! command -v esc &> /dev/null; then
|
|
curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
|
export PATH="$HOME/.pulumi/bin:$PATH"
|
|
echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
|
|
fi
|
|
|
|
echo "--- Setting up Pulumi ESC authentication ---"
|
|
if [ -z "${PULUMI_ACCESS_TOKEN:-}" ]; then
|
|
echo "ERROR: PULUMI_ACCESS_TOKEN environment variable not set"
|
|
echo "Please set it before running this script:"
|
|
echo " export PULUMI_ACCESS_TOKEN=<your-token>"
|
|
exit 1
|
|
fi
|
|
|
|
esc login --token "$PULUMI_ACCESS_TOKEN"
|
|
|
|
echo "--- Loading secrets from Pulumi ESC ---"
|
|
eval $(esc env open beanflows/prod --format shell)
|
|
|
|
echo "--- Cloning repository ---"
|
|
if [ -d "$REPO_DIR" ]; then
|
|
echo "Repository already exists, pulling latest..."
|
|
cd "$REPO_DIR"
|
|
git pull origin master
|
|
else
|
|
git clone "$REPO_URL" "$REPO_DIR"
|
|
cd "$REPO_DIR"
|
|
fi
|
|
|
|
echo "--- Creating data directories ---"
|
|
mkdir -p /data/materia/landing/psd
|
|
|
|
echo "--- Installing Python dependencies ---"
|
|
uv sync
|
|
|
|
echo "--- Creating environment file ---"
|
|
cat > "$REPO_DIR/.env" <<EOF
|
|
# Environment variables for supervisor
|
|
# Loaded from Pulumi ESC: beanflows/prod
|
|
PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
|
|
PATH=/root/.cargo/bin:/root/.pulumi/bin:/usr/local/bin:/usr/bin:/bin
|
|
LANDING_DIR=/data/materia/landing
|
|
DUCKDB_PATH=/data/materia/lakehouse.duckdb
|
|
EOF
|
|
|
|
echo "--- Setting up systemd service ---"
|
|
cp "$REPO_DIR/infra/supervisor/materia-supervisor.service" /etc/systemd/system/materia-supervisor.service
|
|
|
|
echo "--- Enabling and starting service ---"
|
|
systemctl daemon-reload
|
|
systemctl enable materia-supervisor
|
|
systemctl start materia-supervisor
|
|
|
|
echo ""
|
|
echo "=== Bootstrap complete! ==="
|
|
echo ""
|
|
echo "Supervisor is now running. Check status with:"
|
|
echo " systemctl status materia-supervisor"
|
|
echo ""
|
|
echo "View logs with:"
|
|
echo " journalctl -u materia-supervisor -f"
|
|
echo ""
|
|
echo "Repository location: $REPO_DIR"
|
|
echo "Current commit: $(cd $REPO_DIR && git rev-parse --short HEAD)"
|