558829f70b
Addresses GitLab PR comments: 1. Remove hardcoded secrets from Pulumi.prod.yaml, use ESC environment 2. Simplify deployment by using git pull instead of R2 artifacts 3. Add bootstrap script for one-time supervisor setup Major changes: - **Pulumi config**: Use ESC environment (beanflows/prod) for all secrets - **Supervisor script**: Git-based deployment (git pull every 15 min) * No more artifact downloads from R2 * Runs code directly via `uv run materia` * Self-updating from master branch - **Bootstrap script**: New infra/bootstrap_supervisor.sh for initial setup * One-time script to clone repo and setup systemd service * Idempotent and simple - **CI/CD simplification**: Remove build and R2 deployment stages * Eliminated build:extract, build:transform, build:cli jobs * Eliminated deploy:r2 job * Simplified deploy:supervisor to just check bootstrap status * Reduced from 4 stages to 3 stages (Lint → Test → Deploy) - **Documentation**: Updated CLAUDE.md with new architecture * Git-based deployment flow * Bootstrap instructions * Simplified execution model Benefits: - ✅ No hardcoded secrets in config files - ✅ Simpler deployment (no artifact builds) - ✅ Easy to test locally (just git clone + uv sync) - ✅ Auto-updates every 15 minutes - ✅ Fewer CI/CD jobs (faster pipelines) - ✅ Cleaner separation of concerns Inspired by TigerBeetle's CFO supervisor pattern. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
119 lines
3.1 KiB
YAML
119 lines
3.1 KiB
YAML
image: python:3.13
|
|
|
|
stages:
|
|
- lint
|
|
- test
|
|
- deploy
|
|
|
|
variables:
|
|
UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache"
|
|
|
|
cache:
|
|
paths:
|
|
- .uv-cache/
|
|
|
|
.uv_setup: &uv_setup
|
|
- curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
- export PATH="$HOME/.cargo/bin:$PATH"
|
|
- source $HOME/.local/bin/env
|
|
|
|
workflow:
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
- if: $CI_COMMIT_TAG
|
|
|
|
lint:
|
|
stage: lint
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- uv run ruff check .
|
|
|
|
test:cli:
|
|
stage: test
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- uv run pytest tests/ -v --cov=src/materia --cov-report=xml --cov-report=term
|
|
coverage: '/TOTAL.*\s+(\d+%)$/'
|
|
artifacts:
|
|
reports:
|
|
coverage_report:
|
|
coverage_format: cobertura
|
|
path: coverage.xml
|
|
|
|
test:sqlmesh:
|
|
stage: test
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- cd transform/sqlmesh_materia && uv run sqlmesh test
|
|
|
|
deploy:infra:
|
|
stage: deploy
|
|
image: pulumi/pulumi:latest
|
|
before_script:
|
|
- pulumi login --token ${PULUMI_ACCESS_TOKEN}
|
|
script:
|
|
- cd infra
|
|
- pulumi stack select prod
|
|
- pulumi up --yes
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
|
|
deploy:supervisor:
|
|
stage: deploy
|
|
image: alpine:latest
|
|
before_script:
|
|
- apk add --no-cache openssh-client curl bash
|
|
- curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
|
- export PATH="$HOME/.pulumi/bin:$PATH"
|
|
- esc login --token ${PULUMI_ACCESS_TOKEN}
|
|
- eval $(esc env open beanflows/prod --format shell)
|
|
# Install Pulumi CLI to get stack outputs
|
|
- |
|
|
apk add --no-cache pulumi-bin || {
|
|
curl -fsSL https://get.pulumi.com/install.sh | sh
|
|
export PATH="$HOME/.pulumi/bin:$PATH"
|
|
}
|
|
- pulumi login --token ${PULUMI_ACCESS_TOKEN}
|
|
script:
|
|
- |
|
|
# Get supervisor IP from Pulumi
|
|
cd infra
|
|
SUPERVISOR_IP=$(pulumi stack output supervisor_ip -s prod)
|
|
cd ..
|
|
|
|
# Check if supervisor exists
|
|
if [ -z "$SUPERVISOR_IP" ] || [ "$SUPERVISOR_IP" = "null" ]; then
|
|
echo "No supervisor instance found. Run 'pulumi up' first."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Deploying to supervisor at ${SUPERVISOR_IP}..."
|
|
|
|
# Setup SSH
|
|
mkdir -p ~/.ssh
|
|
echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
ssh-keyscan -H $SUPERVISOR_IP >> ~/.ssh/known_hosts
|
|
|
|
# Check if supervisor is bootstrapped
|
|
if ssh -o ConnectTimeout=10 root@${SUPERVISOR_IP} "test -d /opt/materia/repo/.git"; then
|
|
echo "Supervisor already bootstrapped, triggering update..."
|
|
# Just signal supervisor to pull latest - it will do so on next check cycle
|
|
ssh root@${SUPERVISOR_IP} "systemctl is-active materia-supervisor || echo 'Service not running, may need bootstrap'"
|
|
else
|
|
echo "Supervisor not bootstrapped yet. Run bootstrap script:"
|
|
echo " export PULUMI_ACCESS_TOKEN=\${PULUMI_ACCESS_TOKEN}"
|
|
echo " ssh root@${SUPERVISOR_IP} 'bash -s' < infra/bootstrap_supervisor.sh"
|
|
fi
|
|
dependencies:
|
|
- deploy:infra
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|