120fef369a
- Fix SQLMesh config: Add semicolons to SQL statements in before_all hooks - Resolves "unsupported syntax" warning for CREATE SECRET and ATTACH - DuckDB requires semicolons to terminate statements properly - Fix deploy:infra job: Update Pulumi authentication - Remove `pulumi login --token` (not supported in Docker image) - Use PULUMI_ACCESS_TOKEN environment variable directly - Chain commands with && to avoid "unknown command 'sh'" error - Fix deploy:supervisor job: Update esc login syntax - Change `esc login --token` to `esc login` (--token flag doesn't exist) - esc CLI reads token from PULUMI_ACCESS_TOKEN env var - Simplify Pulumi CLI installation (remove apk fallback logic) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
124 lines
3.3 KiB
YAML
124 lines
3.3 KiB
YAML
image: python:3.13
|
|
|
|
stages:
|
|
- lint
|
|
- test
|
|
- deploy
|
|
|
|
variables:
|
|
UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache"
|
|
|
|
cache:
|
|
paths:
|
|
- .uv-cache/
|
|
|
|
.uv_setup: &uv_setup
|
|
- curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
- export PATH="$HOME/.cargo/bin:$PATH"
|
|
- source $HOME/.local/bin/env
|
|
|
|
workflow:
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
- if: $CI_COMMIT_TAG
|
|
|
|
lint:
|
|
stage: lint
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- uv run ruff check .
|
|
|
|
test:cli:
|
|
stage: test
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- uv run pytest tests/ -v --cov=src/materia --cov-report=xml --cov-report=term
|
|
coverage: '/TOTAL.*\s+(\d+%)$/'
|
|
artifacts:
|
|
reports:
|
|
coverage_report:
|
|
coverage_format: cobertura
|
|
path: coverage.xml
|
|
|
|
test:sqlmesh:
|
|
stage: test
|
|
before_script:
|
|
- *uv_setup
|
|
script:
|
|
- uv sync
|
|
- cd transform/sqlmesh_materia && uv run sqlmesh test
|
|
|
|
deploy:infra:
|
|
stage: deploy
|
|
image: pulumi/pulumi:latest
|
|
before_script:
|
|
- export PULUMI_ACCESS_TOKEN="${PULUMI_ACCESS_TOKEN}"
|
|
script:
|
|
- cd infra && pulumi stack select prod && pulumi up --yes
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
|
|
deploy:supervisor:
|
|
stage: deploy
|
|
image: alpine:latest
|
|
before_script:
|
|
- apk add --no-cache openssh-client curl bash
|
|
- curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
|
- export PATH="$HOME/.pulumi/bin:$PATH"
|
|
- export PULUMI_ACCESS_TOKEN="${PULUMI_ACCESS_TOKEN}"
|
|
- esc login
|
|
- eval $(esc env open beanflows/prod --format shell)
|
|
# Install Pulumi CLI to get stack outputs
|
|
- curl -fsSL https://get.pulumi.com/install.sh | sh
|
|
- export PATH="$HOME/.pulumi/bin:$PATH"
|
|
script:
|
|
- |
|
|
# Get supervisor IP from Pulumi
|
|
cd infra
|
|
SUPERVISOR_IP=$(pulumi stack output supervisor_ip -s prod)
|
|
cd ..
|
|
|
|
# Check if supervisor exists
|
|
if [ -z "$SUPERVISOR_IP" ] || [ "$SUPERVISOR_IP" = "null" ]; then
|
|
echo "No supervisor instance found. Run 'pulumi up' first."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Connecting to supervisor at ${SUPERVISOR_IP}..."
|
|
|
|
# Setup SSH
|
|
mkdir -p ~/.ssh
|
|
echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
|
|
chmod 600 ~/.ssh/id_rsa
|
|
ssh-keyscan -H $SUPERVISOR_IP >> ~/.ssh/known_hosts
|
|
|
|
# Wait for SSH to be ready (new instance may take a moment)
|
|
echo "Waiting for SSH to be ready..."
|
|
for i in $(seq 1 30); do
|
|
if ssh -o ConnectTimeout=5 root@${SUPERVISOR_IP} "echo 'SSH ready'"; then
|
|
break
|
|
fi
|
|
echo "Attempt $i/30 failed, retrying..."
|
|
sleep 10
|
|
done
|
|
|
|
# Check if supervisor is bootstrapped
|
|
if ssh root@${SUPERVISOR_IP} "test -d /opt/materia/.git"; then
|
|
echo "Supervisor already bootstrapped and will auto-update"
|
|
ssh root@${SUPERVISOR_IP} "systemctl status materia-supervisor --no-pager"
|
|
else
|
|
echo "Bootstrapping supervisor for the first time..."
|
|
# Export secrets and run bootstrap
|
|
ssh root@${SUPERVISOR_IP} "export PULUMI_ACCESS_TOKEN='${PULUMI_ACCESS_TOKEN}' GITLAB_READ_TOKEN='${GITLAB_READ_TOKEN}' && bash -s" < infra/bootstrap_supervisor.sh
|
|
echo "Bootstrap complete!"
|
|
fi
|
|
dependencies:
|
|
- deploy:infra
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|