beanflows/.gitlab-ci.yml

image: python:3.13

stages:
  - lint
  - test
  - build
  - deploy

variables:
  UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache"

cache:
  paths:
    - .uv-cache/

.uv_setup: &uv_setup
  - curl -LsSf https://astral.sh/uv/install.sh | sh
  - export PATH="$HOME/.cargo/bin:$PATH"
  - source $HOME/.local/bin/env

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_TAG

lint:
  stage: lint
  before_script:
    - *uv_setup
  script:
    - uv sync
    - uv run ruff check .

test:cli:
  stage: test
  before_script:
    - *uv_setup
  script:
    - uv sync
    - uv run pytest tests/ -v --cov=src/materia --cov-report=xml --cov-report=term
  coverage: '/TOTAL.*\s+(\d+%)$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml

test:sqlmesh:
  stage: test
  before_script:
    - *uv_setup
  script:
    - uv sync
    - cd transform/sqlmesh_materia && uv run sqlmesh test

build:extract:
  stage: build
  before_script:
    - *uv_setup
  script:
    - uv sync
    - mkdir -p dist
    - uv build --package psdonline --out-dir dist/extract
    - cd dist/extract && tar -czf ../materia-extract-latest.tar.gz .
  artifacts:
    paths:
      - dist/materia-extract-latest.tar.gz
    expire_in: 1 week
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

build:transform:
  stage: build
  before_script:
    - *uv_setup
  script:
    - uv sync
    - mkdir -p dist
    - uv build --package sqlmesh_materia --out-dir dist/transform
    - cd dist/transform && tar -czf ../materia-transform-latest.tar.gz .
  artifacts:
    paths:
      - dist/materia-transform-latest.tar.gz
    expire_in: 1 week
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

build:cli:
  stage: build
  before_script:
    - *uv_setup
  script:
    - uv sync
    - mkdir -p dist
    - uv build --out-dir dist/cli
    - cd dist/cli && tar -czf ../materia-cli-latest.tar.gz .
  artifacts:
    paths:
      - dist/materia-cli-latest.tar.gz
    expire_in: 1 week
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

deploy:r2:
  stage: deploy
  image: rclone/rclone:latest
  before_script:
    - apk add --no-cache curl unzip
    - curl -fsSL https://get.pulumi.com/esc/install.sh | sh
    - export PATH="$HOME/.pulumi/bin:$PATH"
    - esc login --token ${PULUMI_ACCESS_TOKEN}
    - eval $(esc env open beanflows/prod --format shell)
    - |
      mkdir -p ~/.config/rclone
      cat > ~/.config/rclone/rclone.conf <<EOF
      [r2]
      type = s3
      provider = Cloudflare
      access_key_id = ${R2_ACCESS_KEY_ID}
      secret_access_key = ${R2_SECRET_ACCESS_KEY}
      endpoint = https://${R2_ENDPOINT}
      acl = private
      EOF
  script:
    - rclone copy dist/*.tar.gz r2:${R2_ARTIFACTS_BUCKET}/ -v
  dependencies:
    - build:extract
    - build:transform
    - build:cli
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

deploy:infra:
  stage: deploy
  image: pulumi/pulumi:latest
  before_script:
    - pulumi login --token ${PULUMI_ACCESS_TOKEN}
  script:
    - cd infra
    - pulumi stack select prod
    - pulumi up --yes
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

deploy:supervisor:
  stage: deploy
  image: alpine:latest
  before_script:
    - apk add --no-cache openssh-client curl bash
    - curl -fsSL https://get.pulumi.com/esc/install.sh | sh
    - export PATH="$HOME/.pulumi/bin:$PATH"
    - esc login --token ${PULUMI_ACCESS_TOKEN}
    - eval $(esc env open beanflows/prod --format shell)
  script:
    - |
      # Install pulumi CLI to get stack outputs
      apk add --no-cache pulumi-bin || {
        curl -fsSL https://get.pulumi.com/install.sh | sh
        export PATH="$HOME/.pulumi/bin:$PATH"
      }
      pulumi login --token ${PULUMI_ACCESS_TOKEN}

      # Get supervisor IP from Pulumi
      cd infra
      SUPERVISOR_IP=$(pulumi stack output supervisor_ip -s prod)
      cd ..

      echo "Deploying to supervisor at ${SUPERVISOR_IP}..."

      # Setup SSH
      mkdir -p ~/.ssh
      echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
      chmod 600 ~/.ssh/id_rsa
      ssh-keyscan -H $SUPERVISOR_IP >> ~/.ssh/known_hosts

      # Deploy supervisor script and service
      scp infra/supervisor/supervisor.sh root@${SUPERVISOR_IP}:/opt/materia/supervisor.sh
      scp infra/supervisor/materia-supervisor.service root@${SUPERVISOR_IP}:/etc/systemd/system/materia-supervisor.service

      # Deploy to supervisor
      ssh root@${SUPERVISOR_IP} bash <<'ENDSSH'
        set -e
        cd /opt/materia

        # Create environment file with secrets
        cat > .env <<EOF
      R2_ENDPOINT=${R2_ENDPOINT}
      R2_ARTIFACTS_BUCKET=${R2_ARTIFACTS_BUCKET}
      PULUMI_ACCESS_TOKEN=${PULUMI_ACCESS_TOKEN}
      EOF

        # Download and install CLI
        curl -fsSL -o materia-cli-latest.tar.gz \
          https://${R2_ENDPOINT}/${R2_ARTIFACTS_BUCKET}/materia-cli-latest.tar.gz

        rm -rf cli && mkdir -p cli
        tar -xzf materia-cli-latest.tar.gz -C cli/
        pip3 install --break-system-packages --force-reinstall cli/*.whl

        # Configure Pulumi ESC
        export PATH="$HOME/.pulumi/bin:$PATH"
        esc login --token ${PULUMI_ACCESS_TOKEN}

        # Make supervisor script executable
        chmod +x /opt/materia/supervisor.sh

        # Reload systemd and restart service
        systemctl daemon-reload
        systemctl enable materia-supervisor
        systemctl restart materia-supervisor

        # Show status
        echo "Supervisor service status:"
        systemctl status materia-supervisor --no-pager
      ENDSSH
  dependencies:
    - deploy:r2
    - deploy:infra
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH