cleanup and prefect service setup

2026-02-04 22:24:55 +01:00
parent fc27d5f887
commit 6d4377ccf9
41 changed files with 15888 additions and 2591 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,123 +0,0 @@
-image: python:3.13
-
-stages:
-  - lint
-  - test
-  - deploy
-
-variables:
-  UV_CACHE_DIR: "$CI_PROJECT_DIR/.uv-cache"
-
-cache:
-  paths:
-    - .uv-cache/
-
-.uv_setup: &uv_setup
-  - curl -LsSf https://astral.sh/uv/install.sh | sh
-  - export PATH="$HOME/.cargo/bin:$PATH"
-  - source $HOME/.local/bin/env
-
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_TAG
-
-lint:
-  stage: lint
-  before_script:
-    - *uv_setup
-  script:
-    - uv sync
-    - uv run ruff check .
-
-test:cli:
-  stage: test
-  before_script:
-    - *uv_setup
-  script:
-    - uv sync
-    - uv run pytest tests/ -v --cov=src/materia --cov-report=xml --cov-report=term
-  coverage: '/TOTAL.*\s+(\d+%)$/'
-  artifacts:
-    reports:
-      coverage_report:
-        coverage_format: cobertura
-        path: coverage.xml
-
-test:sqlmesh:
-  stage: test
-  before_script:
-    - *uv_setup
-  script:
-    - uv sync
-    - cd transform/sqlmesh_materia && uv run sqlmesh test
-
-deploy:infra:
-  stage: deploy
-  image: pulumi/pulumi:latest
-  before_script:
-    - export PULUMI_ACCESS_TOKEN="${PULUMI_ACCESS_TOKEN}"
-  script:
-    - cd infra && pulumi stack select prod && pulumi up --yes
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-
-deploy:supervisor:
-  stage: deploy
-  image: alpine:latest
-  before_script:
-    - apk add --no-cache openssh-client curl bash
-    - curl -fsSL https://get.pulumi.com/esc/install.sh | sh
-    - export PATH="$HOME/.pulumi/bin:$PATH"
-    - export PULUMI_ACCESS_TOKEN="${PULUMI_ACCESS_TOKEN}"
-    - esc login
-    - eval $(esc env open beanflows/prod --format shell)
-    # Install Pulumi CLI to get stack outputs
-    - curl -fsSL https://get.pulumi.com/install.sh | sh
-    - export PATH="$HOME/.pulumi/bin:$PATH"
-  script:
-    - |
-      # Get supervisor IP from Pulumi
-      cd infra
-      SUPERVISOR_IP=$(pulumi stack output supervisor_ip -s prod)
-      cd ..
-
-      # Check if supervisor exists
-      if [ -z "$SUPERVISOR_IP" ] || [ "$SUPERVISOR_IP" = "null" ]; then
-        echo "No supervisor instance found. Run 'pulumi up' first."
-        exit 1
-      fi
-
-      echo "Connecting to supervisor at ${SUPERVISOR_IP}..."
-
-      # Setup SSH
-      mkdir -p ~/.ssh
-      echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
-      chmod 600 ~/.ssh/id_rsa
-      ssh-keyscan -H $SUPERVISOR_IP >> ~/.ssh/known_hosts
-
-      # Wait for SSH to be ready (new instance may take a moment)
-      echo "Waiting for SSH to be ready..."
-      for i in $(seq 1 30); do
-        if ssh -o ConnectTimeout=5 root@${SUPERVISOR_IP} "echo 'SSH ready'"; then
-          break
-        fi
-        echo "Attempt $i/30 failed, retrying..."
-        sleep 10
-      done
-
-      # Check if supervisor is bootstrapped
-      if ssh root@${SUPERVISOR_IP} "test -d /opt/materia/.git"; then
-        echo "Supervisor already bootstrapped and will auto-update"
-        ssh root@${SUPERVISOR_IP} "systemctl status materia-supervisor --no-pager"
-      else
-        echo "Bootstrapping supervisor for the first time..."
-        # Export secrets and run bootstrap
-        ssh root@${SUPERVISOR_IP} "export PULUMI_ACCESS_TOKEN='${PULUMI_ACCESS_TOKEN}' GITLAB_READ_TOKEN='${GITLAB_READ_TOKEN}' && bash -s" < infra/bootstrap_supervisor.sh
-        echo "Bootstrap complete!"
-      fi
-  dependencies:
-    - deploy:infra
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH