cleanup and prefect service setup
This commit is contained in:
Deeman
@@ -1,116 +0,0 @@
|
||||
"""
|
||||
BeanFlows.coffee Infrastructure
|
||||
Hetzner compute stack for ephemeral worker orchestration
|
||||
|
||||
Note: R2 buckets are managed manually in Cloudflare dashboard
|
||||
- beanflows-artifacts: Stores CLI and pipeline artifacts
|
||||
- beanflows-data-prod: Iceberg data lakehouse
|
||||
"""
|
||||
|
||||
import pulumi
|
||||
import pulumi_hcloud as hcloud
|
||||
|
||||
# Load configuration
|
||||
config = pulumi.Config()
|
||||
hetzner_location = config.get("hetzner_location") or "nbg1" # Nuremberg datacenter
|
||||
|
||||
# ============================================================
|
||||
# R2 Bucket Names (managed manually in Cloudflare R2 UI)
|
||||
# ============================================================
|
||||
# R2 buckets cannot be managed via Pulumi as they require R2-specific tokens
|
||||
# that don't work with the Cloudflare Pulumi provider.
|
||||
# These are defined here for documentation purposes only.
|
||||
|
||||
ARTIFACTS_BUCKET = "beanflows-artifacts" # CLI + extract/transform packages
|
||||
LAKEHOUSE_BUCKET = "beanflows-data-prod" # Iceberg tables (EEUR region)
|
||||
|
||||
# ============================================================
|
||||
# Hetzner Cloud Infrastructure
|
||||
# ============================================================
|
||||
|
||||
# SSH key for server access (imported from existing key)
|
||||
ssh_key = hcloud.SshKey(
|
||||
"materia-ssh-key",
|
||||
name="deeman@DeemanPC",
|
||||
public_key=config.require_secret("ssh_public_key"),
|
||||
opts=pulumi.ResourceOptions(protect=True),
|
||||
)
|
||||
|
||||
# Small CPX instance for supervisor (runs materia CLI to orchestrate pipelines)
|
||||
# This is an always-on instance that creates/destroys ephemeral workers on-demand
|
||||
supervisor_server = hcloud.Server(
|
||||
"materia-supervisor",
|
||||
name="materia-supervisor",
|
||||
server_type="cpx11", # 2 vCPU (shared), 2GB RAM, ~€4.49/mo (cheapest option)
|
||||
image="ubuntu-24.04",
|
||||
location=hetzner_location,
|
||||
ssh_keys=[ssh_key.id],
|
||||
labels={
|
||||
"role": "supervisor",
|
||||
"project": "materia",
|
||||
},
|
||||
user_data="""#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Basic server setup
|
||||
apt-get update
|
||||
apt-get install -y python3.13 python3-pip curl unzip
|
||||
|
||||
# Install Pulumi ESC CLI
|
||||
curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
||||
export PATH="$HOME/.pulumi/bin:$PATH"
|
||||
echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
|
||||
|
||||
# Create deployment directory
|
||||
mkdir -p /opt/materia
|
||||
|
||||
# Configure environment
|
||||
echo 'Setup complete. Materia CLI will be deployed via CI/CD.' > /opt/materia/README.txt
|
||||
""",
|
||||
)
|
||||
|
||||
# Note: Workers are created on-demand by the materia CLI
|
||||
# No always-on worker instances in this architecture
|
||||
|
||||
# Firewall for servers (restrict to SSH + outbound only)
|
||||
firewall = hcloud.Firewall(
|
||||
"materia-firewall",
|
||||
name="materia-firewall",
|
||||
rules=[
|
||||
# Allow SSH from anywhere (consider restricting to your IP)
|
||||
hcloud.FirewallRuleArgs(
|
||||
direction="in",
|
||||
protocol="tcp",
|
||||
port="22",
|
||||
source_ips=["0.0.0.0/0", "::/0"],
|
||||
),
|
||||
# Allow all outbound traffic
|
||||
hcloud.FirewallRuleArgs(
|
||||
direction="out",
|
||||
protocol="tcp",
|
||||
port="any",
|
||||
destination_ips=["0.0.0.0/0", "::/0"],
|
||||
),
|
||||
hcloud.FirewallRuleArgs(
|
||||
direction="out",
|
||||
protocol="udp",
|
||||
port="any",
|
||||
destination_ips=["0.0.0.0/0", "::/0"],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Apply firewall to supervisor
|
||||
supervisor_firewall = hcloud.FirewallAttachment(
|
||||
"supervisor-firewall",
|
||||
firewall_id=firewall.id,
|
||||
server_ids=[supervisor_server.id],
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Outputs
|
||||
# ============================================================
|
||||
|
||||
pulumi.export("supervisor_ip", supervisor_server.ipv4_address)
|
||||
pulumi.export("artifacts_bucket_name", ARTIFACTS_BUCKET)
|
||||
pulumi.export("lakehouse_bucket_name", LAKEHOUSE_BUCKET)
|
||||
80
infra/prefect.docker-compose.yml
Normal file
80
infra/prefect.docker-compose.yml
Normal file
@@ -0,0 +1,80 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:14
|
||||
environment:
|
||||
POSTGRES_USER: prefect
|
||||
POSTGRES_PASSWORD: prefect
|
||||
POSTGRES_DB: prefect
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U prefect"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
dragonfly:
|
||||
image: 'docker.dragonflydb.io/dragonflydb/dragonfly'
|
||||
ulimits:
|
||||
memlock: -1
|
||||
volumes:
|
||||
- dragonflydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
prefect-server:
|
||||
image: prefecthq/prefect:3-latest
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
dragonfly:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://prefect:prefect@postgres:5432/prefect
|
||||
PREFECT_SERVER_API_HOST: 0.0.0.0
|
||||
PREFECT_UI_API_URL: http://localhost:4200/api
|
||||
PREFECT_MESSAGING_BROKER: prefect_redis.messaging
|
||||
PREFECT_MESSAGING_CACHE: prefect_redis.messaging
|
||||
PREFECT_REDIS_MESSAGING_HOST: dragonfly
|
||||
PREFECT_REDIS_MESSAGING_PORT: 6379
|
||||
PREFECT_REDIS_MESSAGING_DB: 0
|
||||
command: prefect server start --no-services
|
||||
ports:
|
||||
- "4200:4200"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request as u; u.urlopen('http://localhost:4200/api/health', timeout=1)"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
prefect-services:
|
||||
image: prefecthq/prefect:3-latest
|
||||
depends_on:
|
||||
prefect-server:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://prefect:prefect@postgres:5432/prefect
|
||||
PREFECT_MESSAGING_BROKER: prefect_redis.messaging
|
||||
PREFECT_MESSAGING_CACHE: prefect_redis.messaging
|
||||
PREFECT_REDIS_MESSAGING_HOST: dragonfly
|
||||
PREFECT_REDIS_MESSAGING_PORT: 6379
|
||||
PREFECT_REDIS_MESSAGING_DB: 0
|
||||
command: prefect server services start
|
||||
|
||||
prefect-worker:
|
||||
image: prefecthq/prefect:3-latest
|
||||
depends_on:
|
||||
prefect-server:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PREFECT_API_URL: http://prefect-server:4200/api
|
||||
command: prefect worker start --pool local-pool
|
||||
restart: on-failure
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
dragonflydata:
|
||||
@@ -25,10 +25,8 @@ do
|
||||
uv sync
|
||||
|
||||
# Run pipelines (SQLMesh handles scheduling)
|
||||
uv run materia pipeline run extract
|
||||
uv run materia pipeline run transform
|
||||
#uv run materia pipeline run extract
|
||||
#uv run materia pipeline run transform
|
||||
|
||||
) || sleep 600 # Sleep 10 min on failure to avoid busy-loop retries
|
||||
|
||||
sleep 3600 # Run pipelines every hour
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user