719aa8edd9
- R2 buckets (beanflows-artifacts, beanflows-data-prod) managed manually in Cloudflare UI - R2 API tokens don't work with Cloudflare Pulumi provider - Use cpx11 (€4.49/mo) instead of non-existent ccx11 - Import existing SSH key (deeman@DeemanPC) - Successfully deployed supervisor at 49.13.231.178
117 lines
3.8 KiB
Python
117 lines
3.8 KiB
Python
"""
|
|
BeanFlows.coffee Infrastructure
|
|
Hetzner compute stack for ephemeral worker orchestration
|
|
|
|
Note: R2 buckets are managed manually in Cloudflare dashboard
|
|
- beanflows-artifacts: Stores CLI and pipeline artifacts
|
|
- beanflows-data-prod: Iceberg data lakehouse
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_hcloud as hcloud
|
|
|
|
# Load configuration
|
|
config = pulumi.Config()
|
|
hetzner_location = config.get("hetzner_location") or "nbg1" # Nuremberg datacenter
|
|
|
|
# ============================================================
|
|
# R2 Bucket Names (managed manually in Cloudflare R2 UI)
|
|
# ============================================================
|
|
# R2 buckets cannot be managed via Pulumi as they require R2-specific tokens
|
|
# that don't work with the Cloudflare Pulumi provider.
|
|
# These are defined here for documentation purposes only.
|
|
|
|
ARTIFACTS_BUCKET = "beanflows-artifacts" # CLI + extract/transform packages
|
|
LAKEHOUSE_BUCKET = "beanflows-data-prod" # Iceberg tables (EEUR region)
|
|
|
|
# ============================================================
|
|
# Hetzner Cloud Infrastructure
|
|
# ============================================================
|
|
|
|
# SSH key for server access (imported from existing key)
|
|
ssh_key = hcloud.SshKey(
|
|
"materia-ssh-key",
|
|
name="deeman@DeemanPC",
|
|
public_key=config.require_secret("ssh_public_key"),
|
|
opts=pulumi.ResourceOptions(protect=True),
|
|
)
|
|
|
|
# Small CPX instance for supervisor (runs materia CLI to orchestrate pipelines)
|
|
# This is an always-on instance that creates/destroys ephemeral workers on-demand
|
|
supervisor_server = hcloud.Server(
|
|
"materia-supervisor",
|
|
name="materia-supervisor",
|
|
server_type="cpx11", # 2 vCPU (shared), 2GB RAM, ~€4.49/mo (cheapest option)
|
|
image="ubuntu-24.04",
|
|
location=hetzner_location,
|
|
ssh_keys=[ssh_key.id],
|
|
labels={
|
|
"role": "supervisor",
|
|
"project": "materia",
|
|
},
|
|
user_data="""#!/bin/bash
|
|
set -e
|
|
|
|
# Basic server setup
|
|
apt-get update
|
|
apt-get install -y python3.13 python3-pip curl unzip
|
|
|
|
# Install Pulumi ESC CLI
|
|
curl -fsSL https://get.pulumi.com/esc/install.sh | sh
|
|
export PATH="$HOME/.pulumi/bin:$PATH"
|
|
echo 'export PATH="$HOME/.pulumi/bin:$PATH"' >> /root/.bashrc
|
|
|
|
# Create deployment directory
|
|
mkdir -p /opt/materia
|
|
|
|
# Configure environment
|
|
echo 'Setup complete. Materia CLI will be deployed via CI/CD.' > /opt/materia/README.txt
|
|
""",
|
|
)
|
|
|
|
# Note: Workers are created on-demand by the materia CLI
|
|
# No always-on worker instances in this architecture
|
|
|
|
# Firewall for servers (restrict to SSH + outbound only)
|
|
firewall = hcloud.Firewall(
|
|
"materia-firewall",
|
|
name="materia-firewall",
|
|
rules=[
|
|
# Allow SSH from anywhere (consider restricting to your IP)
|
|
hcloud.FirewallRuleArgs(
|
|
direction="in",
|
|
protocol="tcp",
|
|
port="22",
|
|
source_ips=["0.0.0.0/0", "::/0"],
|
|
),
|
|
# Allow all outbound traffic
|
|
hcloud.FirewallRuleArgs(
|
|
direction="out",
|
|
protocol="tcp",
|
|
port="any",
|
|
destination_ips=["0.0.0.0/0", "::/0"],
|
|
),
|
|
hcloud.FirewallRuleArgs(
|
|
direction="out",
|
|
protocol="udp",
|
|
port="any",
|
|
destination_ips=["0.0.0.0/0", "::/0"],
|
|
),
|
|
],
|
|
)
|
|
|
|
# Apply firewall to supervisor
|
|
supervisor_firewall = hcloud.FirewallAttachment(
|
|
"supervisor-firewall",
|
|
firewall_id=firewall.id,
|
|
server_ids=[supervisor_server.id],
|
|
)
|
|
|
|
# ============================================================
|
|
# Outputs
|
|
# ============================================================
|
|
|
|
pulumi.export("supervisor_ip", supervisor_server.ipv4_address)
|
|
pulumi.export("artifacts_bucket_name", ARTIFACTS_BUCKET)
|
|
pulumi.export("lakehouse_bucket_name", LAKEHOUSE_BUCKET)
|