c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
23 lines
412 B
Python
23 lines
412 B
Python
"""Cloud provider for worker management."""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class Instance:
|
|
id: str
|
|
name: str
|
|
ip: str
|
|
status: str
|
|
provider: str
|
|
type: str
|
|
|
|
|
|
def get_provider(provider_name: str):
|
|
if provider_name == "hetzner":
|
|
from materia.providers import hetzner
|
|
|
|
return hetzner
|
|
else:
|
|
raise ValueError(f"Unknown provider: {provider_name}")
|