Refactor to local-first architecture on Hetzner NVMe
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
"""Cloud provider abstraction for worker management."""
|
||||
"""Cloud provider for worker management."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -14,35 +13,10 @@ class Instance:
|
||||
type: str
|
||||
|
||||
|
||||
class ProviderModule(Protocol):
|
||||
def create_instance(
|
||||
self: str,
|
||||
instance_type: str,
|
||||
ssh_key: str,
|
||||
location: str | None = None,
|
||||
) -> Instance: ...
|
||||
|
||||
def destroy_instance(self: str) -> None: ...
|
||||
|
||||
def list_instances(self: str | None = None) -> list[Instance]: ...
|
||||
|
||||
def get_instance(self: str) -> Instance | None: ...
|
||||
|
||||
def wait_for_ssh(self: str, timeout: int = 300) -> bool: ...
|
||||
|
||||
|
||||
def get_provider(provider_name: str) -> ProviderModule:
|
||||
def get_provider(provider_name: str):
|
||||
if provider_name == "hetzner":
|
||||
from materia.providers import hetzner
|
||||
|
||||
return hetzner
|
||||
elif provider_name == "ovh":
|
||||
from materia.providers import ovh
|
||||
return ovh
|
||||
elif provider_name == "scaleway":
|
||||
from materia.providers import scaleway
|
||||
return scaleway
|
||||
elif provider_name == "oracle":
|
||||
from materia.providers import oracle
|
||||
return oracle
|
||||
else:
|
||||
raise ValueError(f"Unknown provider: {provider_name}")
|
||||
|
||||
Reference in New Issue
Block a user