c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
40 lines
1.1 KiB
YAML
40 lines
1.1 KiB
YAML
# --- Gateway Connection ---
|
|
# Single local DuckDB gateway
|
|
# Local dev uses virtual environments (e.g., dev_<username>)
|
|
# Production uses the 'prod' environment
|
|
gateways:
|
|
duckdb:
|
|
connection:
|
|
type: duckdb
|
|
catalogs:
|
|
local: '{{ env_var("DUCKDB_PATH", "local.duckdb") }}'
|
|
|
|
default_gateway: duckdb
|
|
|
|
# --- Variables ---
|
|
variables:
|
|
LANDING_DIR: '{{ env_var("LANDING_DIR", "data/landing") }}'
|
|
|
|
# --- Model Defaults ---
|
|
# https://sqlmesh.readthedocs.io/en/stable/reference/model_configuration/#model-defaults
|
|
|
|
model_defaults:
|
|
dialect: duckdb
|
|
start: 2025-07-07 # Start date for backfill history
|
|
cron: '@daily' # Run models daily at 12am UTC (can override per model)
|
|
|
|
# --- Linting Rules ---
|
|
# https://sqlmesh.readthedocs.io/en/stable/guides/linter/
|
|
|
|
linter:
|
|
enabled: true
|
|
rules:
|
|
- ambiguousorinvalidcolumn
|
|
- invalidselectstarexpansion
|
|
|
|
# --- Default Target Environment ---
|
|
# Prevents accidentally applying plans to prod during local development.
|
|
# https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#default-target-environment
|
|
|
|
default_target_environment: dev_{{ user() }}
|