Refactor to local-first architecture on Hetzner NVMe

Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.

- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

transform/sqlmesh_materia/config.yaml

@@ -1,5 +1,5 @@
 # --- Gateway Connection ---
-# Single gateway connecting to R2 Iceberg catalog
+# Single local DuckDB gateway
 # Local dev uses virtual environments (e.g., dev_<username>)
 # Production uses the 'prod' environment
 gateways:
@@ -7,48 +7,13 @@ gateways:
     connection:
       type: duckdb
       catalogs:
-        local: 'local.duckdb'
-        cloudflare:
-          type: iceberg
-          path: '{{ env_var("ICEBERG_WAREHOUSE_NAME") }}'
-      connector_config:
-        endpoint: '{{ env_var("ICEBERG_CATALOG_URI") }}'
-      extensions:
-        - name: httpfs
-        - name: iceberg
-      secrets:
-        r2_secret:
-          type: iceberg
-          token: "{{ env_var('R2_ADMIN_API_TOKEN') }}"
-        r2_data_secret:
-          type: r2
-          key_id: "{{ env_var('R2_ADMIN_ACCESS_KEY_ID') }}"
-          secret: "{{ env_var('R2_ADMIN_SECRET_ACCESS_KEY') }}"
-          account_id: "{{ var('CLOUDFLARE_ACCOUNT_ID') }}"
-          region: 'eeur'
-
-
+        local: '{{ env_var("DUCKDB_PATH", "local.duckdb") }}'
 
 default_gateway: duckdb
 
 # --- Variables ---
-# Make environment variables available to models
 variables:
-  R2_BUCKET: beanflows-data-prod
-  CLOUDFLARE_ACCOUNT_ID: "{{ env_var('CLOUDFLARE_ACCOUNT_ID') }}"
-
-  # --- Catalog Configuration ---
-# Attach R2 Iceberg catalog and configure default schema
-# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
-# https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
-
-#before_all:
-#  - "ATTACH '{{ env_var('ICEBERG_WAREHOUSE_NAME') }}' AS catalog (TYPE ICEBERG, ENDPOINT '{{ env_var('ICEBERG_CATALOG_URI') }}', SECRET r2_secret);"
-  # Note: R2 data access is configured via r2_data_secret (TYPE R2)
-  # Models can use r2://bucket/path to read landing data
-  # Note: CREATE SCHEMA has a DuckDB/Iceberg bug (missing Content-Type header)
-  # Schema must be pre-created in R2 Data Catalog via Cloudflare dashboard or API
-  # For now, skip USE statement and rely on fully-qualified table names in models
+  LANDING_DIR: '{{ env_var("LANDING_DIR", "data/landing") }}'
 
 # --- Model Defaults ---
 # https://sqlmesh.readthedocs.io/en/stable/reference/model_configuration/#model-defaults
@@ -59,7 +24,6 @@ model_defaults:
   cron: '@daily'    # Run models daily at 12am UTC (can override per model)
 
 # --- Linting Rules ---
-# Enforce standards for your team
 # https://sqlmesh.readthedocs.io/en/stable/guides/linter/
 
 linter:
@@ -68,22 +32,8 @@ linter:
     - ambiguousorinvalidcolumn
     - invalidselectstarexpansion
 
-# FLOW: Minimal prompts, automatic changes, summary output
-# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#plan
-
-#plan:
-#  no_diff: true             # Hide detailed text differences for changed models
-#  no_prompts: true          # No interactive prompts
-#  auto_apply: true          # Apply changes automatically
-
-# --- Optional: Set a default target environment ---
-# This is intended for local development to prevent users from accidentally applying plans to the prod environment.
-# It is a development only config and should NOT be committed to your git repo.
+# --- Default Target Environment ---
+# Prevents accidentally applying plans to prod during local development.
 # https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#default-target-environment
 
-# Uncomment the following line to use a default target environment derived from the logged in user's name.
 default_target_environment: dev_{{ user() }}
-
-# Example usage:
-# sqlmesh plan            # Automatically resolves to: sqlmesh plan dev_yourname
-# sqlmesh plan prod       # Specify `prod` to apply changes to production

transform/sqlmesh_materia/models/raw/psd_data.sql

@@ -21,4 +21,4 @@ MODEL (
 )
 );
 select *
-FROM read_csv('extract/psdonline/src/psdonline/data/*.csv.gzip', delim=',', encoding='utf-8', compression='gzip', max_line_size=10000000, header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)
+FROM read_csv('{{ var("LANDING_DIR") }}/psd/**/*.csv.gzip', delim=',', encoding='utf-8', compression='gzip', max_line_size=10000000, header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)

transform/sqlmesh_materia/models/staging/stg_psdalldata__commodity.sql

@@ -41,7 +41,7 @@ select
     any_value(unit_name) as unit_name,
     any_value(value) as value, 
     hash(commodity_code, commodity_name, country_code, country_name, market_year, calendar_year, month, attribute_id, attribute_name, unit_id, unit_name, value) as hkey,
-    any_value(make_date(split(filename, '/')[-4]::int, split(filename, '/')[-3]::int, 1)) as ingest_date,
+    any_value(make_date(split(filename, '/')[-3]::int, split(filename, '/')[-2]::int, 1)) as ingest_date,
     any_value(if(month!=0,last_day(make_date(market_year, month, 1)),null)) as market_date_month_end,
   from cast_dtypes
   group by hkey

transform/sqlmesh_materia/pyproject.toml

@@ -2,7 +2,6 @@
 name = "sqlmesh_materia"
 version = "0.1.0"
 description = "Add your description here"
-readme = "README.md"
 authors = [
     { name = "Deeman", email = "hendriknote@gmail.com" }
 ]