cleanup and prefect service setup

transform/sqlmesh_materia/config.yaml

@@ -3,9 +3,16 @@
 # Local dev uses virtual environments (e.g., dev_<username>)
 # Production uses the 'prod' environment
 gateways:
-  prod:
+  duckdb:
     connection:
       type: duckdb
+      catalogs:
+        local: 'local.duckdb'
+        cloudflare:
+          type: iceberg
+          path: '{{ env_var("ICEBERG_WAREHOUSE_NAME") }}'
+      connector_config:
+        endpoint: '{{ env_var("ICEBERG_CATALOG_URI") }}'
       extensions:
         - name: httpfs
         - name: iceberg
@@ -13,16 +20,32 @@ gateways:
         r2_secret:
           type: iceberg
           token: "{{ env_var('R2_ADMIN_API_TOKEN') }}"
+        r2_data_secret:
+          type: r2
+          key_id: "{{ env_var('R2_ADMIN_ACCESS_KEY_ID') }}"
+          secret: "{{ env_var('R2_ADMIN_SECRET_ACCESS_KEY') }}"
+          account_id: "{{ var('CLOUDFLARE_ACCOUNT_ID') }}"
+          region: 'eeur'
 
-default_gateway: prod
 
-# --- Catalog Configuration ---
+
+default_gateway: duckdb
+
+# --- Variables ---
+# Make environment variables available to models
+variables:
+  R2_BUCKET: beanflows-data-prod
+  CLOUDFLARE_ACCOUNT_ID: "{{ env_var('CLOUDFLARE_ACCOUNT_ID') }}"
+
+  # --- Catalog Configuration ---
 # Attach R2 Iceberg catalog and configure default schema
 # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
 # https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
 
-before_all:
-  - "ATTACH '{{ env_var('ICEBERG_WAREHOUSE_NAME') }}' AS catalog (TYPE ICEBERG, ENDPOINT '{{ env_var('ICEBERG_CATALOG_URI') }}', SECRET r2_secret);"
+#before_all:
+#  - "ATTACH '{{ env_var('ICEBERG_WAREHOUSE_NAME') }}' AS catalog (TYPE ICEBERG, ENDPOINT '{{ env_var('ICEBERG_CATALOG_URI') }}', SECRET r2_secret);"
+  # Note: R2 data access is configured via r2_data_secret (TYPE R2)
+  # Models can use r2://bucket/path to read landing data
   # Note: CREATE SCHEMA has a DuckDB/Iceberg bug (missing Content-Type header)
   # Schema must be pre-created in R2 Data Catalog via Cloudflare dashboard or API
   # For now, skip USE statement and rely on fully-qualified table names in models
@@ -48,10 +71,10 @@ linter:
 # FLOW: Minimal prompts, automatic changes, summary output
 # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#plan
 
-plan:
-  no_diff: true             # Hide detailed text differences for changed models
-  no_prompts: true          # No interactive prompts
-  auto_apply: true          # Apply changes automatically
+#plan:
+#  no_diff: true             # Hide detailed text differences for changed models
+#  no_prompts: true          # No interactive prompts
+#  auto_apply: true          # Apply changes automatically
 
 # --- Optional: Set a default target environment ---
 # This is intended for local development to prevent users from accidentally applying plans to the prod environment.
@@ -59,7 +82,7 @@ plan:
 # https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#default-target-environment
 
 # Uncomment the following line to use a default target environment derived from the logged in user's name.
-# default_target_environment: dev_{{ user() }}
+default_target_environment: dev_{{ user() }}
 
 # Example usage:
 # sqlmesh plan            # Automatically resolves to: sqlmesh plan dev_yourname

transform/sqlmesh_materia/models/raw/psd_data.sql

@@ -20,5 +20,5 @@ MODEL (
     filename		  varchar
 )
 );
-SELECT *
- FROM read_csv('zip://extract/psdonline/src/psdonline/data/*.zip/*.csv', header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)
+select *
+FROM read_csv('extract/psdonline/src/psdonline/data/*.csv.gzip', delim=',', encoding='utf-8', compression='gzip', max_line_size=10000000, header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)