From 2ad344abf43ed865520015057ef1c7ddbd360c2d Mon Sep 17 00:00:00 2001
From: Deeman <hendriknote@gmail.com>
Date: Mon, 13 Oct 2025 22:04:25 +0200
Subject: [PATCH] Refactor SQLMesh config to use connection-level secrets
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Move Iceberg secret from before_all hook to connection.secrets
  - Fixes SQLMesh warning about unsupported @env_var syntax
  - Uses Jinja templating {{ env_var() }} instead of @env_var()

- Remove database: ':memory:' (incompatible with catalogs)
  - DuckDB doesn't allow both database and catalogs config
  - Connection defaults to in-memory when no database specified

- Simplify before_all hooks to only handle ATTACH and schema setup
  - Secret is now created automatically by SQLMesh
  - Cleaner separation: connection config vs runtime setup

Based on:
- https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
- https://sqlmesh.readthedocs.io/en/latest/integrations/engines/duckdb/

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 transform/sqlmesh_materia/config.yaml | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/transform/sqlmesh_materia/config.yaml b/transform/sqlmesh_materia/config.yaml
index 19a605d..b9fe0de 100644
--- a/transform/sqlmesh_materia/config.yaml
+++ b/transform/sqlmesh_materia/config.yaml
@@ -6,27 +6,25 @@ gateways:
   prod:
     connection:
       type: duckdb
-      database: ':memory:'
       extensions:
         - name: httpfs
         - name: iceberg
+      secrets:
+        r2_secret:
+          type: iceberg
+          token: "{{ env_var('CLOUDFLARE_API_TOKEN') }}"
 
 default_gateway: prod
 
-# --- Hooks ---
-# Run initialization SQL before all plans/runs
+# --- Catalog Configuration ---
+# Configure the Iceberg catalog endpoint
 # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
 
 before_all:
   - |
-    CREATE SECRET IF NOT EXISTS r2_secret (
+    ATTACH '{{ env_var("R2_WAREHOUSE_NAME", "materia") }}' AS catalog (
         TYPE ICEBERG,
-        TOKEN '@env_var("CLOUDFLARE_API_TOKEN")'
-    );
-  - |
-    ATTACH '@env_var("R2_WAREHOUSE_NAME", "materia")' AS catalog (
-        TYPE ICEBERG,
-        ENDPOINT '@env_var("ICEBERG_REST_URI")'
+        ENDPOINT '{{ env_var("ICEBERG_REST_URI") }}'
     );
   - CREATE SCHEMA IF NOT EXISTS catalog.materia;
   - USE catalog.materia;