Refactor SQLMesh config to use connection-level secrets

- Move Iceberg secret from before_all hook to connection.secrets
  - Fixes SQLMesh warning about unsupported @env_var syntax
  - Uses Jinja templating {{ env_var() }} instead of @env_var()

- Remove database: ':memory:' (incompatible with catalogs)
  - DuckDB doesn't allow both database and catalogs config
  - Connection defaults to in-memory when no database specified

- Simplify before_all hooks to only handle ATTACH and schema setup
  - Secret is now created automatically by SQLMesh
  - Cleaner separation: connection config vs runtime setup

Based on:
- https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
- https://sqlmesh.readthedocs.io/en/latest/integrations/engines/duckdb/

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Deeman
2025-10-13 22:04:25 +02:00
parent 120fef369a
commit 2ad344abf4

View File

@@ -6,27 +6,25 @@ gateways:
prod: prod:
connection: connection:
type: duckdb type: duckdb
database: ':memory:'
extensions: extensions:
- name: httpfs - name: httpfs
- name: iceberg - name: iceberg
secrets:
r2_secret:
type: iceberg
token: "{{ env_var('CLOUDFLARE_API_TOKEN') }}"
default_gateway: prod default_gateway: prod
# --- Hooks --- # --- Catalog Configuration ---
# Run initialization SQL before all plans/runs # Configure the Iceberg catalog endpoint
# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
before_all: before_all:
- | - |
CREATE SECRET IF NOT EXISTS r2_secret ( ATTACH '{{ env_var("R2_WAREHOUSE_NAME", "materia") }}' AS catalog (
TYPE ICEBERG, TYPE ICEBERG,
TOKEN '@env_var("CLOUDFLARE_API_TOKEN")' ENDPOINT '{{ env_var("ICEBERG_REST_URI") }}'
);
- |
ATTACH '@env_var("R2_WAREHOUSE_NAME", "materia")' AS catalog (
TYPE ICEBERG,
ENDPOINT '@env_var("ICEBERG_REST_URI")'
); );
- CREATE SCHEMA IF NOT EXISTS catalog.materia; - CREATE SCHEMA IF NOT EXISTS catalog.materia;
- USE catalog.materia; - USE catalog.materia;