Refactor SQLMesh config to use connection-level secrets
- Move Iceberg secret from before_all hook to connection.secrets
- Fixes SQLMesh warning about unsupported @env_var syntax
- Uses Jinja templating {{ env_var() }} instead of @env_var()
- Remove database: ':memory:' (incompatible with catalogs)
- DuckDB doesn't allow both database and catalogs config
- Connection defaults to in-memory when no database specified
- Simplify before_all hooks to only handle ATTACH and schema setup
- Secret is now created automatically by SQLMesh
- Cleaner separation: connection config vs runtime setup
Based on:
- https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
- https://sqlmesh.readthedocs.io/en/latest/integrations/engines/duckdb/
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -6,27 +6,25 @@ gateways:
|
|||||||
prod:
|
prod:
|
||||||
connection:
|
connection:
|
||||||
type: duckdb
|
type: duckdb
|
||||||
database: ':memory:'
|
|
||||||
extensions:
|
extensions:
|
||||||
- name: httpfs
|
- name: httpfs
|
||||||
- name: iceberg
|
- name: iceberg
|
||||||
|
secrets:
|
||||||
|
r2_secret:
|
||||||
|
type: iceberg
|
||||||
|
token: "{{ env_var('CLOUDFLARE_API_TOKEN') }}"
|
||||||
|
|
||||||
default_gateway: prod
|
default_gateway: prod
|
||||||
|
|
||||||
# --- Hooks ---
|
# --- Catalog Configuration ---
|
||||||
# Run initialization SQL before all plans/runs
|
# Configure the Iceberg catalog endpoint
|
||||||
# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
|
# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
|
||||||
|
|
||||||
before_all:
|
before_all:
|
||||||
- |
|
- |
|
||||||
CREATE SECRET IF NOT EXISTS r2_secret (
|
ATTACH '{{ env_var("R2_WAREHOUSE_NAME", "materia") }}' AS catalog (
|
||||||
TYPE ICEBERG,
|
TYPE ICEBERG,
|
||||||
TOKEN '@env_var("CLOUDFLARE_API_TOKEN")'
|
ENDPOINT '{{ env_var("ICEBERG_REST_URI") }}'
|
||||||
);
|
|
||||||
- |
|
|
||||||
ATTACH '@env_var("R2_WAREHOUSE_NAME", "materia")' AS catalog (
|
|
||||||
TYPE ICEBERG,
|
|
||||||
ENDPOINT '@env_var("ICEBERG_REST_URI")'
|
|
||||||
);
|
);
|
||||||
- CREATE SCHEMA IF NOT EXISTS catalog.materia;
|
- CREATE SCHEMA IF NOT EXISTS catalog.materia;
|
||||||
- USE catalog.materia;
|
- USE catalog.materia;
|
||||||
|
|||||||
Reference in New Issue
Block a user