90 lines
3.3 KiB
YAML
90 lines
3.3 KiB
YAML
# --- Gateway Connection ---
|
|
# Single gateway connecting to R2 Iceberg catalog
|
|
# Local dev uses virtual environments (e.g., dev_<username>)
|
|
# Production uses the 'prod' environment
|
|
gateways:
|
|
duckdb:
|
|
connection:
|
|
type: duckdb
|
|
catalogs:
|
|
local: 'local.duckdb'
|
|
cloudflare:
|
|
type: iceberg
|
|
path: '{{ env_var("ICEBERG_WAREHOUSE_NAME") }}'
|
|
connector_config:
|
|
endpoint: '{{ env_var("ICEBERG_CATALOG_URI") }}'
|
|
extensions:
|
|
- name: httpfs
|
|
- name: iceberg
|
|
secrets:
|
|
r2_secret:
|
|
type: iceberg
|
|
token: "{{ env_var('R2_ADMIN_API_TOKEN') }}"
|
|
r2_data_secret:
|
|
type: r2
|
|
key_id: "{{ env_var('R2_ADMIN_ACCESS_KEY_ID') }}"
|
|
secret: "{{ env_var('R2_ADMIN_SECRET_ACCESS_KEY') }}"
|
|
account_id: "{{ var('CLOUDFLARE_ACCOUNT_ID') }}"
|
|
region: 'eeur'
|
|
|
|
|
|
|
|
default_gateway: duckdb
|
|
|
|
# --- Variables ---
|
|
# Make environment variables available to models
|
|
variables:
|
|
R2_BUCKET: beanflows-data-prod
|
|
CLOUDFLARE_ACCOUNT_ID: "{{ env_var('CLOUDFLARE_ACCOUNT_ID') }}"
|
|
|
|
# --- Catalog Configuration ---
|
|
# Attach R2 Iceberg catalog and configure default schema
|
|
# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#execution-hooks
|
|
# https://developers.cloudflare.com/r2/data-catalog/config-examples/duckdb/
|
|
|
|
#before_all:
|
|
# - "ATTACH '{{ env_var('ICEBERG_WAREHOUSE_NAME') }}' AS catalog (TYPE ICEBERG, ENDPOINT '{{ env_var('ICEBERG_CATALOG_URI') }}', SECRET r2_secret);"
|
|
# Note: R2 data access is configured via r2_data_secret (TYPE R2)
|
|
# Models can use r2://bucket/path to read landing data
|
|
# Note: CREATE SCHEMA has a DuckDB/Iceberg bug (missing Content-Type header)
|
|
# Schema must be pre-created in R2 Data Catalog via Cloudflare dashboard or API
|
|
# For now, skip USE statement and rely on fully-qualified table names in models
|
|
|
|
# --- Model Defaults ---
|
|
# https://sqlmesh.readthedocs.io/en/stable/reference/model_configuration/#model-defaults
|
|
|
|
model_defaults:
|
|
dialect: duckdb
|
|
start: 2025-07-07 # Start date for backfill history
|
|
cron: '@daily' # Run models daily at 12am UTC (can override per model)
|
|
|
|
# --- Linting Rules ---
|
|
# Enforce standards for your team
|
|
# https://sqlmesh.readthedocs.io/en/stable/guides/linter/
|
|
|
|
linter:
|
|
enabled: true
|
|
rules:
|
|
- ambiguousorinvalidcolumn
|
|
- invalidselectstarexpansion
|
|
|
|
# FLOW: Minimal prompts, automatic changes, summary output
|
|
# https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#plan
|
|
|
|
#plan:
|
|
# no_diff: true # Hide detailed text differences for changed models
|
|
# no_prompts: true # No interactive prompts
|
|
# auto_apply: true # Apply changes automatically
|
|
|
|
# --- Optional: Set a default target environment ---
|
|
# This is intended for local development to prevent users from accidentally applying plans to the prod environment.
|
|
# It is a development only config and should NOT be committed to your git repo.
|
|
# https://sqlmesh.readthedocs.io/en/stable/guides/configuration/#default-target-environment
|
|
|
|
# Uncomment the following line to use a default target environment derived from the logged in user's name.
|
|
default_target_environment: dev_{{ user() }}
|
|
|
|
# Example usage:
|
|
# sqlmesh plan # Automatically resolves to: sqlmesh plan dev_yourname
|
|
# sqlmesh plan prod # Specify `prod` to apply changes to production
|