From 27e86db6a115656be066db5695f28125889b63fa Mon Sep 17 00:00:00 2001 From: Deeman Date: Fri, 6 Mar 2026 22:33:17 +0100 Subject: [PATCH 1/2] fix(pipeline): use `sqlmesh plan --auto-apply` instead of `sqlmesh run` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sqlmesh run` only re-evaluates intervals for already-planned models — it does not detect new, modified, or deleted models. Switching to `plan prod --auto-apply` ensures schema changes (like the new location_profiles model) are picked up automatically. Co-Authored-By: Claude Sonnet 4.6 --- infra/supervisor/supervisor.sh | 4 ++-- web/src/padelnomics/worker.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/infra/supervisor/supervisor.sh b/infra/supervisor/supervisor.sh index f21f425..736d967 100644 --- a/infra/supervisor/supervisor.sh +++ b/infra/supervisor/supervisor.sh @@ -33,10 +33,10 @@ do DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \ uv run --package padelnomics_extract extract - # Transform — run evaluates missing daily intervals for incremental models. + # Transform — plan detects new/modified/deleted models and applies changes. LANDING_DIR="${LANDING_DIR:-/data/padelnomics/landing}" \ DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \ - uv run sqlmesh -p transform/sqlmesh_padelnomics run prod + uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply # Export serving tables to analytics.duckdb (atomic swap). # The web app detects the inode change on next query — no restart needed. diff --git a/web/src/padelnomics/worker.py b/web/src/padelnomics/worker.py index ceebc76..8b95b8e 100644 --- a/web/src/padelnomics/worker.py +++ b/web/src/padelnomics/worker.py @@ -737,9 +737,9 @@ async def handle_run_extraction(payload: dict) -> None: @task("run_transform") async def handle_run_transform(payload: dict) -> None: - """Run SQLMesh transform (prod run) in the background. + """Run SQLMesh transform (prod plan + apply) in the background. - Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics run prod`. + Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply`. 2-hour absolute timeout — same as extraction. """ import subprocess @@ -748,7 +748,7 @@ async def handle_run_transform(payload: dict) -> None: repo_root = Path(__file__).resolve().parents[4] result = await asyncio.to_thread( subprocess.run, - ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"], + ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"], capture_output=True, text=True, timeout=7200, @@ -803,7 +803,7 @@ async def handle_run_pipeline(payload: dict) -> None: ), ( "transform", - ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"], + ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"], 7200, ), ( From 71e08a5fa6a7ef76fa05ec0127036e7d49ebe499 Mon Sep 17 00:00:00 2001 From: Deeman Date: Fri, 6 Mar 2026 22:33:59 +0100 Subject: [PATCH 2/2] fix(pipeline): also update supervisor.py to use plan --auto-apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Missed the Python supervisor module — same fix as supervisor.sh and worker.py. Co-Authored-By: Claude Sonnet 4.6 --- src/padelnomics/supervisor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/padelnomics/supervisor.py b/src/padelnomics/supervisor.py index c2a8529..fcd2cba 100644 --- a/src/padelnomics/supervisor.py +++ b/src/padelnomics/supervisor.py @@ -247,10 +247,10 @@ def run_shell(cmd: str, timeout_seconds: int = SUBPROCESS_TIMEOUT_SECONDS) -> tu def run_transform() -> None: - """Run SQLMesh — evaluates missing daily intervals.""" + """Run SQLMesh — detects new/modified/deleted models and applies changes.""" logger.info("Running SQLMesh transform") ok, err = run_shell( - "uv run sqlmesh -p transform/sqlmesh_padelnomics run prod", + "uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply", ) if not ok: send_alert(f"[transform] {err}")