fix(pipeline): use sqlmesh plan --auto-apply instead of run
This commit is contained in:
@@ -33,10 +33,10 @@ do
|
||||
DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \
|
||||
uv run --package padelnomics_extract extract
|
||||
|
||||
# Transform — run evaluates missing daily intervals for incremental models.
|
||||
# Transform — plan detects new/modified/deleted models and applies changes.
|
||||
LANDING_DIR="${LANDING_DIR:-/data/padelnomics/landing}" \
|
||||
DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \
|
||||
uv run sqlmesh -p transform/sqlmesh_padelnomics run prod
|
||||
uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply
|
||||
|
||||
# Export serving tables to analytics.duckdb (atomic swap).
|
||||
# The web app detects the inode change on next query — no restart needed.
|
||||
|
||||
@@ -247,10 +247,10 @@ def run_shell(cmd: str, timeout_seconds: int = SUBPROCESS_TIMEOUT_SECONDS) -> tu
|
||||
|
||||
|
||||
def run_transform() -> None:
|
||||
"""Run SQLMesh — evaluates missing daily intervals."""
|
||||
"""Run SQLMesh — detects new/modified/deleted models and applies changes."""
|
||||
logger.info("Running SQLMesh transform")
|
||||
ok, err = run_shell(
|
||||
"uv run sqlmesh -p transform/sqlmesh_padelnomics run prod",
|
||||
"uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply",
|
||||
)
|
||||
if not ok:
|
||||
send_alert(f"[transform] {err}")
|
||||
|
||||
@@ -737,9 +737,9 @@ async def handle_run_extraction(payload: dict) -> None:
|
||||
|
||||
@task("run_transform")
|
||||
async def handle_run_transform(payload: dict) -> None:
|
||||
"""Run SQLMesh transform (prod run) in the background.
|
||||
"""Run SQLMesh transform (prod plan + apply) in the background.
|
||||
|
||||
Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics run prod`.
|
||||
Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply`.
|
||||
2-hour absolute timeout — same as extraction.
|
||||
"""
|
||||
import subprocess
|
||||
@@ -748,7 +748,7 @@ async def handle_run_transform(payload: dict) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
result = await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"],
|
||||
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=7200,
|
||||
@@ -803,7 +803,7 @@ async def handle_run_pipeline(payload: dict) -> None:
|
||||
),
|
||||
(
|
||||
"transform",
|
||||
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"],
|
||||
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
|
||||
7200,
|
||||
),
|
||||
(
|
||||
|
||||
Reference in New Issue
Block a user