Compare commits

..

5 Commits

Author SHA1 Message Date
Deeman
fee0d6913b fix(pipeline): use sqlmesh plan --auto-apply instead of run
All checks were successful
CI / test (push) Successful in 56s
CI / tag (push) Successful in 3s
2026-03-06 22:34:58 +01:00
Deeman
71e08a5fa6 fix(pipeline): also update supervisor.py to use plan --auto-apply
Missed the Python supervisor module — same fix as supervisor.sh and
worker.py.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-06 22:33:59 +01:00
Deeman
27e86db6a1 fix(pipeline): use sqlmesh plan --auto-apply instead of sqlmesh run
`sqlmesh run` only re-evaluates intervals for already-planned models —
it does not detect new, modified, or deleted models. Switching to
`plan prod --auto-apply` ensures schema changes (like the new
location_profiles model) are picked up automatically.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-06 22:33:17 +01:00
Deeman
90754b8d9f chore: move ci.py to ~/.claude/scripts (uv inline script, no project dep)
All checks were successful
CI / test (push) Successful in 53s
CI / tag (push) Successful in 2s
Script now lives globally as a uv inline-dependency script.
Removes per-project scripts/ci.py and the msgspec dev dependency.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-06 15:51:36 +01:00
Deeman
277c92e507 chore: add scripts/ci.py for Gitea CI pipeline status
Copies ci.py from beanflows (same script, shared across projects).
Adds msgspec dev dependency required by the script.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-06 15:38:42 +01:00
4 changed files with 13 additions and 8 deletions

View File

@@ -33,10 +33,10 @@ do
DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \
uv run --package padelnomics_extract extract
# Transform — run evaluates missing daily intervals for incremental models.
# Transform — plan detects new/modified/deleted models and applies changes.
LANDING_DIR="${LANDING_DIR:-/data/padelnomics/landing}" \
DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \
uv run sqlmesh -p transform/sqlmesh_padelnomics run prod
uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply
# Export serving tables to analytics.duckdb (atomic swap).
# The web app detects the inode change on next query — no restart needed.

View File

@@ -247,10 +247,10 @@ def run_shell(cmd: str, timeout_seconds: int = SUBPROCESS_TIMEOUT_SECONDS) -> tu
def run_transform() -> None:
"""Run SQLMesh — evaluates missing daily intervals."""
"""Run SQLMesh — detects new/modified/deleted models and applies changes."""
logger.info("Running SQLMesh transform")
ok, err = run_shell(
"uv run sqlmesh -p transform/sqlmesh_padelnomics run prod",
"uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply",
)
if not ok:
send_alert(f"[transform] {err}")

5
uv.lock generated
View File

@@ -150,6 +150,11 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/84/85/57c314a6b35336efbbdc13e5fc9ae13f6b60a0647cfa7c1221178ac6d8ae/brotlicffi-1.2.0.0.tar.gz", hash = "sha256:34345d8d1f9d534fcac2249e57a4c3c8801a33c9942ff9f8574f67a175e17adb", size = 476682, upload-time = "2025-11-21T18:17:57.334Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7c/87/ba6298c3d7f8d66ce80d7a487f2a487ebae74a79c6049c7c2990178ce529/brotlicffi-1.2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b13fb476a96f02e477a506423cb5e7bc21e0e3ac4c060c20ba31c44056e38c68", size = 433038, upload-time = "2026-03-05T17:57:37.96Z" },
{ url = "https://files.pythonhosted.org/packages/00/49/16c7a77d1cae0519953ef0389a11a9c2e2e62e87d04f8e7afbae40124255/brotlicffi-1.2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17db36fb581f7b951635cd6849553a95c6f2f53c1a707817d06eae5aeff5f6af", size = 1541124, upload-time = "2026-03-05T17:57:39.488Z" },
{ url = "https://files.pythonhosted.org/packages/e8/17/fab2c36ea820e2288f8c1bf562de1b6cd9f30e28d66f1ce2929a4baff6de/brotlicffi-1.2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40190192790489a7b054312163d0ce82b07d1b6e706251036898ce1684ef12e9", size = 1541983, upload-time = "2026-03-05T17:57:41.061Z" },
{ url = "https://files.pythonhosted.org/packages/78/c9/849a669b3b3bb8ac96005cdef04df4db658c33443a7fc704a6d4a2f07a56/brotlicffi-1.2.0.0-cp314-cp314t-win32.whl", hash = "sha256:a8079e8ecc32ecef728036a1d9b7105991ce6a5385cf51ee8c02297c90fb08c2", size = 349046, upload-time = "2026-03-05T17:57:42.76Z" },
{ url = "https://files.pythonhosted.org/packages/a4/25/09c0fd21cfc451fa38ad538f4d18d8be566746531f7f27143f63f8c45a9f/brotlicffi-1.2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ca90c4266704ca0a94de8f101b4ec029624273380574e4cf19301acfa46c61a0", size = 385653, upload-time = "2026-03-05T17:57:44.224Z" },
{ url = "https://files.pythonhosted.org/packages/e4/df/a72b284d8c7bef0ed5756b41c2eb7d0219a1dd6ac6762f1c7bdbc31ef3af/brotlicffi-1.2.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:9458d08a7ccde8e3c0afedbf2c70a8263227a68dea5ab13590593f4c0a4fd5f4", size = 432340, upload-time = "2025-11-21T18:17:42.277Z" },
{ url = "https://files.pythonhosted.org/packages/74/2b/cc55a2d1d6fb4f5d458fba44a3d3f91fb4320aa14145799fd3a996af0686/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:84e3d0020cf1bd8b8131f4a07819edee9f283721566fe044a20ec792ca8fd8b7", size = 1534002, upload-time = "2025-11-21T18:17:43.746Z" },
{ url = "https://files.pythonhosted.org/packages/e4/9c/d51486bf366fc7d6735f0e46b5b96ca58dc005b250263525a1eea3cd5d21/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:33cfb408d0cff64cd50bef268c0fed397c46fbb53944aa37264148614a62e990", size = 1536547, upload-time = "2025-11-21T18:17:45.729Z" },

View File

@@ -737,9 +737,9 @@ async def handle_run_extraction(payload: dict) -> None:
@task("run_transform")
async def handle_run_transform(payload: dict) -> None:
"""Run SQLMesh transform (prod run) in the background.
"""Run SQLMesh transform (prod plan + apply) in the background.
Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics run prod`.
Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply`.
2-hour absolute timeout — same as extraction.
"""
import subprocess
@@ -748,7 +748,7 @@ async def handle_run_transform(payload: dict) -> None:
repo_root = Path(__file__).resolve().parents[4]
result = await asyncio.to_thread(
subprocess.run,
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"],
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
capture_output=True,
text=True,
timeout=7200,
@@ -803,7 +803,7 @@ async def handle_run_pipeline(payload: dict) -> None:
),
(
"transform",
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"],
["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
7200,
),
(