diff --git a/infra/supervisor/supervisor.sh b/infra/supervisor/supervisor.sh index a855b12..f21f425 100644 --- a/infra/supervisor/supervisor.sh +++ b/infra/supervisor/supervisor.sh @@ -33,10 +33,10 @@ do DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \ uv run --package padelnomics_extract extract - # Transform — plan detects new/changed models; run only executes existing plans. + # Transform — run evaluates missing daily intervals for incremental models. LANDING_DIR="${LANDING_DIR:-/data/padelnomics/landing}" \ DUCKDB_PATH="${DUCKDB_PATH:-/data/padelnomics/lakehouse.duckdb}" \ - uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply + uv run sqlmesh -p transform/sqlmesh_padelnomics run prod # Export serving tables to analytics.duckdb (atomic swap). # The web app detects the inode change on next query — no restart needed. diff --git a/src/padelnomics/supervisor.py b/src/padelnomics/supervisor.py index 02ccfbe..c2a8529 100644 --- a/src/padelnomics/supervisor.py +++ b/src/padelnomics/supervisor.py @@ -247,10 +247,10 @@ def run_shell(cmd: str, timeout_seconds: int = SUBPROCESS_TIMEOUT_SECONDS) -> tu def run_transform() -> None: - """Run SQLMesh — it evaluates model staleness internally.""" + """Run SQLMesh — evaluates missing daily intervals.""" logger.info("Running SQLMesh transform") ok, err = run_shell( - "uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply", + "uv run sqlmesh -p transform/sqlmesh_padelnomics run prod", ) if not ok: send_alert(f"[transform] {err}") @@ -358,6 +358,8 @@ def git_pull_and_sync() -> None: run_shell(f"git checkout --detach {latest}") run_shell("sops --input-type dotenv --output-type dotenv -d .env.prod.sops > .env") run_shell("uv sync --all-packages") + # Apply any model changes (FULL→INCREMENTAL, new models, etc.) before re-exec + run_shell("uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply") # Re-exec so the new code is loaded. os.execv replaces this process in-place; # systemd sees it as the same PID and does not restart the unit. logger.info("Deploy complete — re-execing to load new code") diff --git a/transform/sqlmesh_padelnomics/models/foundation/fct_availability_slot.sql b/transform/sqlmesh_padelnomics/models/foundation/fct_availability_slot.sql index 8094e9a..8107f9c 100644 --- a/transform/sqlmesh_padelnomics/models/foundation/fct_availability_slot.sql +++ b/transform/sqlmesh_padelnomics/models/foundation/fct_availability_slot.sql @@ -14,7 +14,10 @@ MODEL ( name foundation.fct_availability_slot, - kind FULL, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column snapshot_date + ), + start '2026-03-01', cron '@daily', grain (snapshot_date, tenant_id, resource_id, slot_start_time) ); @@ -37,7 +40,8 @@ WITH deduped AS ( captured_at_utc DESC ) AS rn FROM staging.stg_playtomic_availability - WHERE price_amount IS NOT NULL + WHERE snapshot_date BETWEEN @start_ds AND @end_ds + AND price_amount IS NOT NULL AND price_amount > 0 ) SELECT diff --git a/transform/sqlmesh_padelnomics/models/foundation/fct_daily_availability.sql b/transform/sqlmesh_padelnomics/models/foundation/fct_daily_availability.sql index 74b8b8a..bdbabb3 100644 --- a/transform/sqlmesh_padelnomics/models/foundation/fct_daily_availability.sql +++ b/transform/sqlmesh_padelnomics/models/foundation/fct_daily_availability.sql @@ -12,7 +12,10 @@ MODEL ( name foundation.fct_daily_availability, - kind FULL, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column snapshot_date + ), + start '2026-03-01', cron '@daily', grain (snapshot_date, tenant_id) ); @@ -37,6 +40,7 @@ WITH slot_agg AS ( MAX(a.price_currency) AS price_currency, MAX(a.captured_at_utc) AS captured_at_utc FROM foundation.fct_availability_slot a + WHERE a.snapshot_date BETWEEN @start_ds AND @end_ds GROUP BY a.snapshot_date, a.tenant_id ) SELECT diff --git a/transform/sqlmesh_padelnomics/models/serving/venue_pricing_benchmarks.sql b/transform/sqlmesh_padelnomics/models/serving/venue_pricing_benchmarks.sql index a305ad4..592ebc8 100644 --- a/transform/sqlmesh_padelnomics/models/serving/venue_pricing_benchmarks.sql +++ b/transform/sqlmesh_padelnomics/models/serving/venue_pricing_benchmarks.sql @@ -27,7 +27,7 @@ WITH venue_stats AS ( MAX(da.active_court_count) AS court_count, COUNT(DISTINCT da.snapshot_date) AS days_observed FROM foundation.fct_daily_availability da - WHERE TRY_CAST(da.snapshot_date AS DATE) >= CURRENT_DATE - INTERVAL '30 days' + WHERE da.snapshot_date >= CURRENT_DATE - INTERVAL '30 days' AND da.occupancy_rate IS NOT NULL AND da.occupancy_rate BETWEEN 0 AND 1.5 GROUP BY da.tenant_id, da.country_code, da.city, da.city_slug, da.price_currency diff --git a/transform/sqlmesh_padelnomics/models/staging/stg_playtomic_availability.sql b/transform/sqlmesh_padelnomics/models/staging/stg_playtomic_availability.sql index 40ea9e9..d6fa37d 100644 --- a/transform/sqlmesh_padelnomics/models/staging/stg_playtomic_availability.sql +++ b/transform/sqlmesh_padelnomics/models/staging/stg_playtomic_availability.sql @@ -13,44 +13,28 @@ MODEL ( name staging.stg_playtomic_availability, - kind FULL, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column snapshot_date + ), + start '2026-03-01', cron '@daily', grain (snapshot_date, tenant_id, resource_id, slot_start_time, snapshot_type, captured_at_utc) ); WITH -morning_jsonl AS ( +all_jsonl AS ( SELECT - date AS snapshot_date, + CAST(date AS DATE) AS snapshot_date, captured_at_utc, - 'morning' AS snapshot_type, - NULL::INTEGER AS recheck_hour, - tenant_id, - slots AS slots_json - FROM read_json( - @LANDING_DIR || '/playtomic/*/*/availability_*.jsonl.gz', - format = 'newline_delimited', - columns = { - date: 'VARCHAR', - captured_at_utc: 'VARCHAR', - tenant_id: 'VARCHAR', - slots: 'JSON' - }, - filename = true - ) - WHERE filename NOT LIKE '%_recheck_%' - AND tenant_id IS NOT NULL -), -recheck_jsonl AS ( - SELECT - date AS snapshot_date, - captured_at_utc, - 'recheck' AS snapshot_type, + CASE + WHEN filename LIKE '%_recheck_%' THEN 'recheck' + ELSE 'morning' + END AS snapshot_type, TRY_CAST(recheck_hour AS INTEGER) AS recheck_hour, tenant_id, slots AS slots_json FROM read_json( - @LANDING_DIR || '/playtomic/*/*/availability_*_recheck_*.jsonl.gz', + @LANDING_DIR || '/playtomic/*/*/availability_' || @start_ds || '*.jsonl.gz', format = 'newline_delimited', columns = { date: 'VARCHAR', @@ -63,11 +47,6 @@ recheck_jsonl AS ( ) WHERE tenant_id IS NOT NULL ), -all_venues AS ( - SELECT * FROM morning_jsonl - UNION ALL - SELECT * FROM recheck_jsonl -), raw_resources AS ( SELECT av.snapshot_date, @@ -76,7 +55,7 @@ raw_resources AS ( av.recheck_hour, av.tenant_id, resource_json - FROM all_venues av, + FROM all_jsonl av, LATERAL UNNEST( from_json(av.slots_json, '["JSON"]') ) AS t(resource_json) diff --git a/web/src/padelnomics/admin/routes.py b/web/src/padelnomics/admin/routes.py index bf649b7..cb37dca 100644 --- a/web/src/padelnomics/admin/routes.py +++ b/web/src/padelnomics/admin/routes.py @@ -2736,13 +2736,13 @@ async def article_edit(article_id: int): body = raw[m.end():].lstrip("\n") if m else raw body_html = mistune.html(body) if body else "" - css_url = url_for("static", filename="css/output.css") preview_doc = ( - f"" - f"" - f"" - f"
{body_html}
" - ) if body_html else "" + await render_template( + "admin/partials/article_preview_doc.html", body_html=body_html + ) + if body_html + else "" + ) data = {**dict(article), "body": body} return await render_template( @@ -2764,13 +2764,13 @@ async def article_preview(): m = _FRONTMATTER_RE.match(body) body = body[m.end():].lstrip("\n") if m else body body_html = mistune.html(body) if body else "" - css_url = url_for("static", filename="css/output.css") preview_doc = ( - f"" - f"" - f"" - f"
{body_html}
" - ) if body_html else "" + await render_template( + "admin/partials/article_preview_doc.html", body_html=body_html + ) + if body_html + else "" + ) return await render_template("admin/partials/article_preview.html", preview_doc=preview_doc) diff --git a/web/src/padelnomics/admin/templates/admin/article_form.html b/web/src/padelnomics/admin/templates/admin/article_form.html index ae2b9f0..8db1daf 100644 --- a/web/src/padelnomics/admin/templates/admin/article_form.html +++ b/web/src/padelnomics/admin/templates/admin/article_form.html @@ -384,7 +384,7 @@ {% else %} diff --git a/web/src/padelnomics/admin/templates/admin/partials/article_preview.html b/web/src/padelnomics/admin/templates/admin/partials/article_preview.html index 7630039..88a27fa 100644 --- a/web/src/padelnomics/admin/templates/admin/partials/article_preview.html +++ b/web/src/padelnomics/admin/templates/admin/partials/article_preview.html @@ -4,7 +4,7 @@ {% else %} diff --git a/web/src/padelnomics/admin/templates/admin/partials/article_preview_doc.html b/web/src/padelnomics/admin/templates/admin/partials/article_preview_doc.html new file mode 100644 index 0000000..a160193 --- /dev/null +++ b/web/src/padelnomics/admin/templates/admin/partials/article_preview_doc.html @@ -0,0 +1,15 @@ +{# Standalone HTML document used as iframe srcdoc for the article editor preview. + Includes Leaflet so map shortcodes render correctly. #} + + + + + + + + +
{{ body_html | safe }}
+ + + + diff --git a/web/src/padelnomics/content/templates/article_detail.html b/web/src/padelnomics/content/templates/article_detail.html index 3a6e1fd..2820e01 100644 --- a/web/src/padelnomics/content/templates/article_detail.html +++ b/web/src/padelnomics/content/templates/article_detail.html @@ -60,106 +60,6 @@ {% endblock %} {% block scripts %} - + + {% endblock %} diff --git a/web/src/padelnomics/static/js/article-maps.js b/web/src/padelnomics/static/js/article-maps.js new file mode 100644 index 0000000..38de362 --- /dev/null +++ b/web/src/padelnomics/static/js/article-maps.js @@ -0,0 +1,108 @@ +/** + * Leaflet map initialisation for article pages (country + city maps). + * + * Looks for #country-map and #city-map elements. If neither exists, does nothing. + * Expects data-* attributes on the map elements and a global LEAFLET_JS_URL + * variable pointing to the Leaflet JS bundle. + */ +(function() { + var countryMapEl = document.getElementById('country-map'); + var cityMapEl = document.getElementById('city-map'); + if (!countryMapEl && !cityMapEl) return; + + var TILES = 'https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png'; + var TILES_ATTR = '© OSM © CARTO'; + + function scoreColor(score) { + if (score >= 60) return '#16A34A'; + if (score >= 30) return '#D97706'; + return '#DC2626'; + } + + function makeIcon(size, color) { + var s = Math.round(size); + return L.divIcon({ + className: '', + html: '
', + iconSize: [s, s], + iconAnchor: [s / 2, s / 2], + }); + } + + function initCountryMap(el) { + var slug = el.dataset.countrySlug; + var map = L.map(el, {scrollWheelZoom: false}); + L.tileLayer(TILES, { attribution: TILES_ATTR, maxZoom: 18 }).addTo(map); + var lang = document.documentElement.lang || 'en'; + fetch('/api/markets/' + slug + '/cities.json') + .then(function(r) { return r.json(); }) + .then(function(data) { + if (!data.length) return; + var maxV = Math.max.apply(null, data.map(function(d) { return d.padel_venue_count || 1; })); + var bounds = []; + data.forEach(function(c) { + if (!c.lat || !c.lon) return; + var size = 10 + 36 * Math.sqrt((c.padel_venue_count || 1) / maxV); + var color = scoreColor(c.market_score); + var pop = c.population >= 1000000 + ? (c.population / 1000000).toFixed(1) + 'M' + : (c.population >= 1000 ? Math.round(c.population / 1000) + 'K' : (c.population || '')); + var tip = '' + c.city_name + '
' + + (c.padel_venue_count || 0) + ' venues' + + (pop ? ' · ' + pop : '') + '
' + + 'Score ' + Math.round(c.market_score) + '/100'; + L.marker([c.lat, c.lon], { icon: makeIcon(size, color) }) + .bindTooltip(tip, { className: 'map-tooltip', direction: 'top', offset: [0, -Math.round(size / 2)] }) + .on('click', function() { window.location = '/' + lang + '/markets/' + slug + '/' + c.city_slug; }) + .addTo(map); + bounds.push([c.lat, c.lon]); + }); + if (bounds.length) map.fitBounds(bounds, { padding: [24, 24] }); + }); + } + + var VENUE_ICON = L.divIcon({ + className: '', + html: '
', + iconSize: [10, 10], + iconAnchor: [5, 5], + }); + + function initCityMap(el) { + var countrySlug = el.dataset.countrySlug; + var citySlug = el.dataset.citySlug; + var lat = parseFloat(el.dataset.lat); + var lon = parseFloat(el.dataset.lon); + var map = L.map(el, {scrollWheelZoom: false}).setView([lat, lon], 13); + L.tileLayer(TILES, { attribution: TILES_ATTR, maxZoom: 18 }).addTo(map); + fetch('/api/markets/' + countrySlug + '/' + citySlug + '/venues.json') + .then(function(r) { return r.json(); }) + .then(function(data) { + data.forEach(function(v) { + if (!v.lat || !v.lon) return; + var indoor = v.indoor_court_count || 0; + var outdoor = v.outdoor_court_count || 0; + var total = v.court_count || (indoor + outdoor); + var courtLine = total + ? total + ' court' + (total > 1 ? 's' : '') + + (indoor || outdoor + ? ' (' + [indoor ? indoor + ' indoor' : '', outdoor ? outdoor + ' outdoor' : ''].filter(Boolean).join(', ') + ')' + : '') + : ''; + var tip = '' + v.name + '' + (courtLine ? '
' + courtLine : ''); + L.marker([v.lat, v.lon], { icon: VENUE_ICON }) + .bindTooltip(tip, { className: 'map-tooltip', direction: 'top', offset: [0, -7] }) + .addTo(map); + }); + }); + } + + /* Dynamically load Leaflet JS then init maps */ + var script = document.createElement('script'); + script.src = window.LEAFLET_JS_URL || '/static/vendor/leaflet/leaflet.min.js'; + script.onload = function() { + if (countryMapEl) initCountryMap(countryMapEl); + if (cityMapEl) initCityMap(cityMapEl); + }; + document.head.appendChild(script); +})(); diff --git a/web/src/padelnomics/worker.py b/web/src/padelnomics/worker.py index 0a91771..ceebc76 100644 --- a/web/src/padelnomics/worker.py +++ b/web/src/padelnomics/worker.py @@ -737,9 +737,9 @@ async def handle_run_extraction(payload: dict) -> None: @task("run_transform") async def handle_run_transform(payload: dict) -> None: - """Run SQLMesh transform (prod plan --auto-apply) in the background. + """Run SQLMesh transform (prod run) in the background. - Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply`. + Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics run prod`. 2-hour absolute timeout — same as extraction. """ import subprocess @@ -748,7 +748,7 @@ async def handle_run_transform(payload: dict) -> None: repo_root = Path(__file__).resolve().parents[4] result = await asyncio.to_thread( subprocess.run, - ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"], + ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"], capture_output=True, text=True, timeout=7200, @@ -803,7 +803,7 @@ async def handle_run_pipeline(payload: dict) -> None: ), ( "transform", - ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"], + ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "run", "prod"], 7200, ), (