From af20c59ced5fbf96f1114b8577138abe09749f7f Mon Sep 17 00:00:00 2001 From: Deeman Date: Tue, 24 Feb 2026 16:44:02 +0100 Subject: [PATCH] feat(content): spinner, batch commits, pre-compiled templates, timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spinner: - article_results.html: replace hidden polling div with a visible animated spinner banner; CSS spin keyframe added to input.css Batch commits: - generate_articles() now commits every 200 articles instead of holding one giant transaction; articles appear in the admin UI progressively without waiting for the full run Performance (pre-compiled Jinja templates): - Create one Environment + compile url/title/meta/body templates once before the loop instead of calling _render_pattern() per iteration; eliminates ~4 × N Environment() constructions and re-parses of the same template strings (N = articles, typically 500+) - Reuse url_tmpl for hreflang alt-lang rendering Scenario override passthrough: - Pass just-computed scenario data directly to bake_scenario_cards() via scenario_overrides, avoiding a DB SELECT that reads an uncommitted row from a potentially separate connection Timing instrumentation: - Accumulate time spent in calc / render / bake phases per run - Log totals at completion: "done — 500 total | calc=1.2s render=4.3s bake=0.1s" Co-Authored-By: Claude Opus 4.6 --- .../admin/partials/article_results.html | 12 ++- web/src/padelnomics/content/__init__.py | 82 +++++++++++++++---- web/src/padelnomics/static/css/input.css | 15 ++++ 3 files changed, 92 insertions(+), 17 deletions(-) diff --git a/web/src/padelnomics/admin/templates/admin/partials/article_results.html b/web/src/padelnomics/admin/templates/admin/partials/article_results.html index d0ef5d8..ecc551e 100644 --- a/web/src/padelnomics/admin/templates/admin/partials/article_results.html +++ b/web/src/padelnomics/admin/templates/admin/partials/article_results.html @@ -1,9 +1,15 @@ {% if is_generating %} - + hx-swap="innerHTML"> + + Generating articles… + {% endif %} {% if articles %}
diff --git a/web/src/padelnomics/content/__init__.py b/web/src/padelnomics/content/__init__.py index 87d4d04..b9be927 100644 --- a/web/src/padelnomics/content/__init__.py +++ b/web/src/padelnomics/content/__init__.py @@ -8,6 +8,7 @@ are stored in SQLite (routing / application state). import json import logging import re +import time from datetime import UTC, date, datetime, timedelta from pathlib import Path @@ -16,7 +17,7 @@ import yaml from jinja2 import ChainableUndefined, Environment from ..analytics import fetch_analytics -from ..core import execute, fetch_one, slugify, transaction, utcnow_iso +from ..core import slugify, transaction, utcnow_iso logger = logging.getLogger(__name__) @@ -301,29 +302,48 @@ async def generate_articles( if not rows: return 0 + # Pre-compile all Jinja templates once — avoids creating a new Environment() + # and re-parsing the same template strings on every iteration. + _env = Environment(undefined=ChainableUndefined) + _env.filters["slugify"] = slugify + _env.filters["datetimeformat"] = _datetimeformat + url_tmpl = _env.from_string(config["url_pattern"]) + title_tmpl = _env.from_string(config["title_pattern"]) + meta_tmpl = _env.from_string(config["meta_description_pattern"]) + body_tmpl = _env.from_string(config["body_template"]) + publish_date = start_date published_today = 0 generated = 0 now_iso = utcnow_iso() + # Timing accumulators — logged at end so we can see where time goes. + t_calc = t_render = t_bake = 0.0 + + _BATCH_SIZE = 200 + async with transaction() as db: for row in rows: for lang in config["languages"]: - # Build render context: row data + language - ctx = {**row, "language": lang} + # Build render context, replacing None with 0 so numeric + # Jinja filters (round, int) don't crash. + safe_ctx = {k: (v if v is not None else 0) for k, v in row.items()} + safe_ctx["language"] = lang # Render URL pattern (no lang prefix — blueprint provides /) - url_path = _render_pattern(config["url_pattern"], ctx) + url_path = url_tmpl.render(**safe_ctx) if is_reserved_path(url_path): continue - title = _render_pattern(config["title_pattern"], ctx) - meta_desc = _render_pattern(config["meta_description_pattern"], ctx) + title = title_tmpl.render(**safe_ctx) + meta_desc = meta_tmpl.render(**safe_ctx) article_slug = slug + "-" + lang + "-" + str(row[config["natural_key"]]) # Calculator content type: create scenario scenario_slug = None + scenario_overrides = None if config["content_type"] == "calculator": + t0 = time.perf_counter() # DuckDB lowercases all column names; build a case-insensitive # reverse map so "ratepeak" (stored) matches "ratePeak" (DEFAULTS). _defaults_ci = {k.lower(): k for k in DEFAULTS} @@ -334,6 +354,7 @@ async def generate_articles( } state = validate_state(calc_overrides) d = calc(state, lang=lang) + t_calc += time.perf_counter() - t0 scenario_slug = slug + "-" + str(row[config["natural_key"]]) dbl = state.get("dblCourts", 0) @@ -360,12 +381,36 @@ async def generate_articles( ), ) - ctx["scenario_slug"] = scenario_slug + safe_ctx["scenario_slug"] = scenario_slug + # Pass scenario data directly so bake_scenario_cards skips the + # DB re-fetch (the row was just upserted and may not be visible + # on a separate connection within the same uncommitted transaction). + scenario_overrides = { + scenario_slug: { + "slug": scenario_slug, + "title": city, + "location": city, + "country": country, + "venue_type": state.get("venue", "indoor"), + "ownership": state.get("own", "rent"), + "court_config": court_config, + "state_json": json.dumps(state), + "calc_json": json.dumps(d), + "created_at": now_iso, + } + } # Render body template - body_md = _render_pattern(config["body_template"], ctx) + t0 = time.perf_counter() + body_md = body_tmpl.render(**safe_ctx) body_html = mistune.html(body_md) - body_html = await bake_scenario_cards(body_html, lang=lang) + t_render += time.perf_counter() - t0 + + t0 = time.perf_counter() + body_html = await bake_scenario_cards( + body_html, lang=lang, scenario_overrides=scenario_overrides + ) + t_bake += time.perf_counter() - t0 # Extract FAQ pairs for structured data faq_pairs = _extract_faq_pairs(body_md) @@ -377,16 +422,16 @@ async def generate_articles( 8, 0, 0, ).isoformat() - # Hreflang links + # Hreflang links — reuse compiled url_tmpl with swapped language hreflang_links = [] for alt_lang in config["languages"]: - alt_url = f"/{alt_lang}" + _render_pattern(config["url_pattern"], {**row, "language": alt_lang}) + alt_url = f"/{alt_lang}" + url_tmpl.render(**{**safe_ctx, "language": alt_lang}) hreflang_links.append( f'' ) # x-default points to English (or first language) default_lang = "en" if "en" in config["languages"] else config["languages"][0] - default_url = f"/{default_lang}" + _render_pattern(config["url_pattern"], {**row, "language": default_lang}) + default_url = f"/{default_lang}" + url_tmpl.render(**{**safe_ctx, "language": default_lang}) hreflang_links.append( f'' ) @@ -451,7 +496,13 @@ async def generate_articles( ) generated += 1 - if generated % 25 == 0: + + # Commit every _BATCH_SIZE articles so the admin UI shows progress + # earlier rather than waiting for the full run to complete. + if generated % _BATCH_SIZE == 0: + await db.commit() + logger.info("%s: committed batch — %d articles", slug, generated) + elif generated % 25 == 0: logger.info("%s: %d articles written…", slug, generated) # Stagger dates @@ -460,7 +511,10 @@ async def generate_articles( published_today = 0 publish_date += timedelta(days=1) - logger.info("%s: done — %d total", slug, generated) + logger.info( + "%s: done — %d total | calc=%.1fs render=%.1fs bake=%.1fs", + slug, generated, t_calc, t_render, t_bake, + ) return generated diff --git a/web/src/padelnomics/static/css/input.css b/web/src/padelnomics/static/css/input.css index 539d63e..1acff6b 100644 --- a/web/src/padelnomics/static/css/input.css +++ b/web/src/padelnomics/static/css/input.css @@ -568,4 +568,19 @@ .article-body details > div { @apply px-4 pb-4 text-slate-dark; } + + /* Article generation spinner banner */ + .generating-banner { + @apply flex items-center gap-3 rounded-xl border border-light-gray bg-white text-sm text-slate-dark mb-4; + padding: 0.75rem 1rem; + } + .spinner-icon { + flex-shrink: 0; + animation: spin-icon 0.9s linear infinite; + } +} + +@keyframes spin-icon { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } }