feat(content): spinner, batch commits, pre-compiled templates, timing

Spinner:
- article_results.html: replace hidden polling div with a visible
  animated spinner banner; CSS spin keyframe added to input.css

Batch commits:
- generate_articles() now commits every 200 articles instead of
  holding one giant transaction; articles appear in the admin UI
  progressively without waiting for the full run

Performance (pre-compiled Jinja templates):
- Create one Environment + compile url/title/meta/body templates once
  before the loop instead of calling _render_pattern() per iteration;
  eliminates ~4 × N Environment() constructions and re-parses of the
  same template strings (N = articles, typically 500+)
- Reuse url_tmpl for hreflang alt-lang rendering

Scenario override passthrough:
- Pass just-computed scenario data directly to bake_scenario_cards()
  via scenario_overrides, avoiding a DB SELECT that reads an uncommitted
  row from a potentially separate connection

Timing instrumentation:
- Accumulate time spent in calc / render / bake phases per run
- Log totals at completion: "done — 500 total | calc=1.2s render=4.3s bake=0.1s"

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
2026-02-24 16:44:02 +01:00
parent 482b4f9fca
commit af20c59ced
3 changed files with 92 additions and 17 deletions

View File

@@ -1,9 +1,15 @@
{% if is_generating %} {% if is_generating %}
<div hx-get="{{ url_for('admin.article_results') }}" <div class="generating-banner"
hx-get="{{ url_for('admin.article_results') }}"
hx-trigger="every 3s" hx-trigger="every 3s"
hx-target="#article-results" hx-target="#article-results"
hx-swap="innerHTML" hx-swap="innerHTML">
style="display:none" aria-hidden="true"></div> <svg class="spinner-icon" width="16" height="16" viewBox="0 0 24 24" fill="none">
<circle cx="12" cy="12" r="10" stroke="#CBD5E1" stroke-width="3"/>
<path d="M12 2a10 10 0 0 1 10 10" stroke="#0EA5E9" stroke-width="3" stroke-linecap="round"/>
</svg>
<span>Generating articles…</span>
</div>
{% endif %} {% endif %}
{% if articles %} {% if articles %}
<div class="card"> <div class="card">

View File

@@ -8,6 +8,7 @@ are stored in SQLite (routing / application state).
import json import json
import logging import logging
import re import re
import time
from datetime import UTC, date, datetime, timedelta from datetime import UTC, date, datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -16,7 +17,7 @@ import yaml
from jinja2 import ChainableUndefined, Environment from jinja2 import ChainableUndefined, Environment
from ..analytics import fetch_analytics from ..analytics import fetch_analytics
from ..core import execute, fetch_one, slugify, transaction, utcnow_iso from ..core import slugify, transaction, utcnow_iso
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -301,29 +302,48 @@ async def generate_articles(
if not rows: if not rows:
return 0 return 0
# Pre-compile all Jinja templates once — avoids creating a new Environment()
# and re-parsing the same template strings on every iteration.
_env = Environment(undefined=ChainableUndefined)
_env.filters["slugify"] = slugify
_env.filters["datetimeformat"] = _datetimeformat
url_tmpl = _env.from_string(config["url_pattern"])
title_tmpl = _env.from_string(config["title_pattern"])
meta_tmpl = _env.from_string(config["meta_description_pattern"])
body_tmpl = _env.from_string(config["body_template"])
publish_date = start_date publish_date = start_date
published_today = 0 published_today = 0
generated = 0 generated = 0
now_iso = utcnow_iso() now_iso = utcnow_iso()
# Timing accumulators — logged at end so we can see where time goes.
t_calc = t_render = t_bake = 0.0
_BATCH_SIZE = 200
async with transaction() as db: async with transaction() as db:
for row in rows: for row in rows:
for lang in config["languages"]: for lang in config["languages"]:
# Build render context: row data + language # Build render context, replacing None with 0 so numeric
ctx = {**row, "language": lang} # Jinja filters (round, int) don't crash.
safe_ctx = {k: (v if v is not None else 0) for k, v in row.items()}
safe_ctx["language"] = lang
# Render URL pattern (no lang prefix — blueprint provides /<lang>) # Render URL pattern (no lang prefix — blueprint provides /<lang>)
url_path = _render_pattern(config["url_pattern"], ctx) url_path = url_tmpl.render(**safe_ctx)
if is_reserved_path(url_path): if is_reserved_path(url_path):
continue continue
title = _render_pattern(config["title_pattern"], ctx) title = title_tmpl.render(**safe_ctx)
meta_desc = _render_pattern(config["meta_description_pattern"], ctx) meta_desc = meta_tmpl.render(**safe_ctx)
article_slug = slug + "-" + lang + "-" + str(row[config["natural_key"]]) article_slug = slug + "-" + lang + "-" + str(row[config["natural_key"]])
# Calculator content type: create scenario # Calculator content type: create scenario
scenario_slug = None scenario_slug = None
scenario_overrides = None
if config["content_type"] == "calculator": if config["content_type"] == "calculator":
t0 = time.perf_counter()
# DuckDB lowercases all column names; build a case-insensitive # DuckDB lowercases all column names; build a case-insensitive
# reverse map so "ratepeak" (stored) matches "ratePeak" (DEFAULTS). # reverse map so "ratepeak" (stored) matches "ratePeak" (DEFAULTS).
_defaults_ci = {k.lower(): k for k in DEFAULTS} _defaults_ci = {k.lower(): k for k in DEFAULTS}
@@ -334,6 +354,7 @@ async def generate_articles(
} }
state = validate_state(calc_overrides) state = validate_state(calc_overrides)
d = calc(state, lang=lang) d = calc(state, lang=lang)
t_calc += time.perf_counter() - t0
scenario_slug = slug + "-" + str(row[config["natural_key"]]) scenario_slug = slug + "-" + str(row[config["natural_key"]])
dbl = state.get("dblCourts", 0) dbl = state.get("dblCourts", 0)
@@ -360,12 +381,36 @@ async def generate_articles(
), ),
) )
ctx["scenario_slug"] = scenario_slug safe_ctx["scenario_slug"] = scenario_slug
# Pass scenario data directly so bake_scenario_cards skips the
# DB re-fetch (the row was just upserted and may not be visible
# on a separate connection within the same uncommitted transaction).
scenario_overrides = {
scenario_slug: {
"slug": scenario_slug,
"title": city,
"location": city,
"country": country,
"venue_type": state.get("venue", "indoor"),
"ownership": state.get("own", "rent"),
"court_config": court_config,
"state_json": json.dumps(state),
"calc_json": json.dumps(d),
"created_at": now_iso,
}
}
# Render body template # Render body template
body_md = _render_pattern(config["body_template"], ctx) t0 = time.perf_counter()
body_md = body_tmpl.render(**safe_ctx)
body_html = mistune.html(body_md) body_html = mistune.html(body_md)
body_html = await bake_scenario_cards(body_html, lang=lang) t_render += time.perf_counter() - t0
t0 = time.perf_counter()
body_html = await bake_scenario_cards(
body_html, lang=lang, scenario_overrides=scenario_overrides
)
t_bake += time.perf_counter() - t0
# Extract FAQ pairs for structured data # Extract FAQ pairs for structured data
faq_pairs = _extract_faq_pairs(body_md) faq_pairs = _extract_faq_pairs(body_md)
@@ -377,16 +422,16 @@ async def generate_articles(
8, 0, 0, 8, 0, 0,
).isoformat() ).isoformat()
# Hreflang links # Hreflang links — reuse compiled url_tmpl with swapped language
hreflang_links = [] hreflang_links = []
for alt_lang in config["languages"]: for alt_lang in config["languages"]:
alt_url = f"/{alt_lang}" + _render_pattern(config["url_pattern"], {**row, "language": alt_lang}) alt_url = f"/{alt_lang}" + url_tmpl.render(**{**safe_ctx, "language": alt_lang})
hreflang_links.append( hreflang_links.append(
f'<link rel="alternate" hreflang="{alt_lang}" href="{base_url}{alt_url}" />' f'<link rel="alternate" hreflang="{alt_lang}" href="{base_url}{alt_url}" />'
) )
# x-default points to English (or first language) # x-default points to English (or first language)
default_lang = "en" if "en" in config["languages"] else config["languages"][0] default_lang = "en" if "en" in config["languages"] else config["languages"][0]
default_url = f"/{default_lang}" + _render_pattern(config["url_pattern"], {**row, "language": default_lang}) default_url = f"/{default_lang}" + url_tmpl.render(**{**safe_ctx, "language": default_lang})
hreflang_links.append( hreflang_links.append(
f'<link rel="alternate" hreflang="x-default" href="{base_url}{default_url}" />' f'<link rel="alternate" hreflang="x-default" href="{base_url}{default_url}" />'
) )
@@ -451,7 +496,13 @@ async def generate_articles(
) )
generated += 1 generated += 1
if generated % 25 == 0:
# Commit every _BATCH_SIZE articles so the admin UI shows progress
# earlier rather than waiting for the full run to complete.
if generated % _BATCH_SIZE == 0:
await db.commit()
logger.info("%s: committed batch — %d articles", slug, generated)
elif generated % 25 == 0:
logger.info("%s: %d articles written…", slug, generated) logger.info("%s: %d articles written…", slug, generated)
# Stagger dates # Stagger dates
@@ -460,7 +511,10 @@ async def generate_articles(
published_today = 0 published_today = 0
publish_date += timedelta(days=1) publish_date += timedelta(days=1)
logger.info("%s: done — %d total", slug, generated) logger.info(
"%s: done — %d total | calc=%.1fs render=%.1fs bake=%.1fs",
slug, generated, t_calc, t_render, t_bake,
)
return generated return generated

View File

@@ -568,4 +568,19 @@
.article-body details > div { .article-body details > div {
@apply px-4 pb-4 text-slate-dark; @apply px-4 pb-4 text-slate-dark;
} }
/* Article generation spinner banner */
.generating-banner {
@apply flex items-center gap-3 rounded-xl border border-light-gray bg-white text-sm text-slate-dark mb-4;
padding: 0.75rem 1rem;
}
.spinner-icon {
flex-shrink: 0;
animation: spin-icon 0.9s linear infinite;
}
}
@keyframes spin-icon {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
} }