merge: spinner, batch commits, pre-compiled Jinja templates, timing
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
@@ -1,9 +1,15 @@
|
||||
{% if is_generating %}
|
||||
<div hx-get="{{ url_for('admin.article_results') }}"
|
||||
<div class="generating-banner"
|
||||
hx-get="{{ url_for('admin.article_results') }}"
|
||||
hx-trigger="every 3s"
|
||||
hx-target="#article-results"
|
||||
hx-swap="innerHTML"
|
||||
style="display:none" aria-hidden="true"></div>
|
||||
hx-swap="innerHTML">
|
||||
<svg class="spinner-icon" width="16" height="16" viewBox="0 0 24 24" fill="none">
|
||||
<circle cx="12" cy="12" r="10" stroke="#CBD5E1" stroke-width="3"/>
|
||||
<path d="M12 2a10 10 0 0 1 10 10" stroke="#0EA5E9" stroke-width="3" stroke-linecap="round"/>
|
||||
</svg>
|
||||
<span>Generating articles…</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if articles %}
|
||||
<div class="card">
|
||||
|
||||
@@ -8,6 +8,7 @@ are stored in SQLite (routing / application state).
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
@@ -16,7 +17,7 @@ import yaml
|
||||
from jinja2 import ChainableUndefined, Environment
|
||||
|
||||
from ..analytics import fetch_analytics
|
||||
from ..core import execute, fetch_one, slugify, transaction, utcnow_iso
|
||||
from ..core import slugify, transaction, utcnow_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -301,29 +302,48 @@ async def generate_articles(
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
# Pre-compile all Jinja templates once — avoids creating a new Environment()
|
||||
# and re-parsing the same template strings on every iteration.
|
||||
_env = Environment(undefined=ChainableUndefined)
|
||||
_env.filters["slugify"] = slugify
|
||||
_env.filters["datetimeformat"] = _datetimeformat
|
||||
url_tmpl = _env.from_string(config["url_pattern"])
|
||||
title_tmpl = _env.from_string(config["title_pattern"])
|
||||
meta_tmpl = _env.from_string(config["meta_description_pattern"])
|
||||
body_tmpl = _env.from_string(config["body_template"])
|
||||
|
||||
publish_date = start_date
|
||||
published_today = 0
|
||||
generated = 0
|
||||
now_iso = utcnow_iso()
|
||||
|
||||
# Timing accumulators — logged at end so we can see where time goes.
|
||||
t_calc = t_render = t_bake = 0.0
|
||||
|
||||
_BATCH_SIZE = 200
|
||||
|
||||
async with transaction() as db:
|
||||
for row in rows:
|
||||
for lang in config["languages"]:
|
||||
# Build render context: row data + language
|
||||
ctx = {**row, "language": lang}
|
||||
# Build render context, replacing None with 0 so numeric
|
||||
# Jinja filters (round, int) don't crash.
|
||||
safe_ctx = {k: (v if v is not None else 0) for k, v in row.items()}
|
||||
safe_ctx["language"] = lang
|
||||
|
||||
# Render URL pattern (no lang prefix — blueprint provides /<lang>)
|
||||
url_path = _render_pattern(config["url_pattern"], ctx)
|
||||
url_path = url_tmpl.render(**safe_ctx)
|
||||
if is_reserved_path(url_path):
|
||||
continue
|
||||
|
||||
title = _render_pattern(config["title_pattern"], ctx)
|
||||
meta_desc = _render_pattern(config["meta_description_pattern"], ctx)
|
||||
title = title_tmpl.render(**safe_ctx)
|
||||
meta_desc = meta_tmpl.render(**safe_ctx)
|
||||
article_slug = slug + "-" + lang + "-" + str(row[config["natural_key"]])
|
||||
|
||||
# Calculator content type: create scenario
|
||||
scenario_slug = None
|
||||
scenario_overrides = None
|
||||
if config["content_type"] == "calculator":
|
||||
t0 = time.perf_counter()
|
||||
# DuckDB lowercases all column names; build a case-insensitive
|
||||
# reverse map so "ratepeak" (stored) matches "ratePeak" (DEFAULTS).
|
||||
_defaults_ci = {k.lower(): k for k in DEFAULTS}
|
||||
@@ -334,6 +354,7 @@ async def generate_articles(
|
||||
}
|
||||
state = validate_state(calc_overrides)
|
||||
d = calc(state, lang=lang)
|
||||
t_calc += time.perf_counter() - t0
|
||||
|
||||
scenario_slug = slug + "-" + str(row[config["natural_key"]])
|
||||
dbl = state.get("dblCourts", 0)
|
||||
@@ -360,12 +381,36 @@ async def generate_articles(
|
||||
),
|
||||
)
|
||||
|
||||
ctx["scenario_slug"] = scenario_slug
|
||||
safe_ctx["scenario_slug"] = scenario_slug
|
||||
# Pass scenario data directly so bake_scenario_cards skips the
|
||||
# DB re-fetch (the row was just upserted and may not be visible
|
||||
# on a separate connection within the same uncommitted transaction).
|
||||
scenario_overrides = {
|
||||
scenario_slug: {
|
||||
"slug": scenario_slug,
|
||||
"title": city,
|
||||
"location": city,
|
||||
"country": country,
|
||||
"venue_type": state.get("venue", "indoor"),
|
||||
"ownership": state.get("own", "rent"),
|
||||
"court_config": court_config,
|
||||
"state_json": json.dumps(state),
|
||||
"calc_json": json.dumps(d),
|
||||
"created_at": now_iso,
|
||||
}
|
||||
}
|
||||
|
||||
# Render body template
|
||||
body_md = _render_pattern(config["body_template"], ctx)
|
||||
t0 = time.perf_counter()
|
||||
body_md = body_tmpl.render(**safe_ctx)
|
||||
body_html = mistune.html(body_md)
|
||||
body_html = await bake_scenario_cards(body_html, lang=lang)
|
||||
t_render += time.perf_counter() - t0
|
||||
|
||||
t0 = time.perf_counter()
|
||||
body_html = await bake_scenario_cards(
|
||||
body_html, lang=lang, scenario_overrides=scenario_overrides
|
||||
)
|
||||
t_bake += time.perf_counter() - t0
|
||||
|
||||
# Extract FAQ pairs for structured data
|
||||
faq_pairs = _extract_faq_pairs(body_md)
|
||||
@@ -377,16 +422,16 @@ async def generate_articles(
|
||||
8, 0, 0,
|
||||
).isoformat()
|
||||
|
||||
# Hreflang links
|
||||
# Hreflang links — reuse compiled url_tmpl with swapped language
|
||||
hreflang_links = []
|
||||
for alt_lang in config["languages"]:
|
||||
alt_url = f"/{alt_lang}" + _render_pattern(config["url_pattern"], {**row, "language": alt_lang})
|
||||
alt_url = f"/{alt_lang}" + url_tmpl.render(**{**safe_ctx, "language": alt_lang})
|
||||
hreflang_links.append(
|
||||
f'<link rel="alternate" hreflang="{alt_lang}" href="{base_url}{alt_url}" />'
|
||||
)
|
||||
# x-default points to English (or first language)
|
||||
default_lang = "en" if "en" in config["languages"] else config["languages"][0]
|
||||
default_url = f"/{default_lang}" + _render_pattern(config["url_pattern"], {**row, "language": default_lang})
|
||||
default_url = f"/{default_lang}" + url_tmpl.render(**{**safe_ctx, "language": default_lang})
|
||||
hreflang_links.append(
|
||||
f'<link rel="alternate" hreflang="x-default" href="{base_url}{default_url}" />'
|
||||
)
|
||||
@@ -451,7 +496,13 @@ async def generate_articles(
|
||||
)
|
||||
|
||||
generated += 1
|
||||
if generated % 25 == 0:
|
||||
|
||||
# Commit every _BATCH_SIZE articles so the admin UI shows progress
|
||||
# earlier rather than waiting for the full run to complete.
|
||||
if generated % _BATCH_SIZE == 0:
|
||||
await db.commit()
|
||||
logger.info("%s: committed batch — %d articles", slug, generated)
|
||||
elif generated % 25 == 0:
|
||||
logger.info("%s: %d articles written…", slug, generated)
|
||||
|
||||
# Stagger dates
|
||||
@@ -460,7 +511,10 @@ async def generate_articles(
|
||||
published_today = 0
|
||||
publish_date += timedelta(days=1)
|
||||
|
||||
logger.info("%s: done — %d total", slug, generated)
|
||||
logger.info(
|
||||
"%s: done — %d total | calc=%.1fs render=%.1fs bake=%.1fs",
|
||||
slug, generated, t_calc, t_render, t_bake,
|
||||
)
|
||||
return generated
|
||||
|
||||
|
||||
|
||||
@@ -568,4 +568,19 @@
|
||||
.article-body details > div {
|
||||
@apply px-4 pb-4 text-slate-dark;
|
||||
}
|
||||
|
||||
/* Article generation spinner banner */
|
||||
.generating-banner {
|
||||
@apply flex items-center gap-3 rounded-xl border border-light-gray bg-white text-sm text-slate-dark mb-4;
|
||||
padding: 0.75rem 1rem;
|
||||
}
|
||||
.spinner-icon {
|
||||
flex-shrink: 0;
|
||||
animation: spin-icon 0.9s linear infinite;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes spin-icon {
|
||||
from { transform: rotate(0deg); }
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user