Merge branch 'worktree-cms-admin-improvement'

This commit is contained in:
Deeman
2026-02-24 01:22:55 +01:00
10 changed files with 371 additions and 98 deletions

View File

@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## [Unreleased]
### Added
- **CMS admin improvement** — articles list now has HTMX filter bar (search,
status, template, language), pagination (50/page), and stats strip
(total/live/scheduled/draft counts). Article actions (publish/unpublish,
delete) are inline HTMX operations — no full page reload. "View" link opens
live articles on the public site. Article generation and rebuild-all now
enqueue to the background worker instead of blocking the HTTP request.
Markdown source is written to disk during generation so the edit form shows
content. Sitemap cache is invalidated when articles are published, deleted,
or created. Fixed broken "Scheduled"/"Published" status display (was always
showing "Scheduled") and stale `template_data_id` column reference.
### Changed
- **Visual test overhaul** — consolidated 3 separate Playwright server processes
(ports 5111/5112/5113) into 1 session-scoped fixture in `conftest.py`; 77 tests

View File

@@ -79,6 +79,7 @@
- [x] City coverage: DE (8), US (6), UK (4), ES (5), FR (3), IT (2), NL, AT, CH, SE, PT (2), BE, AE, AU (2), IE
- [x] Per-city financial model overrides (rates, rent, utilities, permits, court config)
- [x] Admin CMS (template CRUD, data row management, bulk CSV upload, bulk generate, publish toggle, rebuild)
- [x] Admin CMS v2: HTMX filter/search/pagination, background generation, inline actions, sitemap invalidation, markdown editing
- [x] Markets hub (`/<lang>/markets`) — article listing with FTS + country/region filters
- [x] DuckDB refresh script (`refresh_from_daas.py`)

View File

@@ -1314,7 +1314,7 @@ async def template_preview(slug: str, row_key: str):
@csrf_protect
async def template_generate(slug: str):
"""Generate articles from template + DuckDB data."""
from ..content import fetch_template_data, generate_articles, load_template
from ..content import fetch_template_data, load_template
try:
config = load_template(slug)
@@ -1330,15 +1330,20 @@ async def template_generate(slug: str):
start_date_str = form.get("start_date", "")
articles_per_day = int(form.get("articles_per_day", 3) or 3)
if not start_date_str:
start_date = date.today()
else:
start_date = date.fromisoformat(start_date_str)
start_date = date.fromisoformat(start_date_str) if start_date_str else date.today()
generated = await generate_articles(
slug, start_date, articles_per_day, limit=500,
from ..worker import enqueue
await enqueue("generate_articles", {
"template_slug": slug,
"start_date": start_date.isoformat(),
"articles_per_day": articles_per_day,
"limit": 500,
})
await flash(
f"Article generation queued for '{config['name']}'. "
"The worker will process it in the background.",
"success",
)
await flash(f"Generated {generated} articles with staggered publish dates.", "success")
return redirect(url_for("admin.articles"))
return await render_template(
@@ -1354,7 +1359,7 @@ async def template_generate(slug: str):
@csrf_protect
async def template_regenerate(slug: str):
"""Re-generate all articles for a template with fresh DuckDB data."""
from ..content import generate_articles, load_template
from ..content import load_template
try:
load_template(slug)
@@ -1362,9 +1367,14 @@ async def template_regenerate(slug: str):
await flash("Template not found.", "error")
return redirect(url_for("admin.templates"))
# Use today as start date, keep existing publish dates via upsert
generated = await generate_articles(slug, date.today(), articles_per_day=500)
await flash(f"Regenerated {generated} articles from fresh data.", "success")
from ..worker import enqueue
await enqueue("generate_articles", {
"template_slug": slug,
"start_date": date.today().isoformat(),
"articles_per_day": 500,
"limit": 500,
})
await flash("Regeneration queued. The worker will process it in the background.", "success")
return redirect(url_for("admin.template_detail", slug=slug))
@@ -1581,14 +1591,112 @@ async def scenario_pdf(scenario_id: int):
# Article Management
# =============================================================================
async def _get_article_list(
status: str = None,
template_slug: str = None,
language: str = None,
search: str = None,
page: int = 1,
per_page: int = 50,
) -> list[dict]:
"""Get articles with optional filters and pagination."""
wheres = ["1=1"]
params: list = []
if status == "live":
wheres.append("status = 'published' AND published_at <= datetime('now')")
elif status == "scheduled":
wheres.append("status = 'published' AND published_at > datetime('now')")
elif status == "draft":
wheres.append("status = 'draft'")
if template_slug:
wheres.append("template_slug = ?")
params.append(template_slug)
if language:
wheres.append("language = ?")
params.append(language)
if search:
wheres.append("title LIKE ?")
params.append(f"%{search}%")
where = " AND ".join(wheres)
offset = (page - 1) * per_page
params.extend([per_page, offset])
return await fetch_all(
f"""SELECT *,
CASE WHEN status = 'published' AND published_at > datetime('now')
THEN 'scheduled'
WHEN status = 'published' THEN 'live'
ELSE status END AS display_status
FROM articles WHERE {where}
ORDER BY created_at DESC LIMIT ? OFFSET ?""",
tuple(params),
)
async def _get_article_stats() -> dict:
"""Get aggregate article stats for the admin list header."""
row = await fetch_one(
"""SELECT
COUNT(*) AS total,
SUM(CASE WHEN status='published' AND published_at <= datetime('now') THEN 1 ELSE 0 END) AS live,
SUM(CASE WHEN status='published' AND published_at > datetime('now') THEN 1 ELSE 0 END) AS scheduled,
SUM(CASE WHEN status='draft' THEN 1 ELSE 0 END) AS draft
FROM articles"""
)
return dict(row) if row else {"total": 0, "live": 0, "scheduled": 0, "draft": 0}
@bp.route("/articles")
@role_required("admin")
async def articles():
"""List all articles."""
article_list = await fetch_all(
"SELECT * FROM articles ORDER BY created_at DESC"
"""List all articles with filters."""
search = request.args.get("search", "").strip()
status_filter = request.args.get("status", "")
template_filter = request.args.get("template", "")
language_filter = request.args.get("language", "")
page = max(1, int(request.args.get("page", "1") or "1"))
article_list = await _get_article_list(
status=status_filter or None, template_slug=template_filter or None,
language=language_filter or None, search=search or None, page=page,
)
stats = await _get_article_stats()
templates = await fetch_all(
"SELECT DISTINCT template_slug FROM articles WHERE template_slug IS NOT NULL ORDER BY template_slug"
)
return await render_template(
"admin/articles.html",
articles=article_list,
stats=stats,
template_slugs=[t["template_slug"] for t in templates],
current_search=search,
current_status=status_filter,
current_template=template_filter,
current_language=language_filter,
page=page,
)
@bp.route("/articles/results")
@role_required("admin")
async def article_results():
"""HTMX partial for filtered article results."""
search = request.args.get("search", "").strip()
status_filter = request.args.get("status", "")
template_filter = request.args.get("template", "")
language_filter = request.args.get("language", "")
page = max(1, int(request.args.get("page", "1") or "1"))
article_list = await _get_article_list(
status=status_filter or None, template_slug=template_filter or None,
language=language_filter or None, search=search or None, page=page,
)
return await render_template(
"admin/partials/article_results.html", articles=article_list, page=page,
)
return await render_template("admin/articles.html", articles=article_list)
@bp.route("/articles/new", methods=["GET", "POST"])
@@ -1642,6 +1750,9 @@ async def article_new():
(url_path, article_slug, title, meta_description, og_image_url,
country, region, status, pub_dt),
)
from ..sitemap import invalidate_sitemap_cache
invalidate_sitemap_cache()
await flash(f"Article '{title}' created.", "success")
return redirect(url_for("admin.articles"))
@@ -1703,8 +1814,12 @@ async def article_edit(article_id: int):
await flash("Article updated.", "success")
return redirect(url_for("admin.articles"))
# Load markdown source if available
# Load markdown source if available (manual or generated)
from ..content.routes import BUILD_DIR as CONTENT_BUILD_DIR
md_path = Path("data/content/articles") / f"{article['slug']}.md"
if not md_path.exists():
lang = article["language"] or "en"
md_path = CONTENT_BUILD_DIR / lang / "md" / f"{article['slug']}.md"
body = md_path.read_text() if md_path.exists() else ""
data = {**dict(article), "body": body}
@@ -1730,6 +1845,13 @@ async def article_delete(article_id: int):
md_path.unlink()
await execute("DELETE FROM articles WHERE id = ?", (article_id,))
from ..sitemap import invalidate_sitemap_cache
invalidate_sitemap_cache()
if request.headers.get("HX-Request"):
return "" # row removed via hx-swap="outerHTML"
await flash("Article deleted.", "success")
return redirect(url_for("admin.articles"))
@@ -1750,6 +1872,22 @@ async def article_publish(article_id: int):
"UPDATE articles SET status = ?, updated_at = ? WHERE id = ?",
(new_status, now, article_id),
)
from ..sitemap import invalidate_sitemap_cache
invalidate_sitemap_cache()
if request.headers.get("HX-Request"):
updated = await fetch_one(
"""SELECT *,
CASE WHEN status = 'published' AND published_at > datetime('now')
THEN 'scheduled'
WHEN status = 'published' THEN 'live'
ELSE status END AS display_status
FROM articles WHERE id = ?""",
(article_id,),
)
return await render_template("admin/partials/article_row.html", a=updated)
await flash(f"Article status changed to {new_status}.", "success")
return redirect(url_for("admin.articles"))
@@ -1768,13 +1906,29 @@ async def article_rebuild(article_id: int):
@role_required("admin")
@csrf_protect
async def rebuild_all():
"""Re-render all articles."""
articles = await fetch_all("SELECT id FROM articles")
count = 0
for a in articles:
"""Re-render all articles via background worker."""
from ..content import discover_templates
from ..worker import enqueue
templates = discover_templates()
for t in templates:
await enqueue("generate_articles", {
"template_slug": t["slug"],
"start_date": date.today().isoformat(),
"articles_per_day": 500,
"limit": 500,
})
# Manual articles still need inline rebuild
manual = await fetch_all("SELECT id FROM articles WHERE template_slug IS NULL")
for a in manual:
await _rebuild_article(a["id"])
count += 1
await flash(f"Rebuilt {count} articles.", "success")
await flash(
f"Queued rebuild for {len(templates)} templates"
f" + rebuilt {len(manual)} manual articles.",
"success",
)
return redirect(url_for("admin.articles"))

View File

@@ -4,73 +4,72 @@
{% block title %}Articles - Admin - {{ config.APP_NAME }}{% endblock %}
{% block admin_content %}
<header class="flex justify-between items-center mb-8">
<header class="flex justify-between items-center mb-6">
<div>
<h1 class="text-2xl">Articles</h1>
<p class="text-slate text-sm">{{ articles | length }} article{{ 's' if articles | length != 1 }}</p>
<p class="text-sm text-slate mt-1">
{{ stats.total }} total
&middot; {{ stats.live }} live
&middot; {{ stats.scheduled }} scheduled
&middot; {{ stats.draft }} draft
</p>
</div>
<div class="flex gap-2">
<a href="{{ url_for('admin.article_new') }}" class="btn">New Article</a>
<form method="post" action="{{ url_for('admin.rebuild_all') }}" class="m-0" style="display: inline;">
<a href="{{ url_for('admin.article_new') }}" class="btn btn-sm">New Article</a>
<form method="post" action="{{ url_for('admin.rebuild_all') }}" class="m-0" style="display:inline">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button type="submit" class="btn-outline" onclick="return confirm('Rebuild all articles?')">Rebuild All</button>
<button type="submit" class="btn-outline btn-sm" onclick="return confirm('Rebuild all articles?')">Rebuild All</button>
</form>
<a href="{{ url_for('admin.index') }}" class="btn-outline">Back</a>
</div>
</header>
<div class="card">
{% if articles %}
<table class="table">
<thead>
<tr>
<th>Title</th>
<th>URL</th>
<th>Status</th>
<th>Published</th>
<th>Source</th>
<th></th>
</tr>
</thead>
<tbody>
{% for a in articles %}
<tr>
<td>{{ a.title }}</td>
<td class="mono text-sm">{{ a.url_path }}</td>
<td>
{% if a.status == 'published' %}
{% if a.published_at and a.published_at > now.isoformat() %}
<span class="badge-warning">Scheduled</span>
{% else %}
<span class="badge-success">Published</span>
{% endif %}
{% else %}
<span class="badge">Draft</span>
{% endif %}
</td>
<td class="mono text-sm">{{ a.published_at[:10] if a.published_at else '-' }}</td>
<td class="text-sm">{% if a.template_data_id %}Generated{% else %}Manual{% endif %}</td>
<td class="text-right" style="white-space: nowrap;">
<form method="post" action="{{ url_for('admin.article_publish', article_id=a.id) }}" class="m-0" style="display: inline;">
{# Filters #}
<div class="card mb-6" style="padding:1rem 1.25rem">
<form class="flex flex-wrap gap-3 items-end"
hx-get="{{ url_for('admin.article_results') }}"
hx-target="#article-results"
hx-trigger="change, input delay:300ms from:find input">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button type="submit" class="btn-outline btn-sm">{% if a.status == 'published' %}Unpublish{% else %}Publish{% endif %}</button>
</form>
<a href="{{ url_for('admin.article_edit', article_id=a.id) }}" class="btn-outline btn-sm">Edit</a>
<form method="post" action="{{ url_for('admin.article_rebuild', article_id=a.id) }}" class="m-0" style="display: inline;">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button type="submit" class="btn-outline btn-sm">Rebuild</button>
</form>
<form method="post" action="{{ url_for('admin.article_delete', article_id=a.id) }}" class="m-0" style="display: inline;">
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
<button type="submit" class="btn-outline btn-sm" onclick="return confirm('Delete this article?')">Delete</button>
</form>
</td>
</tr>
<div>
<label class="text-xs font-semibold text-slate block mb-1">Search</label>
<input type="text" name="search" value="{{ current_search }}" placeholder="Title..."
class="form-input" style="min-width:180px">
</div>
<div>
<label class="text-xs font-semibold text-slate block mb-1">Status</label>
<select name="status" class="form-input" style="min-width:120px">
<option value="">All</option>
<option value="live" {% if current_status == 'live' %}selected{% endif %}>Live</option>
<option value="scheduled" {% if current_status == 'scheduled' %}selected{% endif %}>Scheduled</option>
<option value="draft" {% if current_status == 'draft' %}selected{% endif %}>Draft</option>
</select>
</div>
<div>
<label class="text-xs font-semibold text-slate block mb-1">Template</label>
<select name="template" class="form-input" style="min-width:140px">
<option value="">All</option>
{% for t in template_slugs %}
<option value="{{ t }}" {% if t == current_template %}selected{% endif %}>{{ t }}</option>
{% endfor %}
</tbody>
</table>
{% else %}
<p class="text-slate text-sm">No articles yet. Create one or generate from a template.</p>
{% endif %}
</select>
</div>
<div>
<label class="text-xs font-semibold text-slate block mb-1">Language</label>
<select name="language" class="form-input" style="min-width:80px">
<option value="">All</option>
<option value="en" {% if current_language == 'en' %}selected{% endif %}>EN</option>
<option value="de" {% if current_language == 'de' %}selected{% endif %}>DE</option>
</select>
</div>
</form>
</div>
{# Results #}
<div id="article-results">
{% include "admin/partials/article_results.html" %}
</div>
{% endblock %}

View File

@@ -0,0 +1,42 @@
{% if articles %}
<div class="card">
<table class="table text-sm">
<thead>
<tr>
<th>Title</th>
<th>Status</th>
<th>Published</th>
<th>Lang</th>
<th>Template</th>
<th></th>
</tr>
</thead>
<tbody>
{% for a in articles %}
{% include "admin/partials/article_row.html" %}
{% endfor %}
</tbody>
</table>
{% if articles | length >= 50 %}
<div class="flex justify-between items-center" style="padding:0.75rem 1rem; border-top:1px solid #E2E8F0">
{% if page > 1 %}
<button class="btn-outline btn-sm"
hx-get="{{ url_for('admin.article_results') }}?page={{ page - 1 }}"
hx-target="#article-results"
hx-include="closest form">Prev</button>
{% else %}
<span></span>
{% endif %}
<span class="text-xs text-slate">Page {{ page }}</span>
<button class="btn-outline btn-sm"
hx-get="{{ url_for('admin.article_results') }}?page={{ page + 1 }}"
hx-target="#article-results"
hx-include="closest form">Next</button>
</div>
{% endif %}
</div>
{% else %}
<div class="card text-center" style="padding:2rem">
<p class="text-slate">No articles match the current filters.</p>
</div>
{% endif %}

View File

@@ -0,0 +1,33 @@
<tr id="article-{{ a.id }}">
<td style="max-width:280px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap"
title="{{ a.url_path }}">{{ a.title }}</td>
<td>
{% if a.display_status == 'live' %}
<span class="badge-success">Live</span>
{% elif a.display_status == 'scheduled' %}
<span class="badge-warning">Scheduled</span>
{% else %}
<span class="badge">Draft</span>
{% endif %}
</td>
<td class="mono">{{ a.published_at[:10] if a.published_at else '-' }}</td>
<td>{{ a.language | upper if a.language else '-' }}</td>
<td class="text-slate">{{ a.template_slug or 'Manual' }}</td>
<td class="text-right" style="white-space:nowrap">
{% if a.display_status == 'live' %}
<a href="/{{ a.language or 'en' }}{{ a.url_path }}" target="_blank" class="btn-outline btn-sm">View</a>
{% endif %}
<a href="{{ url_for('admin.article_edit', article_id=a.id) }}" class="btn-outline btn-sm">Edit</a>
<button hx-post="{{ url_for('admin.article_publish', article_id=a.id) }}"
hx-target="#article-{{ a.id }}" hx-swap="outerHTML"
hx-headers='{"X-CSRF-Token": "{{ csrf_token() }}"}'
class="btn-outline btn-sm">
{% if a.display_status != 'draft' %}Unpublish{% else %}Publish{% endif %}
</button>
<button hx-post="{{ url_for('admin.article_delete', article_id=a.id) }}"
hx-target="#article-{{ a.id }}" hx-swap="outerHTML swap:200ms"
hx-confirm="Delete this article?"
hx-headers='{"X-CSRF-Token": "{{ csrf_token() }}"}'
class="btn-outline btn-sm">Delete</button>
</td>
</tr>

View File

@@ -429,6 +429,11 @@ async def generate_articles(
build_dir.mkdir(parents=True, exist_ok=True)
(build_dir / f"{article_slug}.html").write_text(body_html)
# Write markdown source to disk (for admin editing)
md_dir = BUILD_DIR / lang / "md"
md_dir.mkdir(parents=True, exist_ok=True)
(md_dir / f"{article_slug}.md").write_text(body_md)
# Upsert article in SQLite
existing_article = await fetch_one(
"SELECT id FROM articles WHERE url_path = ?", (url_path,),

View File

@@ -103,6 +103,13 @@ async def _generate_sitemap_xml(base_url: str) -> str:
return xml
def invalidate_sitemap_cache() -> None:
"""Force sitemap regeneration on next request."""
global _cache_xml, _cache_timestamp # noqa: PLW0603
_cache_xml = ""
_cache_timestamp = 0.0
async def sitemap_response(base_url: str) -> Response:
"""Return cached sitemap XML, regenerating if stale (1-hour TTL)."""
global _cache_xml, _cache_timestamp # noqa: PLW0603

View File

@@ -709,6 +709,22 @@ async def handle_cleanup_seo_metrics(payload: dict) -> None:
print(f"[WORKER] Cleaned up {deleted} old SEO metric rows")
@task("generate_articles")
async def handle_generate_articles(payload: dict) -> None:
"""Generate articles from a template in the background."""
from datetime import date as date_cls
from .content import generate_articles
slug = payload["template_slug"]
start_date = date_cls.fromisoformat(payload["start_date"])
articles_per_day = payload.get("articles_per_day", 3)
limit = payload.get("limit", 500)
count = await generate_articles(slug, start_date, articles_per_day, limit=limit)
print(f"[WORKER] Generated {count} articles for template '{slug}'")
# =============================================================================
# Worker Loop
# =============================================================================

View File

@@ -1019,7 +1019,7 @@ class TestAdminTemplateGenerate:
assert "3" in html # 3 rows available
assert "Generate" in html
async def test_generate_creates_articles(self, admin_client, db, pseo_env):
async def test_generate_enqueues_task(self, admin_client, db, pseo_env):
async with admin_client.session_transaction() as sess:
sess["csrf_token"] = "test"
@@ -1030,11 +1030,16 @@ class TestAdminTemplateGenerate:
})
assert resp.status_code == 302
articles = await fetch_all("SELECT * FROM articles")
assert len(articles) == 3
scenarios = await fetch_all("SELECT * FROM published_scenarios")
assert len(scenarios) == 3
# Generation is now queued, not inline
tasks = await fetch_all(
"SELECT * FROM tasks WHERE task_name = 'generate_articles'"
)
assert len(tasks) == 1
import json
payload = json.loads(tasks[0]["payload"])
assert payload["template_slug"] == "test-city"
assert payload["start_date"] == "2026-04-01"
assert payload["articles_per_day"] == 2
async def test_generate_unknown_template_redirects(self, admin_client, db, pseo_env):
resp = await admin_client.get("/admin/templates/nonexistent/generate")
@@ -1042,13 +1047,7 @@ class TestAdminTemplateGenerate:
class TestAdminTemplateRegenerate:
async def test_regenerate_updates_articles(self, admin_client, db, pseo_env):
from padelnomics.content import generate_articles
# First generate
await generate_articles("test-city", date(2026, 3, 1), 10)
initial = await fetch_all("SELECT * FROM articles")
assert len(initial) == 3
async def test_regenerate_enqueues_task(self, admin_client, db, pseo_env):
async with admin_client.session_transaction() as sess:
sess["csrf_token"] = "test"
@@ -1057,9 +1056,14 @@ class TestAdminTemplateRegenerate:
})
assert resp.status_code == 302
# Same count — upserted, not duplicated
articles = await fetch_all("SELECT * FROM articles")
assert len(articles) == 3
# Regeneration is now queued, not inline
tasks = await fetch_all(
"SELECT * FROM tasks WHERE task_name = 'generate_articles'"
)
assert len(tasks) == 1
import json
payload = json.loads(tasks[0]["payload"])
assert payload["template_slug"] == "test-city"
async def test_regenerate_unknown_template_redirects(self, admin_client, db, pseo_env):
async with admin_client.session_transaction() as sess: