Compare commits

..

4 Commits

Author SHA1 Message Date
Deeman
169092c8ea fix(admin): make pipeline data view responsive on mobile
All checks were successful
CI / test (push) Successful in 50s
CI / tag (push) Successful in 2s
- Tab bar: add overflow-x:auto so 5 tabs scroll on narrow screens
- Overview grid: replace hardcoded 1fr 1fr with .pipeline-two-col (stacks below 640px)
- Overview tables: wrap Serving Tables + Landing Zone in overflow-x:auto divs

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 13:16:58 +01:00
Deeman
6ae16f6c1f feat(proxy): per-proxy dead tracking in tiered cycler
All checks were successful
CI / test (push) Successful in 51s
CI / tag (push) Successful in 3s
2026-03-01 12:37:00 +01:00
Deeman
8b33daa4f3 feat(content): remove artificial 500-article generation cap
- fetch_template_data: default limit=0 (all rows); skip LIMIT clause when 0
- generate_articles: default limit=0
- worker handle_generate_articles: default to 0 instead of 500
- Remove "limit": 500 from all 4 enqueue payloads
- template_generate GET handler: use count_template_data() instead of fetch(limit=501) probe

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 12:33:58 +01:00
Deeman
219554b7cb fix(extract): use tiered cycler in playtomic_tenants
Previously the tenants extractor flattened all proxy tiers into a single
round-robin list, bypassing the circuit breaker entirely. When the free
Webshare tier runs out of bandwidth (402), all 20 free proxies fail and
the batch crashes — the paid datacenter/residential proxies are never tried.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 12:13:58 +01:00
6 changed files with 27 additions and 16 deletions

View File

@@ -169,7 +169,6 @@ async def pseo_generate_gaps(slug: str):
"template_slug": slug, "template_slug": slug,
"start_date": date.today().isoformat(), "start_date": date.today().isoformat(),
"articles_per_day": 500, "articles_per_day": 500,
"limit": 500,
}) })
await flash( await flash(
f"Queued generation for {len(gaps)} missing articles in '{config['name']}'.", f"Queued generation for {len(gaps)} missing articles in '{config['name']}'.",

View File

@@ -1865,7 +1865,7 @@ async def template_preview(slug: str, row_key: str):
@csrf_protect @csrf_protect
async def template_generate(slug: str): async def template_generate(slug: str):
"""Generate articles from template + DuckDB data.""" """Generate articles from template + DuckDB data."""
from ..content import fetch_template_data, load_template from ..content import count_template_data, load_template
try: try:
config = load_template(slug) config = load_template(slug)
@@ -1873,8 +1873,7 @@ async def template_generate(slug: str):
await flash("Template not found.", "error") await flash("Template not found.", "error")
return redirect(url_for("admin.templates")) return redirect(url_for("admin.templates"))
data_rows = await fetch_template_data(config["data_table"], limit=501) row_count = await count_template_data(config["data_table"])
row_count = len(data_rows)
if request.method == "POST": if request.method == "POST":
form = await request.form form = await request.form
@@ -1888,7 +1887,6 @@ async def template_generate(slug: str):
"template_slug": slug, "template_slug": slug,
"start_date": start_date.isoformat(), "start_date": start_date.isoformat(),
"articles_per_day": articles_per_day, "articles_per_day": articles_per_day,
"limit": 500,
}) })
await flash( await flash(
f"Article generation queued for '{config['name']}'. " f"Article generation queued for '{config['name']}'. "
@@ -1923,7 +1921,6 @@ async def template_regenerate(slug: str):
"template_slug": slug, "template_slug": slug,
"start_date": date.today().isoformat(), "start_date": date.today().isoformat(),
"articles_per_day": 500, "articles_per_day": 500,
"limit": 500,
}) })
await flash("Regeneration queued. The worker will process it in the background.", "success") await flash("Regeneration queued. The worker will process it in the background.", "success")
return redirect(url_for("admin.template_detail", slug=slug)) return redirect(url_for("admin.template_detail", slug=slug))
@@ -2729,7 +2726,6 @@ async def rebuild_all():
"template_slug": t["slug"], "template_slug": t["slug"],
"start_date": date.today().isoformat(), "start_date": date.today().isoformat(),
"articles_per_day": 500, "articles_per_day": 500,
"limit": 500,
}) })
# Manual articles still need inline rebuild # Manual articles still need inline rebuild

View File

@@ -57,7 +57,7 @@
</div> </div>
<!-- Two-column row: Serving Freshness + Landing Zone --> <!-- Two-column row: Serving Freshness + Landing Zone -->
<div style="display:grid;grid-template-columns:1fr 1fr;gap:1rem"> <div class="pipeline-two-col">
<!-- Serving Freshness --> <!-- Serving Freshness -->
<div class="card"> <div class="card">
@@ -68,6 +68,7 @@
</p> </p>
{% endif %} {% endif %}
{% if serving_tables %} {% if serving_tables %}
<div style="overflow-x:auto">
<table class="table" style="font-size:0.8125rem"> <table class="table" style="font-size:0.8125rem">
<thead> <thead>
<tr> <tr>
@@ -86,6 +87,7 @@
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
</div>
{% else %} {% else %}
<p class="text-sm text-slate">No serving tables found — run the pipeline first.</p> <p class="text-sm text-slate">No serving tables found — run the pipeline first.</p>
{% endif %} {% endif %}
@@ -99,6 +101,7 @@
</span> </span>
</p> </p>
{% if landing_stats %} {% if landing_stats %}
<div style="overflow-x:auto">
<table class="table" style="font-size:0.8125rem"> <table class="table" style="font-size:0.8125rem">
<thead> <thead>
<tr> <tr>
@@ -119,6 +122,7 @@
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
</div>
{% else %} {% else %}
<p class="text-sm text-slate"> <p class="text-sm text-slate">
Landing zone empty or not found at <code>data/landing</code>. Landing zone empty or not found at <code>data/landing</code>.

View File

@@ -15,6 +15,7 @@
.pipeline-tabs { .pipeline-tabs {
display: flex; gap: 0; border-bottom: 2px solid #E2E8F0; margin-bottom: 1.5rem; display: flex; gap: 0; border-bottom: 2px solid #E2E8F0; margin-bottom: 1.5rem;
overflow-x: auto; -webkit-overflow-scrolling: touch;
} }
.pipeline-tabs button { .pipeline-tabs button {
padding: 0.625rem 1.25rem; font-size: 0.8125rem; font-weight: 600; padding: 0.625rem 1.25rem; font-size: 0.8125rem; font-weight: 600;
@@ -33,6 +34,15 @@
.status-dot.stale { background: #D97706; } .status-dot.stale { background: #D97706; }
.status-dot.running { background: #3B82F6; } .status-dot.running { background: #3B82F6; }
.status-dot.pending { background: #CBD5E1; } .status-dot.pending { background: #CBD5E1; }
.pipeline-two-col {
display: grid;
grid-template-columns: 1fr;
gap: 1rem;
}
@media (min-width: 640px) {
.pipeline-two-col { grid-template-columns: 1fr 1fr; }
}
</style> </style>
{% endblock %} {% endblock %}

View File

@@ -123,17 +123,19 @@ async def get_table_columns(data_table: str) -> list[dict]:
async def fetch_template_data( async def fetch_template_data(
data_table: str, data_table: str,
order_by: str | None = None, order_by: str | None = None,
limit: int = 500, limit: int = 0,
) -> list[dict]: ) -> list[dict]:
"""Fetch all rows from a DuckDB serving table.""" """Fetch rows from a DuckDB serving table. limit=0 means all rows."""
assert "." in data_table, "data_table must be schema-qualified" assert "." in data_table, "data_table must be schema-qualified"
_validate_table_name(data_table) _validate_table_name(data_table)
order_clause = f"ORDER BY {order_by} DESC" if order_by else "" order_clause = f"ORDER BY {order_by} DESC" if order_by else ""
return await fetch_analytics( if limit:
f"SELECT * FROM {data_table} {order_clause} LIMIT ?", return await fetch_analytics(
[limit], f"SELECT * FROM {data_table} {order_clause} LIMIT ?",
) [limit],
)
return await fetch_analytics(f"SELECT * FROM {data_table} {order_clause}")
async def count_template_data(data_table: str) -> int: async def count_template_data(data_table: str) -> int:
@@ -290,7 +292,7 @@ async def generate_articles(
start_date: date, start_date: date,
articles_per_day: int, articles_per_day: int,
*, *,
limit: int = 500, limit: int = 0,
base_url: str = "https://padelnomics.io", base_url: str = "https://padelnomics.io",
task_id: int | None = None, task_id: int | None = None,
) -> int: ) -> int:

View File

@@ -745,7 +745,7 @@ async def handle_generate_articles(payload: dict) -> None:
slug = payload["template_slug"] slug = payload["template_slug"]
start_date = date_cls.fromisoformat(payload["start_date"]) start_date = date_cls.fromisoformat(payload["start_date"])
articles_per_day = payload.get("articles_per_day", 3) articles_per_day = payload.get("articles_per_day", 3)
limit = payload.get("limit", 500) limit = payload.get("limit", 0)
task_id = payload.get("_task_id") task_id = payload.get("_task_id")
count = await generate_articles( count = await generate_articles(