Compare commits

..

3 Commits

Author SHA1 Message Date
Deeman
6ae16f6c1f feat(proxy): per-proxy dead tracking in tiered cycler
All checks were successful
CI / test (push) Successful in 51s
CI / tag (push) Successful in 3s
2026-03-01 12:37:00 +01:00
Deeman
8b33daa4f3 feat(content): remove artificial 500-article generation cap
- fetch_template_data: default limit=0 (all rows); skip LIMIT clause when 0
- generate_articles: default limit=0
- worker handle_generate_articles: default to 0 instead of 500
- Remove "limit": 500 from all 4 enqueue payloads
- template_generate GET handler: use count_template_data() instead of fetch(limit=501) probe

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 12:33:58 +01:00
Deeman
219554b7cb fix(extract): use tiered cycler in playtomic_tenants
Previously the tenants extractor flattened all proxy tiers into a single
round-robin list, bypassing the circuit breaker entirely. When the free
Webshare tier runs out of bandwidth (402), all 20 free proxies fail and
the batch crashes — the paid datacenter/residential proxies are never tried.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-01 12:13:58 +01:00
4 changed files with 12 additions and 15 deletions

View File

@@ -169,7 +169,6 @@ async def pseo_generate_gaps(slug: str):
"template_slug": slug,
"start_date": date.today().isoformat(),
"articles_per_day": 500,
"limit": 500,
})
await flash(
f"Queued generation for {len(gaps)} missing articles in '{config['name']}'.",

View File

@@ -1865,7 +1865,7 @@ async def template_preview(slug: str, row_key: str):
@csrf_protect
async def template_generate(slug: str):
"""Generate articles from template + DuckDB data."""
from ..content import fetch_template_data, load_template
from ..content import count_template_data, load_template
try:
config = load_template(slug)
@@ -1873,8 +1873,7 @@ async def template_generate(slug: str):
await flash("Template not found.", "error")
return redirect(url_for("admin.templates"))
data_rows = await fetch_template_data(config["data_table"], limit=501)
row_count = len(data_rows)
row_count = await count_template_data(config["data_table"])
if request.method == "POST":
form = await request.form
@@ -1888,7 +1887,6 @@ async def template_generate(slug: str):
"template_slug": slug,
"start_date": start_date.isoformat(),
"articles_per_day": articles_per_day,
"limit": 500,
})
await flash(
f"Article generation queued for '{config['name']}'. "
@@ -1923,7 +1921,6 @@ async def template_regenerate(slug: str):
"template_slug": slug,
"start_date": date.today().isoformat(),
"articles_per_day": 500,
"limit": 500,
})
await flash("Regeneration queued. The worker will process it in the background.", "success")
return redirect(url_for("admin.template_detail", slug=slug))
@@ -2729,7 +2726,6 @@ async def rebuild_all():
"template_slug": t["slug"],
"start_date": date.today().isoformat(),
"articles_per_day": 500,
"limit": 500,
})
# Manual articles still need inline rebuild

View File

@@ -123,17 +123,19 @@ async def get_table_columns(data_table: str) -> list[dict]:
async def fetch_template_data(
data_table: str,
order_by: str | None = None,
limit: int = 500,
limit: int = 0,
) -> list[dict]:
"""Fetch all rows from a DuckDB serving table."""
"""Fetch rows from a DuckDB serving table. limit=0 means all rows."""
assert "." in data_table, "data_table must be schema-qualified"
_validate_table_name(data_table)
order_clause = f"ORDER BY {order_by} DESC" if order_by else ""
return await fetch_analytics(
f"SELECT * FROM {data_table} {order_clause} LIMIT ?",
[limit],
)
if limit:
return await fetch_analytics(
f"SELECT * FROM {data_table} {order_clause} LIMIT ?",
[limit],
)
return await fetch_analytics(f"SELECT * FROM {data_table} {order_clause}")
async def count_template_data(data_table: str) -> int:
@@ -290,7 +292,7 @@ async def generate_articles(
start_date: date,
articles_per_day: int,
*,
limit: int = 500,
limit: int = 0,
base_url: str = "https://padelnomics.io",
task_id: int | None = None,
) -> int:

View File

@@ -745,7 +745,7 @@ async def handle_generate_articles(payload: dict) -> None:
slug = payload["template_slug"]
start_date = date_cls.fromisoformat(payload["start_date"])
articles_per_day = payload.get("articles_per_day", 3)
limit = payload.get("limit", 500)
limit = payload.get("limit", 0)
task_id = payload.get("_task_id")
count = await generate_articles(