a1faddbed6
Supervisor (replaces supervisor.sh): - supervisor.py — cron-based pipeline orchestration, reads workflows.toml on every tick, runs due extractors in topological waves with parallel execution, then SQLMesh transform + serving export - workflows.toml — workflow registry: overpass (monthly), eurostat (monthly), playtomic_tenants (weekly), playtomic_availability (daily), playtomic_recheck (hourly 6–23) - padelnomics-supervisor.service — updated ExecStart to Python supervisor Extraction enhancements: - proxy.py — optional round-robin/sticky proxy rotation via PROXY_URLS env - playtomic_availability.py — parallel fetch (EXTRACT_WORKERS), recheck mode (main_recheck) re-queries imminent slots for accurate occupancy measurement - _shared.py — realistic browser User-Agent on all extractor sessions - stg_playtomic_availability.sql — reads morning + recheck snapshots, tags each - fct_daily_availability.sql — prefers recheck over morning for same slot Feature flags (replaces WAITLIST_MODE env var): - migration 0019 — feature_flags table, 5 initial flags: markets (on), payments/planner_export/supplier_signup/lead_unlock (off) - core.py — is_flag_enabled() + feature_gate() decorator - routes — payments, markets, planner_export, supplier_signup, lead_unlock gated - admin flags UI — /admin/flags toggle page + nav link - app.py — flag() injected as Jinja2 global Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
25 lines
773 B
TOML
25 lines
773 B
TOML
[project]
|
|
name = "padelnomics_extract"
|
|
version = "0.2.0"
|
|
description = "Data extraction pipelines for padelnomics"
|
|
requires-python = ">=3.11"
|
|
dependencies = [
|
|
"niquests>=3.14.0",
|
|
"python-dotenv>=1.0.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
extract = "padelnomics_extract.all:main"
|
|
extract-overpass = "padelnomics_extract.overpass:main"
|
|
extract-eurostat = "padelnomics_extract.eurostat:main"
|
|
extract-playtomic-tenants = "padelnomics_extract.playtomic_tenants:main"
|
|
extract-playtomic-availability = "padelnomics_extract.playtomic_availability:main"
|
|
extract-playtomic-recheck = "padelnomics_extract.playtomic_availability:main_recheck"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/padelnomics_extract"]
|