chore(secrets): add R2_LANDING_* vars for

landing-zone backup bucket…
merge: GISCO extractor + wire all extractors + load_dotenv fix
2026-03-01 17:32:51 +01:00 · 2026-03-01 17:08:42 +01:00 · 2026-03-01 17:06:09 +01:00 · 2026-03-01 16:41:59 +01:00 · 2026-03-01 15:49:39 +01:00 · 2026-03-01 14:43:18 +01:00
23 changed files with 1302 additions and 235 deletions
--- a/.env.dev.sops
+++ b/.env.dev.sops
@@ -3,6 +3,8 @@ APP_NAME=ENC[AES256_GCM,data:Vic/MJYoxZo8JAI=,iv:n1SEGQaGeZtYMtLmDRFiljDBbNKFvCz
 SECRET_KEY=ENC[AES256_GCM,data:a3Bhj3gSQaE3llRWBYzpjoFDhhhSsNee67jXJs7+qn4=,iv:yvrx78X5Ut4DBSlmBnIn09ESVc/tuDiwiV4njmjcvko=,tag:cbFUTAEpX+isQD9FCVllsw==,type:str]
 BASE_URL=ENC[AES256_GCM,data:LcbPDZf9Pwcuv7RxN9xhNfa9Tufi,iv:cOdjW9nNe+BuDXh+dL4b5LFQL2mKBiKV0FaEsDGMAQc=,tag:3uAn3AIwsztIfGpkQLD5Fg==,type:str]
 DEBUG=ENC[AES256_GCM,data:qrEGkA==,iv:bCyEDWiEzolHo4vabiyYTsqM0eUaBmNbXYYu4wCsaeE=,tag:80gnDNbdZHRWVEYtuA1M2Q==,type:str]
 #ENC[AES256_GCM,data:YB5h,iv:2HFpvHNebAB9M/44rtPk/QpFV9hNKOlV/099OSjPnOA=,tag:BVj8vGy6K3LW/wb1vcZ+Ug==,type:comment]
 GITEA_TOKEN=ENC[AES256_GCM,data:aIM7vQXxFbz7FDdXEdwtelvmXAdLgJfWNCSPeK//NlveQrU5cLDt8w==,iv:9qhjk52ZAs+y5WwP5WebMUwHhu6JNdHzAsEOpznrwBw=,tag:WnCDA4hAccMFs6vXVVKqxw==,type:str]
 #ENC[AES256_GCM,data:YmlGAWpXxRCqam3oTWtGxHDXC+svEXI4HyUxrm/8OcKTuJsYPcL1WcnYqrP5Mf5lU5qPezEXUrrgZy8vjVW6qAbb0IA2PMM4Kg==,iv:dx6Dn99dJgjwyvUp8NAygXjRQ50yKYFeC73Oqt9WvmY=,tag:6JLF2ixSAv39VkKt6+cecQ==,type:comment]
 ADMIN_EMAILS=ENC[AES256_GCM,data:hlG8b32WlD4ems3VKQ==,iv:wWO08dmX4oLhHulXg4HUG0PjRnFiX19RUTkTvjqIw5I=,tag:KMjXsBt7aE/KqlCfV+fdMg==,type:str]
 #ENC[AES256_GCM,data:b2wQxnL8Q2Bp,iv:q8ep3yUPzCumpZpljoVL2jbcPdsI5c2piiZ0x5k10Mw=,tag:IbjkT0Mjgu9n+6FGiPVihg==,type:comment]
@@ -58,7 +60,7 @@ NTFY_TOKEN=
 #ENC[AES256_GCM,data:BCyQYjRnTx8yW9A=,iv:4OPCP+xzRLUJrpoFewVnbZRKnZH4sAbV76SM//2k5wU=,tag:HxwEp7VFVZUN/VjPiL/+Vw==,type:comment]
 RECHECK_WINDOW_MINUTES=ENC[AES256_GCM,data:YWM=,iv:iY5+uMazLAFdwyLT7Gr7MaF1QHBIgHuoi6nF2VbSsOA=,tag:dc6AmuJdTQ55gVe16uzs6A==,type:str]
 PROXY_URLS_RESIDENTIAL=ENC[AES256_GCM,data:lfmlsjXFtL+zo40SNFLiFKaZiYvE7CNH+zRwjMK5pqPfCs0TlMX+Y9e1KmzAS+y/cI69TP5sgMPRBzER0Jn7RvH0KA==,iv:jBN/4/K5L5886G4rSzxt8V8u/57tAuj3R76haltzqeU=,tag:Xe6o9eg2PodfktDqmLgVNA==,type:str]
-PROXY_URLS_DATACENTER=ENC[AES256_GCM,data:X6xpxz5u8Xh3OXjkIz3UwqH847qLvY9cVWVktW5B+lqhmXAKTzoTzHds8vlRGJf5Up9Yx44XcigbvuK33ZJDSq9ovkAIbY55OK4=,iv:3hHyFD+H9HMzQ/27bPjGr59+7yWmEneUdN9XPQasCig=,tag:oBXsSuV5idB7HqNrNOruwg==,type:str]
+PROXY_URLS_DATACENTER=ENC[AES256_GCM,data:Eec0X65EMsV2PD3Qvn+JjGqYaHtLupn0k99H918vmuRuAinP3rv/pwEoyKHmygazrUExg7U2PUELycyzq3lU6RIGtO+r0pRAn/n0S8RwdoZS,iv:T+bfbvULwSLRVD/hyW7rDN8tLLBf1FQkwCEbpiuBB+0=,tag:W/YHfl5U2yaA7ZOXgAFw+Q==,type:str]
 WEBSHARE_DOWNLOAD_URL=ENC[AES256_GCM,data:1D9VRZ3MCXPQWfiMH8+CLcrxeYnVVcQgZDvt5kltvbSTuSHQ2hHDmZpBkTOMIBJnw4JLZ2JQKHgG4OaYDtsM2VltFPnfwaRgVI9G5PSenR3o4PeQmYO1AqWOmjn19jPxNXRhEXdupP9UT+xQNXoBJsl6RR20XOpMA5AipUHmSjD0UIKXoZLU,iv:uWUkAydac//qrOTPUThuOLKAKXK4xcZmK9qBVFwpqt4=,tag:1vYhukBW9kEuSXCLAiZZmQ==,type:str]
 CIRCUIT_BREAKER_THRESHOLD=
 #ENC[AES256_GCM,data:ZcX/OEbrMfKizIQYq3CYGnvzeTEX7KsmQaz2+Jj1rG5tbTy2aljQBIEkjtiwuo8NsNAD+FhIGRGVfBmKe1CAKME1MuiCbgSG,iv:4BSkeD3jZFawP09qECcqyuiWcDnCNSgbIjBATYhazq4=,tag:Ep1d2Uk700MOlWcLWaQ/ig==,type:comment]
@@ -71,7 +73,7 @@ GEONAMES_USERNAME=ENC[AES256_GCM,data:aSkVdLNrhiF6tlg=,iv:eemFGwDIv3EG/P3lVHGZj9
 CENSUS_API_KEY=ENC[AES256_GCM,data:qqG971573aGq9MiHI2xLlanKKFwjfcNNoMXtm8LNbyh0rMbQN2XukQ==,iv:az2i0ldH75nHGah4DeOxaXmDbVYqmC1c77ptZqFA9BI=,tag:zoDdKj9bR7fgIDo1/dEU2g==,type:str]
 sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxNWNmUzVNUGdWRnE0ZFpF\nM0JQZWZ3UDdEVzlwTmIxakxOZXBkT2x2ZlNrClRtV2M3S2daSGxUZmFDSWQ2Nmh4\neU51QndFcUxlSE00RFovOVJTcDZmUUUKLS0tIDcvL3hRMDRoMWZZSXljNzA3WG5o\nMWFic21MV0krMzlIaldBTVU0ZDdlTE0K7euGQtA+9lHNws+x7TMCArZamm9att96\nL8cXoUDWe5fNI5+M1bXReqVfNwPTwZsV6j/+ZtYKybklIzWz02Ex4A==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_0__map_recipient=age1f5002gj4s78jju45jd28kuejtcfhn5cdujz885fl7z2p9ym68pnsgky87a
-sops_lastmodified=2026-02-28T15:50:46Z
+sops_lastmodified=2026-03-01T13:34:16Z
-sops_mac=ENC[AES256_GCM,data:HiLZTLa+p3mqa4hw+tKOK27F/bsJOy4jmDi8MHToi6S7tRfBA/TzcEzXvXUIkkwAixN73NQHvBVeRnbcEsApVpkaxH1OqnjvvyT+B3YFkTEtxczaKGWlCvbqFZNmXYsFvGR9njaWYWsTQPkRIjrroXrSrhr7uxC8F40v7ByxJKo=,iv:qj2IpzWRIh/mM1HtjjkNbyFuhtORKXslVnf/vdEC9Uw=,tag:fr9CZsL74HxRJLXn9eS0xQ==,type:str]
+sops_mac=ENC[AES256_GCM,data:JLfGLbNTEcI6M/sUA5Zez6cfEUObgnUBmX52560PzBmeLZt0F5Y5QpeojIBqEDMuNB0hp1nnPI59WClLJtQ12VlHo9TkL3x9uCNUG+KneQrn1bTmJpA3cwNkWTzIm4l+TGbJbd4FpKJ9H0v1w+sqoKOgG8DqbtOeVdUfsVspAso=,iv:UqYxooXkEtx+y7fYzl+GFncpkjz8dcP7o9fp+kFf6w4=,tag:/maSb1aZGo+Ia8eGpB7PYw==,type:str]
 sops_unencrypted_suffix=_unencrypted
 sops_version=3.12.1
--- a/.env.prod.sops
+++ b/.env.prod.sops
@@ -39,8 +39,8 @@ ALERT_WEBHOOK_URL=ENC[AES256_GCM,data:4sXQk8zklruC525J279TUUatdDJQ43qweuoPhtpI82
 NTFY_TOKEN=ENC[AES256_GCM,data:YlOxhsRJ8P1y4kk6ugWm41iyRCsM6oAWjvbU9lGcD0A=,iv:JZXOvi3wTOPV9A46c7fMiqbszNCvXkOgh9i/H1hob24=,tag:8xnPimgy7sesOAnxhaXmpg==,type:str]
 SUPERVISOR_GIT_PULL=ENC[AES256_GCM,data:mg==,iv:KgqMVYj12FjOzWxtA1T0r0pqCDJ6MtHzMjE+4W/W+s4=,tag:czFaOqhHG8nqrQ8AZ8QiGw==,type:str]
 #ENC[AES256_GCM,data:hzAZvCWc4RTk290=,iv:RsSI4OpAOQGcFVpfXDZ6t705yWmlO0JEWwWF5uQu9As=,tag:UPqFtA2tXiSa0vzJAv8qXg==,type:comment]
-PROXY_URLS_RESIDENTIAL=ENC[AES256_GCM,data:x/F0toXDc8stsUNxaepCmxq1+WuacqqPtdc+R5mxTwcAzsKxCdwt8KpBZWMvz7ku4tHDGsKD949QAX2ANXP9oCMTgW0=,iv:6G9gE9/v7GaYj8aqVTmMrpw6AcQK9yMSCAohNdAD1Ws=,tag:2Jimr1ldVSfkh8LPEwdN3w==,type:str]
+PROXY_URLS_RESIDENTIAL=ENC[AES256_GCM,data:vxRcXQ/8TUTCtr6hKWBD1zVF47GFSfluIHZ8q0tt8SqQOWDdDe2D7Of6boy/kG3lqlpl7TjqMGJ7fLORcr0klKCykQ==,iv:YjegXXtIXm2qr0a3ZHRHxj3L1JoGZ1iQXkVXQupGQ2E=,tag:kahoHRskXbzplZasWOeiig==,type:str]
-PROXY_URLS_DATACENTER=ENC[AES256_GCM,data:6BfXBYmyHpgZU/kJWpZLf8eH5VowVK1n0r6GzFTNAx/OmyaaS1RZVPC1JPkPBnTwEmo0WHYRW8uiUdkABmH9F5ZqqlsAesyfW7zvU9r7yD+D7w==,iv:3CBn2qCoTueQy8xVcQqZS4E3F0qoFYnNbzTZTpJ1veo=,tag:wC3Ecl4uNTwPiT23ATvRZg==,type:str]
+PROXY_URLS_DATACENTER=ENC[AES256_GCM,data:23TgU6oUeO7J+MFkraALQ5/RO38DZ3ib5oYYJr7Lj3KXQSlRsgwA+bJlweI5gcUpFphnPXvmwFGiuL6AeY8LzAQ3bx46dcZa5w9LfKw2PMFt,iv:AGXwYLqWjT5VmU02qqada3PbdjfC0mLK2sPruO0uru8=,tag:Z2IS/JPOqWX+x0LZYwyArA==,type:str]
 WEBSHARE_DOWNLOAD_URL=ENC[AES256_GCM,data:/N77CFf6tJWCk7HrnBOm2Q1ynx7XoblzfbzJySeCjrxqiu4r+CB90aDkaPahlQKI00DUZih3pcy7WhnjdAwI30G5kJZ3P8H8/R0tP7OBK1wPVbsJq8prQJPFOAWewsS4KWNtSURZPYSCxslcBb7DHLX6ZAjv6A5KFOjRK2N8usR9sIabrCWh,iv:G3Ropu/JGytZK/zKsNGFjjSu3Wt6fvHaAqI9RpUHvlI=,tag:fv6xuS94OR+4xfiyKrYELA==,type:str]
 PROXY_CONCURRENCY=ENC[AES256_GCM,data:vdEZ,iv:+eTNQO+s/SsVDBLg1/+fneMzEEsFkuEFxo/FcVV+mWc=,tag:i/EPwi/jOoWl3xW8H0XMdw==,type:str]
 RECHECK_WINDOW_MINUTES=ENC[AES256_GCM,data:L2s=,iv:fV3mCKmK5fxUmIWRePELBDAPTb8JZqasVIhnAl55kYw=,tag:XL+PO6sblz/7WqHC3dtk1w==,type:str]
@@ -52,13 +52,17 @@ BING_SITE_URL=ENC[AES256_GCM,data:M33VI97DyxH8gRR3ZUXoXg4QrEv5og==,iv:GxZtwfbBVi
 #ENC[AES256_GCM,data:OTUMKNkRW0zrupNppXthwE1oieILhNjM+cjx5hFn69g=,iv:48ID2qtSe9ggD2X+G/iUqp3v2uwEc7fZw8lxHIvVXmk=,tag:okBn0Npk1K9dDOFWA/AB1A==,type:comment]
 GEONAMES_USERNAME=ENC[AES256_GCM,data:UXd/S2TzXPiGmLY=,iv:OMURM5E6SFEsaqroUlH76DEnr7C/ujNk9UQnbWT0hK4=,tag:VsjjS12QDbudiEhdAQ/OCQ==,type:str]
 CENSUS_API_KEY=ENC[AES256_GCM,data:9RbKlxSD17LqIuuNXaOKSgZ8LnFh9Wbze3XHgpctfV/1TqBMZTIedQ==,iv:WwsmR3HLUEcgUpLliGRaUPhGM9vFNPMGXSAQQ6+9UVc=,tag:R4EMNy5MxxvK0UTaCL0umA==,type:str]
 #ENC[AES256_GCM,data:SL402gYB8ngjqkrG03FmaA==,iv:I326cYnOWdFnaUwnSfP+s2p9oCDCnqDzUJuPOzSFJc0=,tag:MBW5AqAaq4hTMmNXq1tXKw==,type:comment]
 R2_LANDING_BUCKET=ENC[AES256_GCM,data:yZXLNQb8yN9nQPdxqmqv61fLWbRYCjjOqQ==,iv:fAwBLC/EuU0lgYOxZSkTagWyeQCdEadjssapxpCEGjA=,tag:VUmuVw76WZAaukp71Desag==,type:str]
 R2_LANDING_ACCESS_KEY_ID=ENC[AES256_GCM,data:Y6y+U1ayhpFDcoaDjl7hyMVjU3gVvtORAH5gbd+HXbM=,iv:ra9kuch1DT+2tfz140bvxQRIXypsdiUrX1QYQ59gNRI=,tag:Wt85qliUMFvgbvoUrOXT7A==,type:str]
 R2_LANDING_SECRET_ACCESS_KEY=ENC[AES256_GCM,data:99wB9aKSq2GihW9FOwBSMgHYzNKBHlol2Mf2kg4Ma6Fr4Cr21t/blzPxNQ7YRdeKk6ypFgViXlS4BJz9nC+v0g==,iv:/AmbXtj/uSGcMp+NBhN5tiVb2U56tvO5e1UpG2/ijPo=,tag:Qg2Tt11DUJPyeYcq9iSVnQ==,type:str]
 sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaUVk0UEVqdmtsM3VzQnpZ\nZjJDZ1lsM0VqWFpVVXUvNzdQcCtHbVJLNjFnCmhna01vTkVBaFQ5ZVlXeGhYNXdH\ncWJ5Qi9PdkxLaHBhQnR3cmtoblkxdEUKLS0tIDhHamY4NXhxOG9YN1NpbTN1aVRh\nOHVKcEN1d0QwQldVTDlBWUU4SDVDWlUKRJU+CTfTzIx6LLKin9sTXAHPVAfiUerZ\nCqYVFncsCJE3TbMI424urQj7kragPoGl1z4++yqAXNTRxfZIY4KTkg==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_0__map_recipient=age1f5002gj4s78jju45jd28kuejtcfhn5cdujz885fl7z2p9ym68pnsgky87a
 sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmVEticFRVemlzZnlzek4x\nbWJ0d0h5ejJVUk5remo1VkdxNjVpdllqbFhFClc1UXlNd09xVVA5MnltMlN5MWRy\nYUlNRmNybHh1RGdPVC9yWlYrVmRTdkkKLS0tIHBUbU9qSDMrVGVHZDZGSFdpWlBh\nT3NXTGl0SmszaU9hRmU5bXI0cDRoRW8KLvbNYsBEwz+ITKvn7Yn+iNHiRzyyjtQt\no9/HupykJ3WjSdleGz7ZN6UiPGelHp0D/rzSASTYaI1+0i0xZ4PUoQ==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_1__map_recipient=age1wjepykv3glvsrtegu25tevg7vyn3ngpl607u3yjc9ucay04s045s796msw
 sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFeHhaOURNZnRVMEwxNThu\nUjF4Q0kwUXhTUE1QSzZJbmpubnh3RnpQTmdvCjRmWWxpNkxFUmVGb3NRbnlydW5O\nWEg3ZXJQTU4vcndzS2pUQXY3Q0ttYjAKLS0tIE9IRFJ1c2ZxbGVHa2xTL0swbGN1\nTzgwMThPUDRFTWhuZHJjZUYxOTZrU00KY62qrNBCUQYxwcLMXFEnLkwncxq3BPJB\nKm4NzeHBU87XmPWVrgrKuf+PH1mxJlBsl7Hev8xBTy7l6feiZjLIvQ==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_2__map_recipient=age1c783ym2q5x9tv7py5d28uc4k44aguudjn03g97l9nzs00dd9tsrqum8h4d
-sops_lastmodified=2026-03-01T00:26:54Z
+sops_lastmodified=2026-03-01T16:31:40Z
-sops_mac=ENC[AES256_GCM,data:DdcABGVm9KbAcFrF0iuZlAaugsouNs7Hon2mZISaHs15/2H/Pd9FniXW3KeQ0+/NdZFQkz/h3i3bVFampcpFS1AxuOE5+1/IgWn8sKtaqPc7E9y8g6lxMnwTkUX2z+n/Q2nR8KAcO9IyE0GNjIluMWkxPWQuLzlRYDOjRN4/1e0=,iv:rm+6lXhYu6VUmrdCIrU0BRN2/ooa21Fw1ESWxr7vATg=,tag:GZmLLZf/LQaNeNNAAEg5bA==,type:str]
+sops_mac=ENC[AES256_GCM,data:+9Sk7wVRPMDeDf6FkuNAOyUT6/OD8Rk6jtJuy5CGQXdxxCYY12F6dAGF6V5fE0toqfYxhVTJbSqH32qTZM2Tc28n36zCtXNnaTdv9rS4XFfPq+MrhuIIv5bJYwDXDgW4F5TCeCBB09jgUKDRaQVGBn2hO3+k8auaPdqWp2cd+es=,iv:wtN61uo7vixY1/EQteyTMzG73C6Gz8AFu1qodR9JvQw=,tag:Z1izDo6EAS03OhA1bj0ArA==,type:str]
 sops_unencrypted_suffix=_unencrypted
 sops_version=3.12.1
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,31 +0,0 @@
 stages:
  - test
  - tag
 test:
  stage: test
  image: python:3.12-slim
  before_script:
    - pip install uv
  script:
    - uv sync
    - uv run pytest web/tests/ -x -q -p no:faulthandler
    - uv run ruff check web/src/ web/tests/
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 tag:
  stage: tag
  image:
    name: alpine/git
    entrypoint: [""]
  script:
    - git tag "v${CI_PIPELINE_IID}"
    - git push "https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "v${CI_PIPELINE_IID}"
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
 # Deployment is handled by the on-server supervisor (src/padelnomics/supervisor.py).
 # It polls git every 60s, fetches tags, and deploys only when a new passing tag exists.
 # No CI secrets needed — zero SSH keys, zero deploy credentials.
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## [Unreleased]
 ### Fixed
 - **Stale-tier failures no longer exhaust the next proxy tier** — with parallel workers, threads that fetched a proxy just before tier escalation reported failures after the tier changed, immediately blowing through the new tier's circuit breaker before it ever got tried (Rayobyte was skipped entirely). `record_failure(proxy_url)` now checks which tier the proxy belongs to and ignores the circuit breaker when the proxy is from an already-escalated tier.
 - **Proxy URL scheme validation in `load_proxy_tiers()`** — URLs in `PROXY_URLS_DATACENTER` / `PROXY_URLS_RESIDENTIAL` that are missing an `http://` or `https://` scheme are now logged as a warning and skipped, rather than being passed through and causing SSL handshake failures or connection errors at request time. Also fixed a missing `http://` prefix in the dev `.env` `PROXY_URLS_DATACENTER` entry.
 ### Changed
 - **Per-proxy dead tracking in tiered cycler** — `make_tiered_cycler` now accepts a `proxy_failure_limit` parameter (default 3). Individual proxies that hit the limit are marked dead and permanently skipped by `next_proxy()`. If all proxies in the active tier are dead, `next_proxy()` auto-escalates to the next tier without needing the tier-level threshold. `record_failure(proxy_url)` and `record_success(proxy_url)` accept an optional `proxy_url` argument for per-proxy tracking; callers without `proxy_url` are fully backward-compatible. New `dead_proxy_count()` callable exposed for monitoring.
  - `extract/padelnomics_extract/src/padelnomics_extract/proxy.py`: added per-proxy state (`proxy_failure_counts`, `dead_proxies`), updated `next_proxy`/`record_failure`/`record_success`, added `dead_proxy_count`
@@ -14,6 +19,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
  - `web/tests/test_supervisor.py`: 11 new tests in `TestTieredCyclerDeadProxyTracking` covering dead proxy skipping, auto-escalation, `dead_proxy_count`, backward compat, and thread safety
 ### Added
 - **Visual upgrades for longform articles** — 4 reusable CSS article components added to `input.css` and applied across 6 cornerstone articles (EN + DE):
  - `article-timeline`: horizontal numbered phase diagram with connecting lines; collapses to vertical stack on mobile. Replaces ASCII art code blocks in build guide articles.
  - `article-callout` (warning/tip/info variants): left-bordered callout box with icon, title, and body. Replaces `>` blockquotes and bold-text warnings in build and risk guides.
  - `article-cards`: 2-column card grid with colored accent bars (success/failure/neutral/established/growth/emerging). Replaces sequential bold-text pattern paragraphs in build, risk, and location guides.
  - `severity` pills: inline colored badge for High/Medium-High/Medium/Low-Medium/Low. Applied to risk overview tables in both risk guide articles.
  - Articles updated: `padel-hall-build-guide-en`, `padel-halle-bauen-de`, `padel-hall-investment-risks-en`, `padel-halle-risiken-de`, `padel-hall-location-guide-en`, `padel-standort-analyse-de`
 - **Pipeline Transform tab + live extraction status** — new "Transform" tab in the pipeline admin with status cards for SQLMesh transform and export-serving tasks, a "Run Full Pipeline" button, and a recent run history table. The Overview tab now auto-polls every 5 s while an extraction task is pending and stops automatically when quiet. Per-extractor "Run" buttons use HTMX in-place updates instead of redirects. The header "Run Pipeline" button now enqueues the full ELT pipeline (extract → transform → export) instead of extraction only. Three new worker task handlers: `run_transform` (sqlmesh plan prod --auto-apply, 2 h timeout), `run_export` (export_serving.py, 10 min timeout), `run_pipeline` (sequential, stops on first failure). Concurrency guard prevents double-enqueuing the same step.
  - `web/src/padelnomics/worker.py`: `handle_run_transform`, `handle_run_export`, `handle_run_pipeline`
  - `web/src/padelnomics/admin/pipeline_routes.py`: `_render_overview_partial()`, `_fetch_pipeline_tasks()`, `_format_duration()`, `pipeline_transform()`, `pipeline_trigger_transform()`; `pipeline_trigger_extract()` now HTMX-aware
  - `web/src/padelnomics/admin/templates/admin/pipeline.html`: pulse animation on `.status-dot.running`, Transform tab button, rewired header button
  - `web/src/padelnomics/admin/templates/admin/partials/pipeline_overview.html`: self-polling wrapper, HTMX Run buttons
  - `web/src/padelnomics/admin/templates/admin/partials/pipeline_transform.html`: new file
 - **Affiliate programs management** — centralised retailer config (`affiliate_programs` table) with URL template + tracking tag + commission %. Products now use a program dropdown + product identifier (e.g. ASIN) instead of manually baking full URLs. URL is assembled at redirect time via `build_affiliate_url()`, so changing a tag propagates instantly to all products. Legacy products (baked `affiliate_url`) continue to work via fallback. Amazon OneLink configured in the Associates dashboard handles geo-redirect to local marketplaces — no per-country programs needed.
  - `web/src/padelnomics/migrations/versions/0027_affiliate_programs.py`: `affiliate_programs` table, nullable `program_id` + `product_identifier` columns on `affiliate_products`, seeds "Amazon" program, backfills ASINs from existing URLs
  - `web/src/padelnomics/affiliate.py`: `get_all_programs()`, `get_program()`, `get_program_by_slug()`, `build_affiliate_url()`; `get_product()` JOINs program for redirect assembly; `_parse_product()` extracts `_program` sub-dict
--- a/data/content/articles/padel-hall-build-guide-en.md
+++ b/data/content/articles/padel-hall-build-guide-en.md
@@ -17,15 +17,48 @@ This guide walks through all five phases and 23 steps between your initial marke
 ## The 5 Phases at a Glance
-```
+<div class="article-timeline">
-Phase 1          Phase 2          Phase 3          Phase 4         Phase 5
+  <div class="article-timeline__phase">
-Feasibility   →  Planning &    →  Construction  →  Pre-         →  Operations &
+    <div class="article-timeline__num">1</div>
-& Concept        Design           / Conversion      Opening         Optimization
+    <div class="article-timeline__card">
-
+      <div class="article-timeline__title">Feasibility &amp; Concept</div>
-Month 1–3        Month 3–6        Month 6–12        Month 10–13     Ongoing
+      <div class="article-timeline__subtitle">Market research, concept, site scouting</div>
-
+      <div class="article-timeline__meta">Month 1–3 · Steps 1–5</div>
-Steps 1–5        Steps 6–11       Steps 12–16       Steps 17–20     Steps 21–23
+    </div>
-```
+  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">2</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Planning &amp; Design</div>
      <div class="article-timeline__subtitle">Architect, permits, financing</div>
      <div class="article-timeline__meta">Month 3–6 · Steps 6–11</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">3</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Construction</div>
      <div class="article-timeline__subtitle">Build, courts, IT systems</div>
      <div class="article-timeline__meta">Month 6–12 · Steps 12–16</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">4</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Pre-Opening</div>
      <div class="article-timeline__subtitle">Hiring, marketing, soft launch</div>
      <div class="article-timeline__meta">Month 10–13 · Steps 17–20</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">5</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Operations</div>
      <div class="article-timeline__subtitle">Revenue streams, optimization</div>
      <div class="article-timeline__meta">Ongoing · Steps 21–23</div>
    </div>
  </div>
 </div>
 ---
@@ -105,7 +138,12 @@ Deliverables from this phase:
 - **MEP design (mechanical, electrical, plumbing):** Heating, ventilation, air conditioning, electrical, drainage — typically the most expensive trade package in a sports hall conversion
 - **Fire safety strategy**
-> **The most expensive planning mistake in padel hall builds:** underestimating HVAC complexity and budget. Large indoor courts need precise temperature and humidity control — not just for player comfort, but for playing surface longevity and air quality. Courts installed in a poorly climate-controlled building will degrade faster and generate complaints. Budget for it properly from the start, not as a value-engineering target.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">The most expensive planning mistake in padel hall builds</span>
    <p>Underestimating HVAC complexity and budget. Large indoor courts need precise temperature and humidity control — not just for player comfort, but for playing surface longevity and air quality. Courts installed in a poorly climate-controlled building will degrade faster and generate complaints. Budget for it properly from the start, not as a value-engineering target.</p>
  </div>
 </div>
 ### Step 8: Court Supplier Selection
@@ -160,7 +198,12 @@ Courts are installed after the building envelope is weathertight. This is a hard
 Glass panels, artificial turf, and court metalwork must not be exposed to construction dust, moisture, and site traffic. Projects that try to accelerate schedules by installing courts before the building is properly enclosed regularly end up with surface contamination, glass damage, and voided manufacturer warranties.
-> **The most common construction mistake on padel hall projects:** rushing court installation sequencing under schedule pressure. The pressure to hit an opening date is real — but installing courts into an unenclosed building is one of the most reliable ways to add cost and delay, not reduce them. Hold the sequence.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">The most common construction mistake on padel hall projects</span>
    <p>Rushing court installation sequencing under schedule pressure. The pressure to hit an opening date is real — but installing courts into an unenclosed building is one of the most reliable ways to add cost and delay, not reduce them. Hold the sequence.</p>
  </div>
 </div>
 Allow two to four weeks for court installation per batch, depending on the manufacturer's crew capacity. Build this explicitly into your master program.
@@ -174,7 +217,12 @@ Decide early: which booking platform, which point-of-sale system, and whether yo
 Access control systems must be coordinated with the electrical design. Adding them in the final stages of construction is possible but costs more.
-> **The most common pre-opening mistake:** the booking system isn't fully configured, tested, and working on day one. A broken booking flow, failed test payments, or a QR code that leads to an error page on opening day kills your launch momentum in a way that's difficult to recover from. Test the system end-to-end — including real bookings, real payments, and real cancellations — two to four weeks before opening.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">The most common pre-opening mistake</span>
    <p>The booking system isn't fully configured, tested, and working on day one. A broken booking flow, failed test payments, or a QR code that leads to an error page on opening day kills your launch momentum in a way that's difficult to recover from. Test the system end-to-end — including real bookings, real payments, and real cancellations — two to four weeks before opening.</p>
  </div>
 </div>
 ### Step 16: Inspections and Certifications
@@ -248,13 +296,36 @@ Court bookings are your core revenue, but rarely your only opportunity:
 Patterns emerge when you observe padel hall projects across a market over time.
-**Projects that go over budget** almost always cut at the wrong place early — too little HVAC budget, no construction contingency, a cheap general contractor without adequate contractual protection. The savings on the way in become much larger costs on the way out.
+<div class="article-cards">
-
+  <div class="article-card article-card--failure">
-**Projects that slip their schedule** consistently underestimate the regulatory process. Permits, noise assessments, and change-of-use applications take time that money cannot buy once you've started too late. Start conversations with authorities before you need the approvals, not when you need them.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**Projects that open weakly** started marketing too late and tested the booking system too late. An empty calendar on day one and a broken booking page create impressions that stick longer than the opening week.
+      <span class="article-card__title">Projects that go over budget</span>
-
+      <p class="article-card__body">Almost always cut at the wrong place early — too little HVAC budget, no construction contingency, a cheap general contractor without adequate contractual protection. The savings on the way in become much larger costs on the way out.</p>
-**Projects that succeed long-term** treat all three phases — planning, build, and opening — with equal rigor, and invest early and consistently in community and repeat customers.
+    </div>
  </div>
  <div class="article-card article-card--failure">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projects that slip their schedule</span>
      <p class="article-card__body">Consistently underestimate the regulatory process. Permits, noise assessments, and change-of-use applications take time that money cannot buy once you've started too late. Start conversations with authorities before you need the approvals.</p>
    </div>
  </div>
  <div class="article-card article-card--failure">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projects that open weakly</span>
      <p class="article-card__body">Started marketing too late and tested the booking system too late. An empty calendar on day one and a broken booking page create impressions that stick longer than the opening week.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projects that succeed long-term</span>
      <p class="article-card__body">Treat all three phases — planning, build, and opening — with equal rigor, and invest early and consistently in community and repeat customers.</p>
    </div>
  </div>
 </div>
 Building a padel hall is complex, but it is a solved problem. The failures are nearly always the same failures. So are the successes.
--- a/data/content/articles/padel-hall-investment-risks-en.md
+++ b/data/content/articles/padel-hall-investment-risks-en.md
@@ -21,20 +21,20 @@ This article covers the 14 risks that don't get enough airtime in investor discu
 | # | Risk | Category | Severity |
 |---|------|----------|----------|
-| 1 | Trend / fad risk | Strategic | High |
+| 1 | Trend / fad risk | Strategic | <span class="severity severity--high">High</span> |
-| 2 | Construction cost overruns | Construction & Development | High |
+| 2 | Construction cost overruns | Construction & Development | <span class="severity severity--high">High</span> |
-| 3 | Construction delays | Construction & Development | High |
+| 3 | Construction delays | Construction & Development | <span class="severity severity--high">High</span> |
-| 4 | Landlord risk: sale, insolvency, non-renewal | Property & Lease | High |
+| 4 | Landlord risk: sale, insolvency, non-renewal | Property & Lease | <span class="severity severity--high">High</span> |
-| 5 | New competitor in your catchment | Competition | Medium–High |
+| 5 | New competitor in your catchment | Competition | <span class="severity severity--medium-high">Medium–High</span> |
-| 6 | Key-person dependency | Operations | Medium |
+| 6 | Key-person dependency | Operations | <span class="severity severity--medium">Medium</span> |
-| 7 | Staff retention and wage pressure | Operations | Medium |
+| 7 | Staff retention and wage pressure | Operations | <span class="severity severity--medium">Medium</span> |
-| 8 | Court surface and maintenance cycles | Operations | Medium |
+| 8 | Court surface and maintenance cycles | Operations | <span class="severity severity--medium">Medium</span> |
-| 9 | Energy price volatility | Financial | Medium |
+| 9 | Energy price volatility | Financial | <span class="severity severity--medium">Medium</span> |
-| 10 | Interest rate risk | Financial | Medium |
+| 10 | Interest rate risk | Financial | <span class="severity severity--medium">Medium</span> |
-| 11 | Personal guarantee exposure | Financial | High |
+| 11 | Personal guarantee exposure | Financial | <span class="severity severity--high">High</span> |
-| 12 | Customer concentration | Financial | Medium |
+| 12 | Customer concentration | Financial | <span class="severity severity--medium">Medium</span> |
-| 13 | Noise complaints and regulatory restrictions | Regulatory & Legal | Medium |
+| 13 | Noise complaints and regulatory restrictions | Regulatory & Legal | <span class="severity severity--medium">Medium</span> |
-| 14 | Booking platform dependency | Regulatory & Legal | Low–Medium |
+| 14 | Booking platform dependency | Regulatory & Legal | <span class="severity severity--low-medium">Low–Medium</span> |
 ---
@@ -137,9 +137,12 @@ Your costs will increase three to five percent per year. Whether you can pass th
 ## The Risk No One Talks About: Personal Guarantees
-**This section gets skipped in almost every padel hall investment conversation. That's a serious mistake.**
+<div class="article-callout article-callout--warning">
-
+  <div class="article-callout__body">
-Banks financing a single-asset leisure facility without corporate backing will almost universally require personal guarantees from the principal shareholders. Not as an unusual request — as standard terms for this type of deal.
+    <span class="article-callout__title">This section gets skipped in almost every padel hall investment conversation. That's a serious mistake.</span>
    <p>Banks financing a single-asset leisure facility without corporate backing will almost universally require personal guarantees from the principal shareholders. Not as an unusual request — as standard terms for this type of deal.</p>
  </div>
 </div>
 Here is what that means in practice:
@@ -180,13 +183,36 @@ Building a parallel booking capability — even a simple direct booking option
 The investors who succeed long-term in padel aren't the ones who found a risk-free opportunity. There isn't one. They're the ones who went in with their eyes open.
-**They modeled the bad scenarios before assuming the good ones.** A business plan that shows only the base case isn't a planning tool — it's wishful thinking. Explicit downside modeling — 40% utilization, six-month delay, new competitor in year three — is the baseline, not an optional exercise.
+<div class="article-cards">
-
+  <div class="article-card article-card--success">
-**They built structural buffers into the plan.** Liquid reserves covering at least six months of fixed costs. Construction contingency treated as a budget line, not a hedge. These aren't comfort margins; they're operational requirements.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**They got the contractual foundations right from the start.** Lease terms. Financing conditions. Guarantee scope. The cost of good legal and financial advice at the planning stage is trivial relative to the downside exposure it addresses.
+      <span class="article-card__title">Model the bad scenarios first</span>
-
+      <p class="article-card__body">A business plan showing only the base case isn't a planning tool — it's wishful thinking. Explicit downside modeling — 40% utilization, six-month delay, new competitor in year three — is the baseline, not an optional exercise.</p>
-**They planned for competition.** Not by hoping it wouldn't come, but by building a product — community, quality, service — that gives existing customers a reason to stay when someone cheaper opens nearby.
+    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Build structural buffers in</span>
      <p class="article-card__body">Liquid reserves covering at least six months of fixed costs. Construction contingency treated as a budget line, not a hedge. These aren't comfort margins; they're operational requirements.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Get the contractual foundations right</span>
      <p class="article-card__body">Lease terms. Financing conditions. Guarantee scope. The cost of good legal and financial advice at the planning stage is trivial relative to the downside exposure it addresses.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Plan for competition</span>
      <p class="article-card__body">Not by hoping it won't come, but by building a product — community, quality, service — that gives existing customers a reason to stay when someone cheaper opens nearby.</p>
    </div>
  </div>
 </div>
 ---
--- a/data/content/articles/padel-hall-location-guide-en.md
+++ b/data/content/articles/padel-hall-location-guide-en.md
@@ -148,11 +148,29 @@ The matrix also reveals where trade-offs are being made explicitly, which makes
 The 8 criteria above evaluate specific sites. But before shortlisting sites, it is worth stepping back to read the stage of the overall market — because the right operational strategy differs fundamentally depending on where a city sits in its padel development cycle.
-**Established markets**: Booking platforms show consistent peak-hour sell-out across most venues. Waiting lists are common. Demand is validated beyond doubt. The challenge here is elevated rent, elevated build costs, and entrenched operators who have already captured community loyalty. New entrants need a genuine differentiation angle — a superior facility specification, a better location within the city, or an F&B and coaching product that existing venues don't offer. Entry costs are high; returns, if execution is strong, are also high. Munich is the canonical German example.
+<div class="article-cards">
-
+  <div class="article-card article-card--established">
-**Growth markets**: Demand is clearly building — booking availability tightens at weekends, new facilities are announced regularly, and the sport is gaining local media visibility. Supply hasn't caught up, so identifiable gaps still exist in specific districts or the surrounding hinterland. The risk profile is lower than in emerging markets, but the window for securing good real estate at reasonable rent is narrowing. The premium for moving decisively goes to those who arrive before the obvious sites are taken.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**Emerging markets**: Limited current supply, a small but growing player base, and padel not yet mainstream enough to generate organic walk-in demand. Entry costs — rent especially — are lower. The constraint is that demand must be actively created rather than captured. Operators who succeed here invest in community: beginner programmes, local leagues, school partnerships, conversions from tennis clubs. The time to first profitability is longer, but the competitive position built in the first two years is often decisive for the long term.
+      <span class="article-card__title">Established markets</span>
      <p class="article-card__body">Booking platforms show consistent peak-hour sell-out. Demand is validated. The challenge: elevated rent, high build costs, entrenched operators. New entrants need a genuine differentiation angle — superior spec, better location, or F&B and coaching that existing venues don't offer. Entry costs are high; returns, if execution is strong, are also high. Munich is the canonical German example.</p>
    </div>
  </div>
  <div class="article-card article-card--growth">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Growth markets</span>
      <p class="article-card__body">Demand is clearly building — booking availability tightens at weekends, new facilities are announced regularly. Supply hasn't caught up; identifiable gaps still exist. The risk profile is lower, but the window for securing good real estate at reasonable rent is narrowing. The premium goes to those who arrive before the obvious sites are taken.</p>
    </div>
  </div>
  <div class="article-card article-card--emerging">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Emerging markets</span>
      <p class="article-card__body">Limited supply, a small but growing player base, padel not yet mainstream. Entry costs — rent especially — are lower. The constraint: demand must be actively created rather than captured. Operators who succeed invest in community: beginner programmes, local leagues, school partnerships. Time to profitability is longer, but the competitive position built in the first two years is often decisive.</p>
    </div>
  </div>
 </div>
 Before committing to a site search in any city, calibrate where it sits on this spectrum. The 8-criteria framework then tells you whether a specific site works; market maturity tells you what kind of operator and strategy is required to make it work at all.
--- a/data/content/articles/padel-halle-bauen-de.md
+++ b/data/content/articles/padel-halle-bauen-de.md
@@ -17,15 +17,48 @@ Dieser Leitfaden zeigt Ihnen alle 5 Phasen und 23 Schritte, die zwischen Ihrer e
 ## Die 5 Phasen im Überblick
-```
+<div class="article-timeline">
-Phase 1          Phase 2          Phase 3          Phase 4         Phase 5
+  <div class="article-timeline__phase">
-Machbarkeit   →  Planung &     →  Bau /         →  Voreröff-   →  Betrieb &
+    <div class="article-timeline__num">1</div>
-& Konzept        Design           Umbau             nung            Optimierung
+    <div class="article-timeline__card">
-
+      <div class="article-timeline__title">Machbarkeit &amp; Konzept</div>
-Monat 1–3        Monat 3–6        Monat 6–12        Monat 10–13     laufend
+      <div class="article-timeline__subtitle">Marktanalyse, Konzept, Standortsuche</div>
-
+      <div class="article-timeline__meta">Monat 1–3 · Schritte 1–5</div>
-Schritte 1–5     Schritte 6–11    Schritte 12–16    Schritte 17–20  Schritte 21–23
+    </div>
-```
+  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">2</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Planung &amp; Design</div>
      <div class="article-timeline__subtitle">Architekt, Genehmigungen, Finanzierung</div>
      <div class="article-timeline__meta">Monat 3–6 · Schritte 6–11</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">3</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Bau / Umbau</div>
      <div class="article-timeline__subtitle">Rohbau, Courts, IT-Systeme</div>
      <div class="article-timeline__meta">Monat 6–12 · Schritte 12–16</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">4</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Voreröffnung</div>
      <div class="article-timeline__subtitle">Personal, Marketing, Soft Launch</div>
      <div class="article-timeline__meta">Monat 10–13 · Schritte 17–20</div>
    </div>
  </div>
  <div class="article-timeline__phase">
    <div class="article-timeline__num">5</div>
    <div class="article-timeline__card">
      <div class="article-timeline__title">Betrieb &amp; Optimierung</div>
      <div class="article-timeline__subtitle">Einnahmen, Community, Optimierung</div>
      <div class="article-timeline__meta">laufend · Schritte 21–23</div>
    </div>
  </div>
 </div>
 ---
@@ -104,7 +137,12 @@ Was in dieser Phase entsteht:
 - MEP-Planung (Haustechnik): Heizung, Lüftung, Klimaanlage, Elektro, Sanitär — das sind bei Sporthallen oft die kostenintensivsten Gewerke
 - Brandschutzkonzept
-**Häufiger Fehler in dieser Phase:** Die Haustechnik wird unterschätzt. Eine große Innenhalle braucht präzise Temperatur- und Feuchtigkeitskontrolle — für die Spielqualität, für die Langlebigkeit des Belags und für das Wohlbefinden der Spieler. Eine schlechte HVAC-Anlage ist eine Dauerbaustelle.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">Häufiger Fehler in dieser Phase</span>
    <p>Die Haustechnik wird unterschätzt. Eine große Innenhalle braucht präzise Temperatur- und Feuchtigkeitskontrolle — für die Spielqualität, für die Langlebigkeit des Belags und für das Wohlbefinden der Spieler. Eine schlechte HVAC-Anlage ist eine Dauerbaustelle.</p>
  </div>
 </div>
 ### Schritt 8: Courtlieferant auswählen
@@ -155,7 +193,12 @@ Verhandeln Sie Festpreise, wo möglich. Lesen Sie die Risikoverteilung in den Ve
 Courts werden nach Fertigstellung der Gebäudehülle montiert — das ist eine harte Reihenfolge, keine Empfehlung. Glaselemente dürfen nicht Feuchtigkeit, Staub und Baustellenverkehr ausgesetzt werden, bevor das Gebäude dicht ist.
-**Ein häufiger und vermeidbarer Fehler:** Projekte, die unter Zeitdruck stehen, versuchen, Court-Montage vorzuziehen. Das Ergebnis sind beschädigte Oberflächen, Glasschäden, Verschmutzungen im Belag und Gewährleistungsprobleme mit dem Hersteller. Halten Sie die Reihenfolge ein — konsequent.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">Ein häufiger und vermeidbarer Fehler</span>
    <p>Projekte unter Zeitdruck versuchen, die Court-Montage vorzuziehen. Das Ergebnis sind beschädigte Oberflächen, Glasschäden, Verschmutzungen im Belag und Gewährleistungsprobleme mit dem Hersteller. Halten Sie die Reihenfolge ein — konsequent.</p>
  </div>
 </div>
 Die Montage von Courts dauert je nach Hersteller und Parallelkapazität zwei bis vier Wochen pro Charge. Planen Sie das in den Gesamtablauf ein.
@@ -169,7 +212,12 @@ Frühzeitig entscheiden: Playtomic, Matchi, ein anderes System oder eine Hybridl
 Zugangskontrolle (falls gewünscht) muss mit der Elektroplanung koordiniert werden. Wer das in der letzten Bauphase ergänzen möchte, zahlt dafür.
-**Der häufigste Fehler kurz vor der Eröffnung:** Am Tag der Eröffnung ist das Buchungssystem noch nicht richtig konfiguriert, Testzahlungen schlagen fehl, der QR-Code am Eingang führt auf eine Fehlerseite. Der Eröffnungsbuzz ist ein einmaliges Gut. Testen Sie das System zwei bis vier Wochen vorher vollständig — inklusive echter Buchungen, echter Zahlungen und echter Stornierungen.
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">Der häufigste Fehler kurz vor der Eröffnung</span>
    <p>Am Tag der Eröffnung ist das Buchungssystem noch nicht richtig konfiguriert, Testzahlungen schlagen fehl, der QR-Code am Eingang führt auf eine Fehlerseite. Der Eröffnungsbuzz ist ein einmaliges Gut. Testen Sie das System zwei bis vier Wochen vorher vollständig — inklusive echter Buchungen, echter Zahlungen und echter Stornierungen.</p>
  </div>
 </div>
 ### Schritt 16: Abnahmen und Zertifizierungen
@@ -243,13 +291,36 @@ Die Court-Buchung ist Ihr Kernangebot — aber nicht die einzige Einnahmequelle:
 Wer Dutzende Padelhallenprojekte in Europa beobachtet, sieht Muster auf beiden Seiten:
-**Die Projekte, die über Budget laufen**, haben fast immer früh an der falschen Stelle gespart — zu wenig Haustechnikbudget, kein Baukostenpuffer, zu günstiger Generalunternehmer ohne ausreichende Vertragsabsicherung.
+<div class="article-cards">
-
+  <div class="article-card article-card--failure">
-**Die Projekte, die terminlich entgleisen**, haben die behördlichen Prozesse unterschätzt. Genehmigungen, Lärmschutzgutachten, Nutzungsänderungen brauchen Zeit — und diese Zeit lässt sich nicht kaufen, sobald man zu spät damit anfängt.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**Die Projekte, die schwach starten**, haben das Marketing zu spät begonnen und das Buchungssystem zu spät getestet. Ein leerer Kalender am Eröffnungstag und eine kaputte Buchungsseite erzeugen Eindrücke, die sich festsetzen.
+      <span class="article-card__title">Projekte, die über Budget laufen</span>
-
+      <p class="article-card__body">Haben fast immer früh an der falschen Stelle gespart — zu wenig Haustechnikbudget, kein Baukostenpuffer, zu günstiger Generalunternehmer ohne ausreichende Vertragsabsicherung.</p>
-**Die Projekte, die langfristig erfolgreich sind**, haben alle drei Phasen — Planung, Bau, Eröffnung — mit derselben Sorgfalt behandelt und früh in Community und Stammkundschaft investiert.
+    </div>
  </div>
  <div class="article-card article-card--failure">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projekte, die terminlich entgleisen</span>
      <p class="article-card__body">Haben die behördlichen Prozesse unterschätzt. Genehmigungen, Lärmschutzgutachten, Nutzungsänderungen brauchen Zeit — und diese Zeit lässt sich nicht kaufen, sobald man zu spät damit anfängt.</p>
    </div>
  </div>
  <div class="article-card article-card--failure">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projekte, die schwach starten</span>
      <p class="article-card__body">Haben das Marketing zu spät begonnen und das Buchungssystem zu spät getestet. Ein leerer Kalender am Eröffnungstag und eine kaputte Buchungsseite erzeugen Eindrücke, die sich festsetzen.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Projekte, die langfristig erfolgreich sind</span>
      <p class="article-card__body">Behandeln alle drei Phasen — Planung, Bau, Eröffnung — mit derselben Sorgfalt und investieren früh in Community und Stammkundschaft.</p>
    </div>
  </div>
 </div>
 Eine Padelhalle zu bauen ist komplex — aber kein ungelöstes Problem. Die Fehler, die Projekte scheitern lassen, sind fast immer dieselben. Genauso wie die Entscheidungen, die sie gelingen lassen.
--- a/data/content/articles/padel-halle-risiken-de.md
+++ b/data/content/articles/padel-halle-risiken-de.md
@@ -21,20 +21,20 @@ Dieser Artikel zeigt Ihnen die 14 Risiken, über die in Investorenrunden zu weni
 | # | Risiko | Kategorie | Schwere |
 |---|--------|-----------|---------|
-| 1 | Trend-/Modeerscheinung | Strategisch | Hoch |
+| 1 | Trend-/Modeerscheinung | Strategisch | <span class="severity severity--high">Hoch</span> |
-| 2 | Baukostenüberschreitungen | Bau & Entwicklung | Hoch |
+| 2 | Baukostenüberschreitungen | Bau & Entwicklung | <span class="severity severity--high">Hoch</span> |
-| 3 | Verzögerungen während des Baus | Bau & Entwicklung | Hoch |
+| 3 | Verzögerungen während des Baus | Bau & Entwicklung | <span class="severity severity--high">Hoch</span> |
-| 4 | Vermieterproblem: Verkauf, Insolvenz, keine Verlängerung | Immobilie & Mietvertrag | Hoch |
+| 4 | Vermieterproblem: Verkauf, Insolvenz, keine Verlängerung | Immobilie & Mietvertrag | <span class="severity severity--high">Hoch</span> |
-| 5 | Neue Konkurrenz im Einzugsgebiet | Wettbewerb | Mittel–Hoch |
+| 5 | Neue Konkurrenz im Einzugsgebiet | Wettbewerb | <span class="severity severity--medium-high">Mittel–Hoch</span> |
-| 6 | Schlüsselpersonen-Abhängigkeit | Betrieb | Mittel |
+| 6 | Schlüsselpersonen-Abhängigkeit | Betrieb | <span class="severity severity--medium">Mittel</span> |
-| 7 | Fachkräftemangel und Lohndruck | Betrieb | Mittel |
+| 7 | Fachkräftemangel und Lohndruck | Betrieb | <span class="severity severity--medium">Mittel</span> |
-| 8 | Instandhaltungszyklen für Belag, Glas, Kunstrasen | Betrieb | Mittel |
+| 8 | Instandhaltungszyklen für Belag, Glas, Kunstrasen | Betrieb | <span class="severity severity--medium">Mittel</span> |
-| 9 | Energiepreisvolatilität | Finanzen | Mittel |
+| 9 | Energiepreisvolatilität | Finanzen | <span class="severity severity--medium">Mittel</span> |
-| 10 | Zinsänderungsrisiko | Finanzen | Mittel |
+| 10 | Zinsänderungsrisiko | Finanzen | <span class="severity severity--medium">Mittel</span> |
-| 11 | Persönliche Bürgschaft | Finanzen | Hoch |
+| 11 | Persönliche Bürgschaft | Finanzen | <span class="severity severity--high">Hoch</span> |
-| 12 | Kundenkonzentration | Finanzen | Mittel |
+| 12 | Kundenkonzentration | Finanzen | <span class="severity severity--medium">Mittel</span> |
-| 13 | Lärmbeschwerden und behördliche Auflagen | Regulatorisch & Rechtlich | Mittel |
+| 13 | Lärmbeschwerden und behördliche Auflagen | Regulatorisch & Rechtlich | <span class="severity severity--medium">Mittel</span> |
-| 14 | Buchungsplattform-Abhängigkeit | Regulatorisch & Rechtlich | Niedrig–Mittel |
+| 14 | Buchungsplattform-Abhängigkeit | Regulatorisch & Rechtlich | <span class="severity severity--low-medium">Niedrig–Mittel</span> |
 ---
@@ -133,9 +133,14 @@ Ihre Kosten steigen jedes Jahr um drei bis fünf Prozent. Können Sie diese Stei
 ## Sonderbox: Persönliche Bürgschaft — das unterschätzte Risiko Nr. 1
-**Dieses Thema wird in fast jedem Gespräch über Padelhallen-Investitionen ausgelassen. Das ist ein Fehler.**
+<div class="article-callout article-callout--warning">
  <div class="article-callout__body">
    <span class="article-callout__title">Dieses Thema wird in fast jedem Gespräch über Padelhallen-Investitionen ausgelassen. Das ist ein Fehler.</span>
    <p>Banken, die einer Einzelanlage ohne Konzernrückhalt Kapital bereitstellen, verlangen in der Praxis fast immer eine persönliche Bürgschaft des oder der Hauptgesellschafter.</p>
  </div>
 </div>
-Banken, die einer Einzelanlage ohne Konzernrückhalt Kapital bereitstellen, verlangen in der Praxis fast immer eine persönliche Bürgschaft des oder der Hauptgesellschafter. Das bedeutet: Wenn das Unternehmen in Zahlungsschwierigkeiten gerät, haftet nicht die GmbH allein — Sie haften persönlich. Mit dem Eigenheim. Mit dem Ersparten. Mit dem Depot.
+Das bedeutet: Wenn das Unternehmen in Zahlungsschwierigkeiten gerät, haftet nicht die GmbH allein — Sie haften persönlich. Mit dem Eigenheim. Mit dem Ersparten. Mit dem Depot.
 Die Struktur sieht dann typischerweise so aus:
@@ -176,13 +181,36 @@ Mittel- bis langfristig sollten Sie eine eigene Buchungsfähigkeit aufbauen —
 Niemand kann alle Risiken eliminieren. Aber die Investoren, die langfristig erfolgreich sind, tun Folgendes:
-**Sie rechnen mit den schlechten Szenarien, bevor sie das Gute annehmen.** Ein Businessplan, der nur das Base-Case zeigt, ist kein Werkzeug — er ist Wunschdenken. Rechnen Sie explizit durch: Was passiert bei 40 Prozent Auslastung? Bei einem Bauverzug von sechs Monaten? Bei einem neuen Wettbewerber in Jahr drei?
+<div class="article-cards">
-
+  <div class="article-card article-card--success">
-**Sie bauen Puffer ein, nicht als Komfortpolster, sondern als betriebliche Notwendigkeit.** Liquide Reserven von mindestens sechs Monaten Fixkosten sind kein Luxus.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**Sie sichern Mietverträge und Finanzierungskonditionen von Anfang an sorgfältig ab.** Die Kosten für gute Rechts- und Finanzberatung sind verglichen mit dem Downside verschwindend gering.
+      <span class="article-card__title">Schlechte Szenarien zuerst durchrechnen</span>
-
+      <p class="article-card__body">Ein Businessplan, der nur das Base-Case zeigt, ist kein Werkzeug — er ist Wunschdenken. Was passiert bei 40 Prozent Auslastung? Bei sechs Monaten Bauverzug? Bei einem neuen Wettbewerber in Jahr drei?</p>
-**Sie planen für Wettbewerb.** Nicht indem sie auf keine Konkurrenz hoffen, sondern indem sie ein Produkt aufbauen, das Stammkunden bindet — durch Qualität, Community und Dienstleistung.
+    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Puffer als betriebliche Notwendigkeit</span>
      <p class="article-card__body">Liquide Reserven von mindestens sechs Monaten Fixkosten sind kein Luxus, sondern Pflicht. Baukostenpuffer ist eine Budgetlinie — kein optionales Polster.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Verträge von Anfang an absichern</span>
      <p class="article-card__body">Mietvertrag, Finanzierungskonditionen, Bürgschaftsumfang. Die Kosten für gute Rechts- und Finanzberatung in der Planungsphase sind verglichen mit dem Downside verschwindend gering.</p>
    </div>
  </div>
  <div class="article-card article-card--success">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Für Wettbewerb planen</span>
      <p class="article-card__body">Nicht indem man auf keine Konkurrenz hofft, sondern indem man ein Produkt aufbaut, das Stammkunden bindet — durch Qualität, Community und Dienstleistungsqualität.</p>
    </div>
  </div>
 </div>
 ---
--- a/data/content/articles/padel-standort-analyse-de.md
+++ b/data/content/articles/padel-standort-analyse-de.md
@@ -138,11 +138,29 @@ Das Ergebnis ist ein Gesamtscore pro Standort, der einen strukturierten Vergleic
 Die acht Kriterien oben bewerten konkrete Objekte. Bevor Sie aber mit der Objektsuche beginnen, lohnt ein Schritt zurück: In welcher Entwicklungsphase befindet sich der Markt in Ihrer Zielstadt? Die Antwort bestimmt, welche Betreiberstrategie überhaupt Aussicht auf Erfolg hat.
-**Etablierte Märkte**: Buchungsplattformen zeigen durchgehende Vollauslastung zu Stoßzeiten, Wartelisten sind verbreitet, und die Nachfrage ist über jeden Zweifel hinaus belegt. Die Herausforderung liegt nicht mehr in der Nachfrage — sie liegt im Wettbewerb. Etablierte Betreiber haben Markenloyalität aufgebaut, günstige Flächen sind längst vergeben, und Bau- sowie Mietkosten spiegeln die Nachfragesituation wider. Wer in einem solchen Markt neu eintritt, braucht einen echten Differenzierungsansatz: eine bessere Standortlage innerhalb der Stadt, ein überlegenes Hallenprofil oder ein Gastronomie- und Coaching-Angebot, das die bestehenden Anlagen nicht haben. Das Eintrittsinvestment ist hoch — das Ertragspotenzial bei konsequenter Umsetzung aber auch. München ist das paradigmatische Beispiel für Deutschland.
+<div class="article-cards">
-
+  <div class="article-card article-card--established">
-**Wachstumsmärkte**: Die Nachfrage wächst sichtbar — Buchungszeiten füllen sich an Wochenenden, neue Anlagen werden regelmäßig eröffnet, und der Sport erreicht lokale Medienöffentlichkeit. Das Angebot hat die Nachfrage noch nicht vollständig eingeholt; in bestimmten Stadtteilen oder im Umland sind Versorgungslücken erkennbar. Das Risikoprofil ist geringer als in Frühmärkten, aber das Fenster für attraktive Flächen zu vertretbaren Konditionen schließt sich. Wer wartet, bis der Markt offensichtlich attraktiv ist, zahlt für dieses Wissen einen Aufpreis — in Form höherer Mieten, weniger Auswahl und mehr Konkurrenz beim Eintritt.
+    <div class="article-card__accent"></div>
-
+    <div class="article-card__inner">
-**Frühmärkte**: Geringes aktuelles Angebot, eine kleine aber wachsende Spielerbasis und ein noch nicht hinreichend bekannter Sport — die Rahmenbedingungen für günstigen Markteintritt sind vorhanden, aber Nachfrage muss aktiv aufgebaut werden, nicht abgeschöpft. Mietkosten sind niedriger, Standortauswahl größer. Der limitierende Faktor ist Geduld und Marketingfähigkeit: Anfängerkurse, Vereinskooperationen, lokale Ligen und die Konversion bestehender Tennisclubs sind die Instrumente, mit denen Betreiber in Frühmärkten Community und damit Auslastung aufbauen. Der Weg zur ersten Profitabilität ist länger — aber die Wettbewerbsposition, die in den ersten zwei Betriebsjahren aufgebaut wird, erweist sich oft als strukturell dauerhaft.
+      <span class="article-card__title">Etablierte Märkte</span>
      <p class="article-card__body">Buchungsplattformen zeigen durchgehende Vollauslastung zu Stoßzeiten, Wartelisten sind verbreitet. Die Herausforderung liegt im Wettbewerb: Etablierte Betreiber haben Markenloyalität aufgebaut, günstige Flächen sind vergeben. Neueintretende Betreiber brauchen echten Differenzierungsansatz. Eintrittsinvestment ist hoch — das Ertragspotenzial bei konsequenter Umsetzung ebenfalls. München ist das paradigmatische Beispiel.</p>
    </div>
  </div>
  <div class="article-card article-card--growth">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Wachstumsmärkte</span>
      <p class="article-card__body">Die Nachfrage wächst sichtbar — Buchungszeiten füllen sich, neue Anlagen werden eröffnet. Das Angebot hat die Nachfrage noch nicht eingeholt; Versorgungslücken sind erkennbar. Das Fenster für attraktive Flächen zu vertretbaren Konditionen schließt sich. Wer wartet, zahlt den Aufpreis des offensichtlich attraktiven Markts.</p>
    </div>
  </div>
  <div class="article-card article-card--emerging">
    <div class="article-card__accent"></div>
    <div class="article-card__inner">
      <span class="article-card__title">Frühmärkte</span>
      <p class="article-card__body">Geringes Angebot, kleine aber wachsende Spielerbasis. Mietkosten niedriger, Standortauswahl größer — aber Nachfrage muss aktiv aufgebaut werden. Anfängerkurse, Vereinskooperationen, lokale Ligen und Konversion von Tennisclubs sind die zentralen Instrumente. Der Weg zur Profitabilität ist länger; die aufgebaute Wettbewerbsposition erweist sich oft als dauerhaft.</p>
    </div>
  </div>
 </div>
 Bevor Sie in einer Stadt konkret nach Objekten suchen, sollten Sie deren Marktreife einordnen. Der Kriterienkatalog zeigt, ob ein bestimmtes Objekt geeignet ist; die Marktreife zeigt, welches Betreiberprofil und welche Strategie überhaupt die Voraussetzung für Erfolg ist.
--- a/extract/padelnomics_extract/pyproject.toml
+++ b/extract/padelnomics_extract/pyproject.toml
@@ -21,6 +21,7 @@ extract-census-usa = "padelnomics_extract.census_usa:main"
 extract-census-usa-income = "padelnomics_extract.census_usa_income:main"
 extract-ons-uk = "padelnomics_extract.ons_uk:main"
 extract-geonames = "padelnomics_extract.geonames:main"
 extract-gisco = "padelnomics_extract.gisco:main"
 [build-system]
 requires = ["hatchling"]
--- a/extract/padelnomics_extract/src/padelnomics_extract/_shared.py
+++ b/extract/padelnomics_extract/src/padelnomics_extract/_shared.py
@@ -11,9 +11,12 @@ from datetime import UTC, datetime
 from pathlib import Path
 import niquests
 from dotenv import load_dotenv
 from .utils import end_run, open_state_db, start_run
 load_dotenv()
 LANDING_DIR = Path(os.environ.get("LANDING_DIR", "data/landing"))
 HTTP_TIMEOUT_SECONDS = 30
--- a/extract/padelnomics_extract/src/padelnomics_extract/all.py
+++ b/extract/padelnomics_extract/src/padelnomics_extract/all.py
@@ -7,7 +7,7 @@ A graphlib.TopologicalSorter schedules them: tasks with no unmet dependencies
 run immediately in parallel; each completion may unlock new tasks.
 Current dependency graph:
-  - All 8 non-availability extractors have no dependencies (run in parallel)
+  - All 9 non-availability extractors have no dependencies (run in parallel)
  - playtomic_availability depends on playtomic_tenants (starts as soon as
    tenants finishes, even if other extractors are still running)
 """
@@ -26,6 +26,8 @@ from .eurostat_city_labels import EXTRACTOR_NAME as EUROSTAT_CITY_LABELS_NAME
 from .eurostat_city_labels import extract as extract_eurostat_city_labels
 from .geonames import EXTRACTOR_NAME as GEONAMES_NAME
 from .geonames import extract as extract_geonames
 from .gisco import EXTRACTOR_NAME as GISCO_NAME
 from .gisco import extract as extract_gisco
 from .ons_uk import EXTRACTOR_NAME as ONS_UK_NAME
 from .ons_uk import extract as extract_ons_uk
 from .overpass import EXTRACTOR_NAME as OVERPASS_NAME
@@ -50,6 +52,7 @@ EXTRACTORS: dict[str, tuple] = {
    CENSUS_USA_INCOME_NAME:    (extract_census_usa_income, []),
    ONS_UK_NAME:               (extract_ons_uk, []),
    GEONAMES_NAME:             (extract_geonames, []),
    GISCO_NAME:                (extract_gisco, []),
    TENANTS_NAME:              (extract_tenants, []),
    AVAILABILITY_NAME:         (extract_availability, [TENANTS_NAME]),
 }
--- a/extract/padelnomics_extract/src/padelnomics_extract/gisco.py
+++ b/extract/padelnomics_extract/src/padelnomics_extract/gisco.py
@@ -0,0 +1,95 @@
 """GISCO NUTS-2 boundary GeoJSON extractor.
 Downloads NUTS-2 boundary polygons from Eurostat GISCO. The file is stored
 uncompressed because DuckDB's ST_Read cannot read gzipped files.
 NUTS classification revises approximately every 7 years (current: 2021).
 The partition path is fixed to the revision year, not the run date, making
 the source version explicit. Cursor tracking still uses year_month to avoid
 re-downloading on every monthly run.
 Landing: {LANDING_DIR}/gisco/2024/01/nuts2_boundaries.geojson  (~5 MB, uncompressed)
 """
 import sqlite3
 from pathlib import Path
 import niquests
 from ._shared import HTTP_TIMEOUT_SECONDS, run_extractor, setup_logging
 from .utils import get_last_cursor
 logger = setup_logging("padelnomics.extract.gisco")
 EXTRACTOR_NAME = "gisco"
 # NUTS 2021 revision, 20M scale (1:20,000,000), WGS84 (EPSG:4326), LEVL_2 only.
 # 20M resolution gives simplified polygons that are fast for point-in-polygon
 # matching without sacrificing accuracy at the NUTS-2 boundary level.
 GISCO_URL = (
    "https://gisco-services.ec.europa.eu/distribution/v2/nuts/geojson/"
    "NUTS_RG_20M_2021_4326_LEVL_2.geojson"
 )
 # Fixed partition: NUTS boundaries are a static reference file, not time-series data.
 # The 2024/01 partition reflects when this NUTS 2021 dataset was first ingested.
 DEST_REL = Path("gisco/2024/01/nuts2_boundaries.geojson")
 _GISCO_TIMEOUT_SECONDS = HTTP_TIMEOUT_SECONDS * 4  # ~5 MB; generous for slow upstreams
 def extract(
    landing_dir: Path,
    year_month: str,
    conn: sqlite3.Connection,
    session: niquests.Session,
 ) -> dict:
    """Download NUTS-2 GeoJSON. Skips if already run this month or file exists."""
    last_cursor = get_last_cursor(conn, EXTRACTOR_NAME)
    if last_cursor == year_month:
        logger.info("already ran for %s — skipping", year_month)
        return {"files_written": 0, "files_skipped": 1, "bytes_written": 0}
    dest = landing_dir / DEST_REL
    if dest.exists():
        logger.info("file already exists (skipping download): %s", dest)
        return {
            "files_written": 0,
            "files_skipped": 1,
            "bytes_written": 0,
            "cursor_value": year_month,
        }
    dest.parent.mkdir(parents=True, exist_ok=True)
    logger.info("GET %s", GISCO_URL)
    resp = session.get(GISCO_URL, timeout=_GISCO_TIMEOUT_SECONDS)
    resp.raise_for_status()
    content = resp.content
    assert len(content) > 100_000, (
        f"GeoJSON too small ({len(content)} bytes) — download may have failed"
    )
    assert b'"FeatureCollection"' in content, "Response does not look like GeoJSON"
    # Write uncompressed — ST_Read requires a plain file, not .gz
    tmp = dest.with_suffix(".geojson.tmp")
    tmp.write_bytes(content)
    tmp.rename(dest)
    size_mb = len(content) / 1_000_000
    logger.info("written %s (%.1f MB)", dest, size_mb)
    return {
        "files_written": 1,
        "files_skipped": 0,
        "bytes_written": len(content),
        "cursor_value": year_month,
    }
 def main() -> None:
    run_extractor(EXTRACTOR_NAME, extract)
 if __name__ == "__main__":
    main()
--- a/extract/padelnomics_extract/src/padelnomics_extract/proxy.py
+++ b/extract/padelnomics_extract/src/padelnomics_extract/proxy.py
@@ -88,8 +88,14 @@ def load_proxy_tiers() -> list[list[str]]:
    for var in ("PROXY_URLS_DATACENTER", "PROXY_URLS_RESIDENTIAL"):
        raw = os.environ.get(var, "")
        urls = [u.strip() for u in raw.split(",") if u.strip()]
-        if urls:
+        valid = []
-            tiers.append(urls)
+        for url in urls:
            if not url.startswith(("http://", "https://")):
                logger.warning("%s contains URL without scheme, skipping: %s", var, url[:60])
                continue
            valid.append(url)
        if valid:
            tiers.append(valid)
    return tiers
@@ -151,6 +157,13 @@ def make_tiered_cycler(tiers: list[list[str]], threshold: int, proxy_failure_lim
      per-proxy dead tracking removes broken individuals; tier-level threshold
      catches systemic failure even before any single proxy hits the limit.
    Stale-failure protection:
      With parallel workers, some threads may fetch a proxy just before the tier
      escalates and report failure after. record_failure(proxy_url) checks which
      tier the proxy belongs to and ignores the tier-level circuit breaker if the
      proxy is from an already-escalated tier. This prevents in-flight failures
      from a dead tier instantly exhausting the freshly-escalated one.
    Returns a dict of callables:
      next_proxy()          -> str | None  — URL from active tier (skips dead), or None
      record_success(proxy_url=None) -> None  — resets consecutive failure counter
@@ -168,6 +181,15 @@ def make_tiered_cycler(tiers: list[list[str]], threshold: int, proxy_failure_lim
    assert isinstance(tiers, list), f"tiers must be a list, got {type(tiers)}"
    assert proxy_failure_limit >= 0, f"proxy_failure_limit must be >= 0, got {proxy_failure_limit}"
    # Reverse map: proxy URL -> tier index. Used in record_failure to ignore
    # "in-flight" failures from workers that fetched a proxy before escalation —
    # those failures belong to the old tier and must not count against the new one.
    proxy_to_tier_idx: dict[str, int] = {
        url: tier_idx
        for tier_idx, tier in enumerate(tiers)
        for url in tier
    }
    lock = threading.Lock()
    cycles = [itertools.cycle(t) for t in tiers]
    state = {
@@ -239,6 +261,15 @@ def make_tiered_cycler(tiers: list[list[str]], threshold: int, proxy_failure_lim
            if idx >= len(tiers):
                # Already exhausted — no-op
                return False
            # Ignore failures from proxies that belong to an already-escalated tier.
            # With parallel workers, some threads fetch a proxy just before escalation
            # and report back after — those stale failures must not penalise the new tier.
            if proxy_url is not None:
                proxy_tier = proxy_to_tier_idx.get(proxy_url)
                if proxy_tier is not None and proxy_tier < idx:
                    return False
            state["consecutive_failures"] += 1
            if state["consecutive_failures"] < threshold:
                return False
--- a/infra/supervisor/workflows.toml
+++ b/infra/supervisor/workflows.toml
@@ -39,3 +39,23 @@ module = "padelnomics_extract.playtomic_availability"
 entry = "main_recheck"
 schedule = "0,30 6-23 * * *"
 depends_on = ["playtomic_availability"]
 [census_usa]
 module = "padelnomics_extract.census_usa"
 schedule = "monthly"
 [census_usa_income]
 module = "padelnomics_extract.census_usa_income"
 schedule = "monthly"
 [eurostat_city_labels]
 module = "padelnomics_extract.eurostat_city_labels"
 schedule = "monthly"
 [ons_uk]
 module = "padelnomics_extract.ons_uk"
 schedule = "monthly"
 [gisco]
 module = "padelnomics_extract.gisco"
 schedule = "monthly"
--- a/scripts/download_gisco_nuts.py
+++ b/scripts/download_gisco_nuts.py
@@ -1,81 +0,0 @@
 """Download NUTS-2 boundary GeoJSON from Eurostat GISCO.
 One-time (or on NUTS revision) download of NUTS-2 boundary polygons used for
 spatial income resolution in dim_locations. Stored uncompressed because DuckDB's
 ST_Read function cannot read gzipped files.
 NUTS classification changes approximately every 7 years. Current revision: 2021.
 Output: {LANDING_DIR}/gisco/2024/01/nuts2_boundaries.geojson  (~5MB, uncompressed)
 Usage:
    uv run python scripts/download_gisco_nuts.py [--landing-dir data/landing]
 Idempotent: skips download if the file already exists.
 """
 import argparse
 import sys
 from pathlib import Path
 import niquests
 # NUTS 2021 revision, 20M scale (1:20,000,000), WGS84 (EPSG:4326), LEVL_2 only.
 # 20M resolution gives simplified polygons that are fast for point-in-polygon
 # matching without sacrificing accuracy at the NUTS-2 boundary level.
 GISCO_URL = (
    "https://gisco-services.ec.europa.eu/distribution/v2/nuts/geojson/"
    "NUTS_RG_20M_2021_4326_LEVL_2.geojson"
 )
 # Fixed partition: NUTS boundaries are a static reference file, not time-series data.
 # Use the NUTS revision year as the partition to make the source version explicit.
 DEST_REL_PATH = "gisco/2024/01/nuts2_boundaries.geojson"
 HTTP_TIMEOUT_SECONDS = 120
 def download_nuts_boundaries(landing_dir: Path) -> None:
    dest = landing_dir / DEST_REL_PATH
    if dest.exists():
        print(f"Already exists (skipping): {dest}")
        return
    dest.parent.mkdir(parents=True, exist_ok=True)
    print(f"Downloading NUTS-2 boundaries from GISCO...")
    print(f"  URL: {GISCO_URL}")
    with niquests.Session() as session:
        resp = session.get(GISCO_URL, timeout=HTTP_TIMEOUT_SECONDS)
        resp.raise_for_status()
    content = resp.content
    assert len(content) > 100_000, (
        f"GeoJSON too small ({len(content)} bytes) — download may have failed"
    )
    assert b'"FeatureCollection"' in content, "Response does not look like GeoJSON"
    # Write uncompressed — ST_Read requires a plain file
    tmp = dest.with_suffix(".geojson.tmp")
    tmp.write_bytes(content)
    tmp.rename(dest)
    size_mb = len(content) / 1_000_000
    print(f"  Written: {dest} ({size_mb:.1f} MB)")
    print("Done. Run SQLMesh plan to rebuild stg_nuts2_boundaries.")
 def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--landing-dir", default="data/landing", type=Path)
    args = parser.parse_args()
    if not args.landing_dir.is_dir():
        print(f"Error: landing dir does not exist: {args.landing_dir}", file=sys.stderr)
        sys.exit(1)
    download_nuts_boundaries(args.landing_dir)
 if __name__ == "__main__":
    main()
--- a/web/src/padelnomics/admin/pipeline_routes.py
+++ b/web/src/padelnomics/admin/pipeline_routes.py
@@ -6,7 +6,9 @@ Operational visibility for the data extraction and transformation pipeline:
  /admin/pipeline/overview     → HTMX tab: extraction status, serving freshness, landing stats
  /admin/pipeline/extractions  → HTMX tab: filterable extraction run history
  /admin/pipeline/extractions/<id>/mark-stale → POST: mark stuck "running" row as failed
-  /admin/pipeline/extract/trigger             → POST: enqueue full extraction run
+  /admin/pipeline/extract/trigger             → POST: enqueue extraction run (HTMX-aware)
  /admin/pipeline/transform    → HTMX tab: SQLMesh + export status, run history
  /admin/pipeline/transform/trigger           → POST: enqueue transform/export/pipeline step
  /admin/pipeline/catalog      → HTMX tab: data catalog (tables, columns, sample data)
  /admin/pipeline/catalog/<table>             → HTMX partial: table detail (columns + sample)
  /admin/pipeline/query        → HTMX tab: SQL query editor
@@ -18,6 +20,7 @@ Data sources:
  - analytics.duckdb            (DuckDB read-only via analytics.execute_user_query)
  - LANDING_DIR/                (filesystem scan for file sizes + dates)
  - infra/supervisor/workflows.toml (schedule definitions — tomllib, stdlib)
  - app.db tasks table          (run_transform, run_export, run_pipeline task rows)
 """
 import asyncio
 import json
@@ -626,10 +629,8 @@ async def pipeline_dashboard():
 # ── Overview tab ─────────────────────────────────────────────────────────────
-@bp.route("/overview")
+async def _render_overview_partial():
-@role_required("admin")
+    """Build and render the pipeline overview partial (shared by GET and POST triggers)."""
 async def pipeline_overview():
    """HTMX tab: extraction status per source, serving freshness, landing zone."""
    latest_runs, landing_stats, workflows, serving_meta = await asyncio.gather(
        asyncio.to_thread(_fetch_latest_per_extractor_sync),
        asyncio.to_thread(_get_landing_zone_stats_sync),
@@ -650,6 +651,13 @@ async def pipeline_overview():
            "stale": _is_stale(run) if run else False,
        })
    # Treat pending extraction tasks as "running" (queued or active).
    from ..core import fetch_all as _fetch_all  # noqa: PLC0415
    pending_extraction = await _fetch_all(
        "SELECT id FROM tasks WHERE task_name = 'run_extraction' AND status = 'pending' LIMIT 1"
    )
    any_running = bool(pending_extraction)
    # Compute landing zone totals
    total_landing_bytes = sum(s["total_bytes"] for s in landing_stats)
@@ -677,10 +685,18 @@ async def pipeline_overview():
        total_landing_bytes=total_landing_bytes,
        serving_tables=serving_tables,
        last_export=last_export,
        any_running=any_running,
        format_bytes=_format_bytes,
    )
@bp.route("/overview")
@role_required("admin")
 async def pipeline_overview():
    """HTMX tab: extraction status per source, serving freshness, landing zone."""
    return await _render_overview_partial()
 # ── Extractions tab ────────────────────────────────────────────────────────────
@@ -745,7 +761,11 @@ async def pipeline_mark_stale(run_id: int):
@role_required("admin")
@csrf_protect
 async def pipeline_trigger_extract():
-    """Enqueue an extraction run — all extractors, or a single named one."""
+    """Enqueue an extraction run — all extractors, or a single named one.
    HTMX-aware: if the HX-Request header is present, returns the overview partial
    directly so the UI can update in-place without a redirect.
    """
    from ..worker import enqueue
    form = await request.form
@@ -757,11 +777,15 @@ async def pipeline_trigger_extract():
            await flash(f"Unknown extractor '{extractor}'.", "warning")
            return redirect(url_for("pipeline.pipeline_dashboard"))
        await enqueue("run_extraction", {"extractor": extractor})
        await flash(f"Extractor '{extractor}' queued. Check the task queue for progress.", "success")
    else:
        await enqueue("run_extraction")
        await flash("Extraction run queued. Check the task queue for progress.", "success")
    is_htmx = request.headers.get("HX-Request") == "true"
    if is_htmx:
        return await _render_overview_partial()
    msg = f"Extractor '{extractor}' queued." if extractor else "Extraction run queued."
    await flash(f"{msg} Check the task queue for progress.", "success")
    return redirect(url_for("pipeline.pipeline_dashboard"))
@@ -847,6 +871,156 @@ async def pipeline_lineage_schema(model: str):
    )
 # ── Transform tab ─────────────────────────────────────────────────────────────
 _TRANSFORM_TASK_NAMES = ("run_transform", "run_export", "run_pipeline")
 async def _fetch_pipeline_tasks() -> dict:
    """Fetch the latest task row for each transform task type, plus recent run history.
    Returns:
        {
            "latest": {"run_transform": row|None, "run_export": row|None, "run_pipeline": row|None},
            "history": [row, ...],   # last 20 rows across all three task types, newest first
        }
    """
    from ..core import fetch_all as _fetch_all  # noqa: PLC0415
    # Latest row per task type (may be pending, complete, or failed)
    latest_rows = await _fetch_all(
        """
        SELECT t.*
        FROM tasks t
        INNER JOIN (
            SELECT task_name, MAX(id) AS max_id
            FROM tasks
            WHERE task_name IN ('run_transform', 'run_export', 'run_pipeline')
            GROUP BY task_name
        ) latest ON t.id = latest.max_id
        """
    )
    latest: dict = {"run_transform": None, "run_export": None, "run_pipeline": None}
    for row in latest_rows:
        latest[row["task_name"]] = dict(row)
    history = await _fetch_all(
        """
        SELECT id, task_name, status, created_at, completed_at, error
        FROM tasks
        WHERE task_name IN ('run_transform', 'run_export', 'run_pipeline')
        ORDER BY id DESC
        LIMIT 20
        """
    )
    return {"latest": latest, "history": [dict(r) for r in history]}
 def _format_duration(created_at: str | None, completed_at: str | None) -> str:
    """Human-readable duration between created_at and completed_at, or '' if unavailable."""
    if not created_at or not completed_at:
        return ""
    try:
        fmt = "%Y-%m-%d %H:%M:%S"
        start = datetime.strptime(created_at, fmt)
        end = datetime.strptime(completed_at, fmt)
        delta = int((end - start).total_seconds())
        if delta < 0:
            return ""
        if delta < 60:
            return f"{delta}s"
        return f"{delta // 60}m {delta % 60}s"
    except ValueError:
        return ""
 async def _render_transform_partial():
    """Build and render the transform tab partial."""
    task_data = await _fetch_pipeline_tasks()
    latest = task_data["latest"]
    history = task_data["history"]
    # Enrich history rows with duration
    for row in history:
        row["duration"] = _format_duration(row.get("created_at"), row.get("completed_at"))
        # Truncate error for display
        if row.get("error"):
            row["error_short"] = row["error"][:120]
        else:
            row["error_short"] = None
    any_running = any(
        t is not None and t["status"] == "pending" for t in latest.values()
    )
    serving_meta = await asyncio.to_thread(_load_serving_meta)
    return await render_template(
        "admin/partials/pipeline_transform.html",
        latest=latest,
        history=history,
        any_running=any_running,
        serving_meta=serving_meta,
        format_duration=_format_duration,
    )
@bp.route("/transform")
@role_required("admin")
 async def pipeline_transform():
    """HTMX tab: SQLMesh transform + export status, run history."""
    return await _render_transform_partial()
@bp.route("/transform/trigger", methods=["POST"])
@role_required("admin")
@csrf_protect
 async def pipeline_trigger_transform():
    """Enqueue a transform, export, or full pipeline task.
    form field `step`: 'transform' | 'export' | 'pipeline'
    Concurrency guard: rejects if the same task type is already pending.
    HTMX-aware: returns the transform partial for HTMX requests.
    """
    from ..core import fetch_one as _fetch_one  # noqa: PLC0415
    from ..worker import enqueue
    form = await request.form
    step = (form.get("step") or "").strip()
    step_to_task = {
        "transform": "run_transform",
        "export": "run_export",
        "pipeline": "run_pipeline",
    }
    if step not in step_to_task:
        await flash(f"Unknown step '{step}'.", "warning")
        return redirect(url_for("pipeline.pipeline_dashboard"))
    task_name = step_to_task[step]
    # Concurrency guard: reject if same task type is already pending
    existing = await _fetch_one(
        "SELECT id FROM tasks WHERE task_name = ? AND status = 'pending' LIMIT 1",
        (task_name,),
    )
    if existing:
        is_htmx = request.headers.get("HX-Request") == "true"
        if is_htmx:
            return await _render_transform_partial()
        await flash(f"A '{step}' task is already queued (task #{existing['id']}).", "warning")
        return redirect(url_for("pipeline.pipeline_dashboard"))
    await enqueue(task_name)
    is_htmx = request.headers.get("HX-Request") == "true"
    if is_htmx:
        return await _render_transform_partial()
    await flash(f"'{step}' task queued. Check the task queue for progress.", "success")
    return redirect(url_for("pipeline.pipeline_dashboard"))
 # ── Catalog tab ───────────────────────────────────────────────────────────────
--- a/web/src/padelnomics/admin/templates/admin/partials/pipeline_overview.html
+++ b/web/src/padelnomics/admin/templates/admin/partials/pipeline_overview.html
@@ -1,4 +1,11 @@
-<!-- Pipeline Overview Tab: extraction status, serving freshness, landing zone -->
+<!-- Pipeline Overview Tab: extraction status, serving freshness, landing zone
     Self-polls every 5s while any extraction task is pending, stops when quiet. -->
 <div id="pipeline-overview-content"
     hx-get="{{ url_for('pipeline.pipeline_overview') }}"
     hx-target="this"
     hx-swap="outerHTML"
     {% if any_running %}hx-trigger="every 5s"{% endif %}>
 <!-- Extraction Status Grid -->
 <div class="card mb-4">
@@ -26,12 +33,14 @@
        {% if stale %}
        <span class="badge-warning" style="font-size:10px;padding:1px 6px;margin-left:auto">stale</span>
        {% endif %}
-        <form method="post" action="{{ url_for('pipeline.pipeline_trigger_extract') }}" class="m-0 ml-auto">
+        <button type="button"
-          <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                class="btn btn-sm ml-auto"
-          <input type="hidden" name="extractor" value="{{ wf.name }}">
+                style="padding:2px 8px;font-size:11px"
-          <button type="button" class="btn btn-sm" style="padding:2px 8px;font-size:11px"
+                hx-post="{{ url_for('pipeline.pipeline_trigger_extract') }}"
-                  onclick="confirmAction('Run {{ wf.name }} extractor?', this.closest('form'))">Run</button>
+                hx-target="#pipeline-overview-content"
-        </form>
+                hx-swap="outerHTML"
                hx-vals='{"extractor": "{{ wf.name }}", "csrf_token": "{{ csrf_token() }}"}'
                onclick="if (!confirm('Run {{ wf.name }} extractor?')) return false;">Run</button>
      </div>
      <p class="text-xs text-slate">{{ wf.schedule_label }}</p>
      {% if run %}
@@ -57,7 +66,7 @@
 </div>
 <!-- Two-column row: Serving Freshness + Landing Zone -->
-<div style="display:grid;grid-template-columns:1fr 1fr;gap:1rem">
+<div class="pipeline-two-col">
  <!-- Serving Freshness -->
  <div class="card">
@@ -68,6 +77,7 @@
    </p>
    {% endif %}
    {% if serving_tables %}
    <div style="overflow-x:auto">
    <table class="table" style="font-size:0.8125rem">
      <thead>
        <tr>
@@ -86,6 +96,7 @@
        {% endfor %}
      </tbody>
    </table>
    </div>
    {% else %}
    <p class="text-sm text-slate">No serving tables found — run the pipeline first.</p>
    {% endif %}
@@ -99,6 +110,7 @@
      </span>
    </p>
    {% if landing_stats %}
    <div style="overflow-x:auto">
    <table class="table" style="font-size:0.8125rem">
      <thead>
        <tr>
@@ -119,6 +131,7 @@
        {% endfor %}
      </tbody>
    </table>
    </div>
    {% else %}
    <p class="text-sm text-slate">
      Landing zone empty or not found at <code>data/landing</code>.
@@ -127,3 +140,5 @@
  </div>
 </div>
 </div>{# end #pipeline-overview-content #}
--- a/web/src/padelnomics/admin/templates/admin/partials/pipeline_transform.html
+++ b/web/src/padelnomics/admin/templates/admin/partials/pipeline_transform.html
@@ -0,0 +1,197 @@
 <!-- Pipeline Transform Tab: SQLMesh + export status, run history
     Self-polls every 5s while any transform/export task is pending. -->
 <div id="pipeline-transform-content"
     hx-get="{{ url_for('pipeline.pipeline_transform') }}"
     hx-target="this"
     hx-swap="outerHTML"
     {% if any_running %}hx-trigger="every 5s"{% endif %}>
 <!-- Status Cards: Transform + Export -->
 <div class="pipeline-two-col mb-4">
  <!-- SQLMesh Transform -->
  {% set tx = latest['run_transform'] %}
  <div class="card">
    <p class="card-header">SQLMesh Transform</p>
    <div class="flex items-center gap-2 mb-3">
      {% if tx is none %}
        <span class="status-dot pending"></span>
        <span class="text-sm text-slate">Never run</span>
      {% elif tx.status == 'pending' %}
        <span class="status-dot running"></span>
        <span class="text-sm text-slate">Running…</span>
      {% elif tx.status == 'complete' %}
        <span class="status-dot ok"></span>
        <span class="text-sm text-slate">Complete</span>
      {% else %}
        <span class="status-dot failed"></span>
        <span class="text-sm text-danger">Failed</span>
      {% endif %}
    </div>
    {% if tx %}
    <p class="text-xs text-slate mono">
      Started: {{ (tx.created_at or '')[:19] or '—' }}
    </p>
    {% if tx.completed_at %}
    <p class="text-xs text-slate mono">
      Finished: {{ tx.completed_at[:19] }}
    </p>
    {% endif %}
    {% if tx.status == 'failed' and tx.error %}
    <details class="mt-2">
      <summary class="text-xs text-danger cursor-pointer">Error</summary>
      <pre class="text-xs mt-1 p-2 bg-gray-50 rounded overflow-auto" style="max-height:8rem;white-space:pre-wrap">{{ tx.error[:400] }}</pre>
    </details>
    {% endif %}
    {% endif %}
    <div class="mt-3">
      <button type="button"
              class="btn btn-sm"
              {% if any_running %}disabled{% endif %}
              hx-post="{{ url_for('pipeline.pipeline_trigger_transform') }}"
              hx-target="#pipeline-transform-content"
              hx-swap="outerHTML"
              hx-vals='{"step": "transform", "csrf_token": "{{ csrf_token() }}"}'
              onclick="if (!confirm('Run SQLMesh transform (prod --auto-apply)?')) return false;">
        Run Transform
      </button>
    </div>
  </div>
  <!-- Export Serving -->
  {% set ex = latest['run_export'] %}
  <div class="card">
    <p class="card-header">Export Serving</p>
    <div class="flex items-center gap-2 mb-3">
      {% if ex is none %}
        <span class="status-dot pending"></span>
        <span class="text-sm text-slate">Never run</span>
      {% elif ex.status == 'pending' %}
        <span class="status-dot running"></span>
        <span class="text-sm text-slate">Running…</span>
      {% elif ex.status == 'complete' %}
        <span class="status-dot ok"></span>
        <span class="text-sm text-slate">Complete</span>
      {% else %}
        <span class="status-dot failed"></span>
        <span class="text-sm text-danger">Failed</span>
      {% endif %}
    </div>
    {% if ex %}
    <p class="text-xs text-slate mono">
      Started: {{ (ex.created_at or '')[:19] or '—' }}
    </p>
    {% if ex.completed_at %}
    <p class="text-xs text-slate mono">
      Finished: {{ ex.completed_at[:19] }}
    </p>
    {% endif %}
    {% if serving_meta %}
    <p class="text-xs text-slate mt-1">
      Last export: <span class="font-semibold mono">{{ (serving_meta.exported_at_utc or '')[:19].replace('T', ' ') or '—' }}</span>
    </p>
    {% endif %}
    {% if ex.status == 'failed' and ex.error %}
    <details class="mt-2">
      <summary class="text-xs text-danger cursor-pointer">Error</summary>
      <pre class="text-xs mt-1 p-2 bg-gray-50 rounded overflow-auto" style="max-height:8rem;white-space:pre-wrap">{{ ex.error[:400] }}</pre>
    </details>
    {% endif %}
    {% endif %}
    <div class="mt-3">
      <button type="button"
              class="btn btn-sm"
              {% if any_running %}disabled{% endif %}
              hx-post="{{ url_for('pipeline.pipeline_trigger_transform') }}"
              hx-target="#pipeline-transform-content"
              hx-swap="outerHTML"
              hx-vals='{"step": "export", "csrf_token": "{{ csrf_token() }}"}'
              onclick="if (!confirm('Export serving tables (lakehouse → analytics.duckdb)?')) return false;">
        Run Export
      </button>
    </div>
  </div>
 </div>
 <!-- Run Full Pipeline -->
 {% set pl = latest['run_pipeline'] %}
 <div class="card mb-4">
  <div class="flex items-center justify-between flex-wrap gap-3">
    <div>
      <p class="font-semibold text-navy text-sm">Full Pipeline</p>
      <p class="text-xs text-slate mt-1">Runs extract → transform → export sequentially</p>
      {% if pl %}
      <p class="text-xs text-slate mono mt-1">
        Last: {{ (pl.created_at or '')[:19] or '—' }}
        {% if pl.status == 'complete' %}<span class="badge-success ml-2">Complete</span>{% endif %}
        {% if pl.status == 'pending' %}<span class="badge-warning ml-2">Running…</span>{% endif %}
        {% if pl.status == 'failed' %}<span class="badge-danger ml-2">Failed</span>{% endif %}
      </p>
      {% endif %}
    </div>
    <button type="button"
            class="btn btn-sm"
            {% if any_running %}disabled{% endif %}
            hx-post="{{ url_for('pipeline.pipeline_trigger_transform') }}"
            hx-target="#pipeline-transform-content"
            hx-swap="outerHTML"
            hx-vals='{"step": "pipeline", "csrf_token": "{{ csrf_token() }}"}'
            onclick="if (!confirm('Run full ELT pipeline (extract → transform → export)?')) return false;">
      Run Full Pipeline
    </button>
  </div>
 </div>
 <!-- Recent Runs -->
 <div class="card">
  <p class="card-header">Recent Runs</p>
  {% if history %}
  <div style="overflow-x:auto">
  <table class="table" style="font-size:0.8125rem">
    <thead>
      <tr>
        <th>#</th>
        <th>Step</th>
        <th>Started</th>
        <th>Duration</th>
        <th>Status</th>
        <th>Error</th>
      </tr>
    </thead>
    <tbody>
      {% for row in history %}
      <tr>
        <td class="text-xs text-slate">{{ row.id }}</td>
        <td class="mono text-xs">{{ row.task_name | replace('run_', '') }}</td>
        <td class="mono text-xs text-slate">{{ (row.created_at or '')[:19] or '—' }}</td>
        <td class="mono text-xs text-slate">{{ row.duration or '—' }}</td>
        <td>
          {% if row.status == 'complete' %}
            <span class="badge-success">Complete</span>
          {% elif row.status == 'failed' %}
            <span class="badge-danger">Failed</span>
          {% else %}
            <span class="badge-warning">Running…</span>
          {% endif %}
        </td>
        <td>
          {% if row.error_short %}
          <details>
            <summary class="text-xs text-danger cursor-pointer">Error</summary>
            <pre class="text-xs mt-1 p-2 bg-gray-50 rounded overflow-auto" style="max-width:24rem;white-space:pre-wrap">{{ row.error_short }}</pre>
          </details>
          {% else %}—{% endif %}
        </td>
      </tr>
      {% endfor %}
    </tbody>
  </table>
  </div>
  {% else %}
  <p class="text-sm text-slate">No transform runs yet.</p>
  {% endif %}
 </div>
 </div>{# end #pipeline-transform-content #}
--- a/web/src/padelnomics/admin/templates/admin/pipeline.html
+++ b/web/src/padelnomics/admin/templates/admin/pipeline.html
@@ -15,6 +15,7 @@
  .pipeline-tabs {
    display: flex; gap: 0; border-bottom: 2px solid #E2E8F0; margin-bottom: 1.5rem;
    overflow-x: auto; -webkit-overflow-scrolling: touch;
  }
  .pipeline-tabs button {
    padding: 0.625rem 1.25rem; font-size: 0.8125rem; font-weight: 600;
@@ -32,7 +33,19 @@
  .status-dot.failed  { background: #EF4444; }
  .status-dot.stale   { background: #D97706; }
  .status-dot.running { background: #3B82F6; }
  @keyframes pulse-dot { 0%,100%{opacity:1} 50%{opacity:0.4} }
  .status-dot.running { animation: pulse-dot 1.5s ease-in-out infinite; }
  .status-dot.pending { background: #CBD5E1; }
  .pipeline-two-col {
    display: grid;
    grid-template-columns: 1fr;
    gap: 1rem;
  }
  @media (min-width: 640px) {
    .pipeline-two-col { grid-template-columns: 1fr 1fr; }
  }
 </style>
 {% endblock %}
@@ -43,10 +56,11 @@
      <p class="text-sm text-slate mt-1">Extraction status, data catalog, and ad-hoc query editor</p>
    </div>
    <div class="flex gap-2">
-      <form method="post" action="{{ url_for('pipeline.pipeline_trigger_extract') }}" class="m-0">
+      <form method="post" action="{{ url_for('pipeline.pipeline_trigger_transform') }}" class="m-0">
        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
        <input type="hidden" name="step" value="pipeline">
        <button type="button" class="btn btn-sm"
-                onclick="confirmAction('Enqueue a full extraction run? This will run all extractors in the background.', this.closest('form'))">
+                onclick="confirmAction('Run full ELT pipeline (extract → transform → export)? This runs in the background.', this.closest('form'))">
          Run Pipeline
        </button>
      </form>
@@ -106,6 +120,10 @@
            hx-get="{{ url_for('pipeline.pipeline_lineage') }}"
            hx-target="#pipeline-tab-content" hx-swap="innerHTML"
            hx-trigger="click">Lineage</button>
    <button data-tab="transform"
            hx-get="{{ url_for('pipeline.pipeline_transform') }}"
            hx-target="#pipeline-tab-content" hx-swap="innerHTML"
            hx-trigger="click">Transform</button>
  </div>
  <!-- Tab content (Overview loads on page load) -->
--- a/web/src/padelnomics/static/css/input.css
+++ b/web/src/padelnomics/static/css/input.css
@@ -570,6 +570,270 @@
    @apply px-4 pb-4 text-slate-dark;
  }
  /* ── Article Timeline (phase/process diagrams) ── */
  .article-timeline {
    display: flex;
    gap: 0;
    margin: 1.5rem 0 2rem;
    position: relative;
    overflow-x: auto;
    padding-bottom: 0.5rem;
  }
  .article-timeline__phase {
    flex: 1;
    min-width: 130px;
    display: flex;
    flex-direction: column;
    align-items: center;
    position: relative;
  }
  /* Connecting line between phases */
  .article-timeline__phase + .article-timeline__phase::before {
    content: '';
    position: absolute;
    top: 22px;
    left: calc(-50% + 22px);
    right: calc(50% + 22px);
    height: 2px;
    background: #CBD5E1;
    z-index: 0;
  }
  .article-timeline__phase + .article-timeline__phase::after {
    content: '›';
    position: absolute;
    top: 10px;
    left: calc(-50% + 18px);
    font-size: 1rem;
    line-height: 1;
    color: #94A3B8;
    z-index: 1;
  }
  .article-timeline__num {
    width: 44px;
    height: 44px;
    border-radius: 50%;
    background: #0F172A;
    color: #fff;
    display: flex;
    align-items: center;
    justify-content: center;
    font-size: 0.75rem;
    font-weight: 700;
    font-family: var(--font-display);
    flex-shrink: 0;
    position: relative;
    z-index: 2;
  }
  .article-timeline__card {
    margin-top: 0.75rem;
    background: #F8FAFC;
    border: 1px solid #E2E8F0;
    border-radius: 12px;
    padding: 0.75rem 0.875rem;
    text-align: center;
    width: 100%;
  }
  .article-timeline__title {
    font-weight: 700;
    font-size: 0.8125rem;
    color: #0F172A;
    line-height: 1.3;
    margin-bottom: 0.25rem;
    font-family: var(--font-display);
  }
  .article-timeline__subtitle {
    font-size: 0.75rem;
    color: #64748B;
    margin-bottom: 0.375rem;
    line-height: 1.3;
  }
  .article-timeline__meta {
    font-size: 0.6875rem;
    color: #94A3B8;
    line-height: 1.4;
  }
  /* Mobile: vertical timeline */
  @media (max-width: 600px) {
    .article-timeline {
      flex-direction: column;
      gap: 0.75rem;
      overflow-x: visible;
    }
    .article-timeline__phase {
      flex-direction: row;
      align-items: flex-start;
      min-width: auto;
      gap: 0.75rem;
    }
    .article-timeline__phase + .article-timeline__phase::before {
      content: '';
      position: absolute;
      top: calc(-0.375rem);
      left: 21px;
      right: auto;
      width: 2px;
      height: 0.75rem;
      background: #CBD5E1;
    }
    .article-timeline__phase + .article-timeline__phase::after {
      content: '›';
      position: absolute;
      top: calc(-0.3rem);
      left: 15px;
      font-size: 0.9rem;
      transform: rotate(90deg);
    }
    .article-timeline__card {
      margin-top: 0;
      text-align: left;
      flex: 1;
    }
    .article-timeline__num {
      flex-shrink: 0;
    }
  }
  /* ── Article Callout Boxes ── */
  .article-callout {
    display: flex;
    gap: 0.875rem;
    padding: 1rem 1.25rem;
    border-radius: 12px;
    border-left: 4px solid;
    margin: 1.5rem 0;
  }
  .article-callout::before {
    font-size: 1.1rem;
    flex-shrink: 0;
    line-height: 1.5;
  }
  .article-callout__body {
    flex: 1;
  }
  .article-callout__title {
    font-weight: 700;
    font-size: 0.875rem;
    margin-bottom: 0.375rem;
    display: block;
  }
  .article-callout p {
    font-size: 0.875rem;
    line-height: 1.6;
    margin: 0;
    color: inherit;
  }
  .article-callout--warning {
    background: #FFFBEB;
    border-color: #D97706;
    color: #78350F;
  }
  .article-callout--warning::before {
    content: '⚠';
    color: #D97706;
  }
  .article-callout--warning .article-callout__title {
    color: #92400E;
  }
  .article-callout--tip {
    background: #F0FDF4;
    border-color: #16A34A;
    color: #14532D;
  }
  .article-callout--tip::before {
    content: '💡';
  }
  .article-callout--tip .article-callout__title {
    color: #166534;
  }
  .article-callout--info {
    background: #EFF6FF;
    border-color: #1D4ED8;
    color: #1E3A5F;
  }
  .article-callout--info::before {
    content: 'ℹ';
    color: #1D4ED8;
  }
  .article-callout--info .article-callout__title {
    color: #1E40AF;
  }
  /* ── Article Cards (2-col comparison grid) ── */
  .article-cards {
    display: grid;
    grid-template-columns: repeat(2, 1fr);
    gap: 1rem;
    margin: 1.5rem 0;
  }
  @media (max-width: 580px) {
    .article-cards {
      grid-template-columns: 1fr;
    }
  }
  .article-card {
    border-radius: 12px;
    border: 1px solid #E2E8F0;
    overflow: hidden;
    background: #fff;
  }
  .article-card__accent {
    height: 4px;
  }
  .article-card--success .article-card__accent { background: #16A34A; }
  .article-card--failure .article-card__accent { background: #EF4444; }
  .article-card--neutral .article-card__accent { background: #1D4ED8; }
  .article-card--established .article-card__accent { background: #0F172A; }
  .article-card--growth .article-card__accent { background: #1D4ED8; }
  .article-card--emerging .article-card__accent { background: #16A34A; }
  .article-card__inner {
    padding: 1rem 1.125rem;
  }
  .article-card__title {
    font-weight: 700;
    font-size: 0.875rem;
    color: #0F172A;
    margin-bottom: 0.5rem;
    font-family: var(--font-display);
    display: block;
  }
  .article-card__body {
    font-size: 0.8125rem;
    color: #475569;
    line-height: 1.6;
    margin: 0;
  }
  /* ── Severity Pills (risk table badges) ── */
  .severity {
    display: inline-block;
    padding: 0.125rem 0.5rem;
    border-radius: 9999px;
    font-size: 0.6875rem;
    font-weight: 700;
    letter-spacing: 0.03em;
    white-space: nowrap;
  }
  .severity--high {
    background: #FEE2E2;
    color: #991B1B;
  }
  .severity--medium-high {
    background: #FEF3C7;
    color: #92400E;
  }
  .severity--medium {
    background: #FEF9C3;
    color: #713F12;
  }
  .severity--low-medium {
    background: #ECFDF5;
    color: #065F46;
  }
  .severity--low {
    background: #F0FDF4;
    color: #166534;
  }
  /* Inline HTMX loading indicator for search forms.
     Opacity is handled by HTMX's built-in .htmx-indicator CSS.
     This class only adds positioning and the spin animation. */
--- a/web/src/padelnomics/worker.py
+++ b/web/src/padelnomics/worker.py
@@ -735,6 +735,107 @@ async def handle_run_extraction(payload: dict) -> None:
    logger.info("Extraction completed: %s", result.stdout[-300:] if result.stdout else "(no output)")
@task("run_transform")
 async def handle_run_transform(payload: dict) -> None:
    """Run SQLMesh transform (prod plan --auto-apply) in the background.
    Shells out to `uv run sqlmesh -p transform/sqlmesh_padelnomics plan prod --auto-apply`.
    2-hour absolute timeout — same as extraction.
    """
    import subprocess
    from pathlib import Path
    repo_root = Path(__file__).resolve().parents[4]
    result = await asyncio.to_thread(
        subprocess.run,
        ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
        capture_output=True,
        text=True,
        timeout=7200,
        cwd=str(repo_root),
    )
    if result.returncode != 0:
        raise RuntimeError(
            f"SQLMesh transform failed (exit {result.returncode}): {result.stderr[:500]}"
        )
    logger.info("SQLMesh transform completed: %s", result.stdout[-300:] if result.stdout else "(no output)")
@task("run_export")
 async def handle_run_export(payload: dict) -> None:
    """Export serving tables from lakehouse.duckdb → analytics.duckdb.
    Shells out to `uv run python src/padelnomics/export_serving.py`.
    10-minute absolute timeout.
    """
    import subprocess
    from pathlib import Path
    repo_root = Path(__file__).resolve().parents[4]
    result = await asyncio.to_thread(
        subprocess.run,
        ["uv", "run", "python", "src/padelnomics/export_serving.py"],
        capture_output=True,
        text=True,
        timeout=600,
        cwd=str(repo_root),
    )
    if result.returncode != 0:
        raise RuntimeError(
            f"Export failed (exit {result.returncode}): {result.stderr[:500]}"
        )
    logger.info("Export completed: %s", result.stdout[-300:] if result.stdout else "(no output)")
@task("run_pipeline")
 async def handle_run_pipeline(payload: dict) -> None:
    """Run full ELT pipeline: extract → transform → export, stopping on first failure."""
    import subprocess
    from pathlib import Path
    repo_root = Path(__file__).resolve().parents[4]
    steps = [
        (
            "extraction",
            ["uv", "run", "--package", "padelnomics_extract", "extract"],
            7200,
        ),
        (
            "transform",
            ["uv", "run", "sqlmesh", "-p", "transform/sqlmesh_padelnomics", "plan", "prod", "--auto-apply"],
            7200,
        ),
        (
            "export",
            ["uv", "run", "python", "src/padelnomics/export_serving.py"],
            600,
        ),
    ]
    for step_name, cmd, timeout_seconds in steps:
        logger.info("Pipeline step starting: %s", step_name)
        result = await asyncio.to_thread(
            subprocess.run,
            cmd,
            capture_output=True,
            text=True,
            timeout=timeout_seconds,
            cwd=str(repo_root),
        )
        if result.returncode != 0:
            raise RuntimeError(
                f"Pipeline failed at {step_name} (exit {result.returncode}): {result.stderr[:500]}"
            )
        logger.info(
            "Pipeline step complete: %s — %s",
            step_name,
            result.stdout[-200:] if result.stdout else "(no output)",
        )
    logger.info("Full pipeline complete (extract → transform → export)")
@task("generate_articles")
 async def handle_generate_articles(payload: dict) -> None:
    """Generate articles from a template in the background."""
Author	SHA1	Message	Date
Deeman	c5d872ec55	chore(secrets): add R2_LANDING_* vars for All checks were successful CI / test (push) Successful in 49s Details CI / tag (push) Successful in 2s Details landing-zone backup bucket…	2026-03-01 17:32:51 +01:00
Deeman	75305935bd	merge: GISCO extractor + wire all extractors + load_dotenv fix All checks were successful CI / test (push) Successful in 50s Details CI / tag (push) Successful in 3s Details	2026-03-01 17:08:42 +01:00
Deeman	99cb0ac005	chore: remove .gitlab-ci.yml (GitLab now backup-only mirror) All checks were successful CI / test (push) Successful in 50s Details CI / tag (push) Successful in 2s Details CI runs on Gitea only. GitLab is a passive push mirror — no runners, no tagging, no deploy involvement. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 17:06:09 +01:00
Deeman	a15c32d398	fix(extract): load .env automatically via load_dotenv() PROXY_URLS_* and other secrets were defined in .env but never loaded, causing availability to run in slow serial mode (1 req/s) instead of parallel mode with proxies. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 16:41:59 +01:00
Deeman	97c5846d51	feat(extract): GISCO extractor + wire all unscheduled extractors - New gisco.py: proper extractor module replacing scripts/download_gisco_nuts.py. Writes uncompressed .geojson (ST_Read can't handle .gz). Fixed partition path gisco/2024/01/nuts2_boundaries.geojson; cursor tracking skips re-download monthly. - all.py: import + register gisco in EXTRACTORS (9 independent, 1 dep) - pyproject.toml: add extract-gisco entry point - workflows.toml: add census_usa, census_usa_income, eurostat_city_labels, ons_uk, gisco — all monthly, no dependencies - Delete scripts/download_gisco_nuts.py (superseded) Unblocks: stg_nuts2_boundaries, stg_regional_income, stg_income_usa, and 4 downstream models (dim_locations, pseo_city_costs_de, location_opportunity_profile, pseo_country_overview). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 15:49:39 +01:00
Deeman	0d903ec926	chore(changelog): document stale-tier circuit breaker fix All checks were successful CI / test (push) Successful in 51s Details CI / tag (push) Successful in 3s Details Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:43:18 +01:00
Deeman	42c49e383c	fix(proxy): ignore stale-tier failures in record_failure() With parallel workers, threads that fetch a proxy just before escalation can report failures after the tier has already changed — those failures were silently counting against the new tier, immediately exhausting it before it ever got tried (Rayobyte being skipped entirely in favour of DataImpulse because 10 in-flight Webshare failures hit the threshold). Fix: build a proxy_url → tier_idx reverse map at construction time and skip the tier-level circuit breaker when the failing proxy belongs to an already-escalated tier. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:43:05 +01:00
Deeman	1c0edff3e5	chore(changelog): document visual upgrades for longform articles Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:29:21 +01:00
Deeman	8a28b94ec2	merge: visual upgrades for longform articles (timeline, callouts, cards, severity pills)	2026-03-01 14:28:57 +01:00
Deeman	9b54f2d544	fix(secrets): add http:// scheme to proxy URLs in dev + prod SOPS All checks were successful CI / test (push) Successful in 51s Details CI / tag (push) Successful in 3s Details PROXY_URLS_DATACENTER was missing the scheme prefix, causing SSL handshake failures on the Rayobyte HTTP-only proxy. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:28:35 +01:00
Deeman	08bd2b2989	chore(changelog): document proxy URL scheme validation fix Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:26:57 +01:00
Deeman	81a57db272	fix(proxy): skip URLs without scheme in load_proxy_tiers() Validates each URL in PROXY_URLS_DATACENTER / PROXY_URLS_RESIDENTIAL: logs a warning and skips any entry missing an http:// or https:// scheme instead of passing malformed URLs that cause SSL or connection errors. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:26:41 +01:00
Deeman	bce6b2d340	feat(articles): visual upgrades — timeline, callouts, cards, severity pills Add 4 reusable CSS article components and apply them across 6 cornerstone articles: CSS (input.css): - article-timeline: horizontal phase diagram with numbered cards, collapses to vertical on mobile - article-callout (warning/tip/info): left-bordered callout boxes with icon and title - article-cards: 2-col grid of accent-topped cards (success/failure/neutral/established/growth/emerging) - severity: inline pill badges (high/medium-high/medium/low-medium/low) for risk tables Articles updated: - padel-hall-build-guide-en + padel-halle-bauen-de: ASCII code block → timeline HTML; 3 bold/blockquote warnings → callout boxes; success/failure patterns → 4 cards - padel-hall-investment-risks-en + padel-halle-risiken-de: risk overview table severity → pills; personal guarantee section → callout; risk management section → 4 cards - padel-hall-location-guide-en + padel-standort-analyse-de: market maturity paragraphs → 3 stage cards Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 14:24:11 +01:00
Deeman	f92d863781	feat(pipeline): live extraction status + Transform tab Adds HTMX live polling to the Overview tab (stops when quiet) and a new Transform tab for managing the SQLMesh + export steps of the ELT pipeline. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 13:47:17 +01:00
Deeman	a3dd37b1be	chore(changelog): document pipeline transform tab + live status feature Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 13:47:07 +01:00
Deeman	e5cbcf462e	feat(pipeline): live extraction status + Transform tab - worker: add run_transform, run_export, run_pipeline task handlers - run_transform: sqlmesh plan prod --auto-apply, 2h timeout - run_export: export_serving.py, 10min timeout - run_pipeline: sequential extract → transform → export, stops on first failure - pipeline_routes: refactor overview into _render_overview_partial() helper, make pipeline_trigger_extract() HTMX-aware (returns partial on HX-Request), add _fetch_pipeline_tasks(), _format_duration() helpers, add pipeline_transform() + pipeline_trigger_transform() with concurrency guard - pipeline_overview.html: wrap in self-polling div (every 5s while any_running), convert Run buttons to hx-post targeting #pipeline-overview-content - pipeline.html: add pulse animation for .status-dot.running, add Transform tab button, rewire header "Run Pipeline" button to enqueue run_pipeline task - pipeline_transform.html: new partial — status cards for transform + export, "Run Full Pipeline" card, recent runs table with duration + error details Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 13:46:11 +01:00
Deeman	169092c8ea	fix(admin): make pipeline data view responsive on mobile All checks were successful CI / test (push) Successful in 50s Details CI / tag (push) Successful in 2s Details - Tab bar: add overflow-x:auto so 5 tabs scroll on narrow screens - Overview grid: replace hardcoded 1fr 1fr with .pipeline-two-col (stacks below 640px) - Overview tables: wrap Serving Tables + Landing Zone in overflow-x:auto divs Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>	2026-03-01 13:16:58 +01:00