Files
beanflows/web/src/beanflows/api/routes.py
Deeman 0a83b2cb74 Add CFTC COT data integration with foundation data model layer
- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs
  from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization
- SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart)
- dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology
- fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities
- obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios
- Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest]
- Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart
- 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-20 23:28:10 +01:00

228 lines
8.2 KiB
Python

"""
API domain: REST API for commodity analytics with key authentication and rate limiting.
"""
import csv
import hashlib
import io
from datetime import datetime
from functools import wraps
from quart import Blueprint, Response, g, jsonify, request
from .. import analytics
from ..core import check_rate_limit, execute, fetch_one
bp = Blueprint("api", __name__)
# =============================================================================
# SQL Queries
# =============================================================================
async def verify_api_key(raw_key: str) -> dict | None:
"""Verify API key and return key data with user info."""
key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
result = await fetch_one(
"""
SELECT ak.*, u.email, u.id as user_id,
COALESCE(s.plan, 'free') as plan
FROM api_keys ak
JOIN users u ON u.id = ak.user_id
LEFT JOIN subscriptions s ON s.user_id = u.id AND s.status = 'active'
WHERE ak.key_hash = ? AND ak.deleted_at IS NULL AND u.deleted_at IS NULL
""",
(key_hash,),
)
if result:
await execute(
"UPDATE api_keys SET last_used_at = ? WHERE id = ?",
(datetime.utcnow().isoformat(), result["id"]),
)
return result
async def log_api_request(user_id: int, endpoint: str, method: str) -> None:
"""Log API request for analytics and rate limiting."""
await execute(
"""
INSERT INTO api_requests (user_id, endpoint, method, created_at)
VALUES (?, ?, ?, ?)
""",
(user_id, endpoint, method, datetime.utcnow().isoformat()),
)
# =============================================================================
# Decorators
# =============================================================================
def api_key_required(scopes: list[str] = None):
"""Require valid API key with optional scope check."""
def decorator(f):
@wraps(f)
async def decorated(*args, **kwargs):
auth = request.headers.get("Authorization", "")
if not auth.startswith("Bearer "):
return jsonify({"error": "Missing API key"}), 401
raw_key = auth[7:]
key_data = await verify_api_key(raw_key)
if not key_data:
return jsonify({"error": "Invalid API key"}), 401
# Check scopes
if scopes:
key_scopes = (key_data.get("scopes") or "").split(",")
if not any(s in key_scopes for s in scopes):
return jsonify({"error": "Insufficient permissions"}), 403
# Check plan allows API access
plan = key_data.get("plan", "free")
if plan == "free":
return jsonify({"error": "API access requires a Starter or Pro plan"}), 403
# Rate limiting
rate_key = f"api:{key_data['id']}"
allowed, info = await check_rate_limit(rate_key, limit=1000, window=3600)
if not allowed:
response = jsonify({"error": "Rate limit exceeded", **info})
response.headers["X-RateLimit-Limit"] = str(info["limit"])
response.headers["X-RateLimit-Remaining"] = str(info["remaining"])
response.headers["X-RateLimit-Reset"] = str(info["reset"])
return response, 429
await log_api_request(key_data["user_id"], request.path, request.method)
g.api_key = key_data
g.user_id = key_data["user_id"]
g.plan = plan
return await f(*args, **kwargs)
return decorated
return decorator
# =============================================================================
# Routes
# =============================================================================
@bp.route("/me")
@api_key_required()
async def me():
"""Get current user info."""
return jsonify({
"user_id": g.user_id,
"email": g.api_key["email"],
"plan": g.plan,
"key_name": g.api_key["name"],
"scopes": g.api_key["scopes"].split(","),
})
@bp.route("/commodities")
@api_key_required(scopes=["read"])
async def list_commodities():
"""List available commodities."""
commodities = await analytics.get_available_commodities()
return jsonify({"commodities": commodities})
@bp.route("/commodities/<int:code>/metrics")
@api_key_required(scopes=["read"])
async def commodity_metrics(code: int):
"""Time series metrics for a commodity. Query params: metrics, start_year, end_year."""
raw_metrics = request.args.getlist("metrics") or ["Production", "Exports", "Imports", "Ending_Stocks"]
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_METRICS]
if not metrics:
return jsonify({"error": f"No valid metrics. Allowed: {sorted(analytics.ALLOWED_METRICS)}"}), 400
start_year = request.args.get("start_year", type=int)
end_year = request.args.get("end_year", type=int)
data = await analytics.get_global_time_series(code, metrics, start_year, end_year)
return jsonify({"commodity_code": code, "metrics": metrics, "data": data})
@bp.route("/commodities/<int:code>/countries")
@api_key_required(scopes=["read"])
async def commodity_countries(code: int):
"""Country ranking for a commodity. Query params: metric, limit."""
metric = request.args.get("metric", "Production")
if metric not in analytics.ALLOWED_METRICS:
return jsonify({"error": f"Invalid metric. Allowed: {sorted(analytics.ALLOWED_METRICS)}"}), 400
limit = min(int(request.args.get("limit", 20)), 100)
data = await analytics.get_top_countries(code, metric, limit)
return jsonify({"commodity_code": code, "metric": metric, "data": data})
@bp.route("/commodities/<code>/positioning")
@api_key_required(scopes=["read"])
async def commodity_positioning(code: str):
"""COT trader positioning time series for a commodity.
Query params:
metrics — repeated param, e.g. ?metrics=managed_money_net&metrics=cot_index_26w
start_date — ISO date filter (YYYY-MM-DD)
end_date — ISO date filter (YYYY-MM-DD)
limit — max rows returned (default 260, max 2000)
"""
raw_metrics = request.args.getlist("metrics") or [
"managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w"
]
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_COT_METRICS]
if not metrics:
return jsonify({"error": f"No valid metrics. Allowed: {sorted(analytics.ALLOWED_COT_METRICS)}"}), 400
start_date = request.args.get("start_date")
end_date = request.args.get("end_date")
limit = min(int(request.args.get("limit", 260)), 2000)
data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit)
return jsonify({"cftc_commodity_code": code, "metrics": metrics, "data": data})
@bp.route("/commodities/<code>/positioning/latest")
@api_key_required(scopes=["read"])
async def commodity_positioning_latest(code: str):
"""Latest week's full COT positioning snapshot for a commodity."""
data = await analytics.get_cot_positioning_latest(code)
if not data:
return jsonify({"error": "No positioning data found for this commodity"}), 404
return jsonify({"cftc_commodity_code": code, "data": data})
@bp.route("/commodities/<int:code>/metrics.csv")
@api_key_required(scopes=["read"])
async def commodity_metrics_csv(code: int):
"""CSV export of time series metrics."""
if g.plan == "free":
return jsonify({"error": "CSV export requires a Starter or Pro plan"}), 403
raw_metrics = request.args.getlist("metrics") or [
"Production", "Exports", "Imports", "Ending_Stocks", "Total_Distribution",
]
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_METRICS]
if not metrics:
return jsonify({"error": "No valid metrics"}), 400
data = await analytics.get_global_time_series(code, metrics)
output = io.StringIO()
if data:
writer = csv.DictWriter(output, fieldnames=data[0].keys())
writer.writeheader()
writer.writerows(data)
return Response(
output.getvalue(),
mimetype="text/csv",
headers={"Content-Disposition": f"attachment; filename=commodity_{code}_metrics.csv"},
)