- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization - SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart) - dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology - fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities - obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios - Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest] - Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart - 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
228 lines
8.2 KiB
Python
228 lines
8.2 KiB
Python
"""
|
|
API domain: REST API for commodity analytics with key authentication and rate limiting.
|
|
"""
|
|
import csv
|
|
import hashlib
|
|
import io
|
|
from datetime import datetime
|
|
from functools import wraps
|
|
|
|
from quart import Blueprint, Response, g, jsonify, request
|
|
|
|
from .. import analytics
|
|
from ..core import check_rate_limit, execute, fetch_one
|
|
|
|
bp = Blueprint("api", __name__)
|
|
|
|
|
|
# =============================================================================
|
|
# SQL Queries
|
|
# =============================================================================
|
|
|
|
async def verify_api_key(raw_key: str) -> dict | None:
|
|
"""Verify API key and return key data with user info."""
|
|
key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
|
|
|
|
result = await fetch_one(
|
|
"""
|
|
SELECT ak.*, u.email, u.id as user_id,
|
|
COALESCE(s.plan, 'free') as plan
|
|
FROM api_keys ak
|
|
JOIN users u ON u.id = ak.user_id
|
|
LEFT JOIN subscriptions s ON s.user_id = u.id AND s.status = 'active'
|
|
WHERE ak.key_hash = ? AND ak.deleted_at IS NULL AND u.deleted_at IS NULL
|
|
""",
|
|
(key_hash,),
|
|
)
|
|
|
|
if result:
|
|
await execute(
|
|
"UPDATE api_keys SET last_used_at = ? WHERE id = ?",
|
|
(datetime.utcnow().isoformat(), result["id"]),
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
async def log_api_request(user_id: int, endpoint: str, method: str) -> None:
|
|
"""Log API request for analytics and rate limiting."""
|
|
await execute(
|
|
"""
|
|
INSERT INTO api_requests (user_id, endpoint, method, created_at)
|
|
VALUES (?, ?, ?, ?)
|
|
""",
|
|
(user_id, endpoint, method, datetime.utcnow().isoformat()),
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Decorators
|
|
# =============================================================================
|
|
|
|
def api_key_required(scopes: list[str] = None):
|
|
"""Require valid API key with optional scope check."""
|
|
def decorator(f):
|
|
@wraps(f)
|
|
async def decorated(*args, **kwargs):
|
|
auth = request.headers.get("Authorization", "")
|
|
if not auth.startswith("Bearer "):
|
|
return jsonify({"error": "Missing API key"}), 401
|
|
|
|
raw_key = auth[7:]
|
|
key_data = await verify_api_key(raw_key)
|
|
|
|
if not key_data:
|
|
return jsonify({"error": "Invalid API key"}), 401
|
|
|
|
# Check scopes
|
|
if scopes:
|
|
key_scopes = (key_data.get("scopes") or "").split(",")
|
|
if not any(s in key_scopes for s in scopes):
|
|
return jsonify({"error": "Insufficient permissions"}), 403
|
|
|
|
# Check plan allows API access
|
|
plan = key_data.get("plan", "free")
|
|
if plan == "free":
|
|
return jsonify({"error": "API access requires a Starter or Pro plan"}), 403
|
|
|
|
# Rate limiting
|
|
rate_key = f"api:{key_data['id']}"
|
|
allowed, info = await check_rate_limit(rate_key, limit=1000, window=3600)
|
|
|
|
if not allowed:
|
|
response = jsonify({"error": "Rate limit exceeded", **info})
|
|
response.headers["X-RateLimit-Limit"] = str(info["limit"])
|
|
response.headers["X-RateLimit-Remaining"] = str(info["remaining"])
|
|
response.headers["X-RateLimit-Reset"] = str(info["reset"])
|
|
return response, 429
|
|
|
|
await log_api_request(key_data["user_id"], request.path, request.method)
|
|
|
|
g.api_key = key_data
|
|
g.user_id = key_data["user_id"]
|
|
g.plan = plan
|
|
|
|
return await f(*args, **kwargs)
|
|
return decorated
|
|
return decorator
|
|
|
|
|
|
# =============================================================================
|
|
# Routes
|
|
# =============================================================================
|
|
|
|
@bp.route("/me")
|
|
@api_key_required()
|
|
async def me():
|
|
"""Get current user info."""
|
|
return jsonify({
|
|
"user_id": g.user_id,
|
|
"email": g.api_key["email"],
|
|
"plan": g.plan,
|
|
"key_name": g.api_key["name"],
|
|
"scopes": g.api_key["scopes"].split(","),
|
|
})
|
|
|
|
|
|
@bp.route("/commodities")
|
|
@api_key_required(scopes=["read"])
|
|
async def list_commodities():
|
|
"""List available commodities."""
|
|
commodities = await analytics.get_available_commodities()
|
|
return jsonify({"commodities": commodities})
|
|
|
|
|
|
@bp.route("/commodities/<int:code>/metrics")
|
|
@api_key_required(scopes=["read"])
|
|
async def commodity_metrics(code: int):
|
|
"""Time series metrics for a commodity. Query params: metrics, start_year, end_year."""
|
|
raw_metrics = request.args.getlist("metrics") or ["Production", "Exports", "Imports", "Ending_Stocks"]
|
|
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_METRICS]
|
|
if not metrics:
|
|
return jsonify({"error": f"No valid metrics. Allowed: {sorted(analytics.ALLOWED_METRICS)}"}), 400
|
|
|
|
start_year = request.args.get("start_year", type=int)
|
|
end_year = request.args.get("end_year", type=int)
|
|
|
|
data = await analytics.get_global_time_series(code, metrics, start_year, end_year)
|
|
return jsonify({"commodity_code": code, "metrics": metrics, "data": data})
|
|
|
|
|
|
@bp.route("/commodities/<int:code>/countries")
|
|
@api_key_required(scopes=["read"])
|
|
async def commodity_countries(code: int):
|
|
"""Country ranking for a commodity. Query params: metric, limit."""
|
|
metric = request.args.get("metric", "Production")
|
|
if metric not in analytics.ALLOWED_METRICS:
|
|
return jsonify({"error": f"Invalid metric. Allowed: {sorted(analytics.ALLOWED_METRICS)}"}), 400
|
|
|
|
limit = min(int(request.args.get("limit", 20)), 100)
|
|
|
|
data = await analytics.get_top_countries(code, metric, limit)
|
|
return jsonify({"commodity_code": code, "metric": metric, "data": data})
|
|
|
|
|
|
@bp.route("/commodities/<code>/positioning")
|
|
@api_key_required(scopes=["read"])
|
|
async def commodity_positioning(code: str):
|
|
"""COT trader positioning time series for a commodity.
|
|
|
|
Query params:
|
|
metrics — repeated param, e.g. ?metrics=managed_money_net&metrics=cot_index_26w
|
|
start_date — ISO date filter (YYYY-MM-DD)
|
|
end_date — ISO date filter (YYYY-MM-DD)
|
|
limit — max rows returned (default 260, max 2000)
|
|
"""
|
|
raw_metrics = request.args.getlist("metrics") or [
|
|
"managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w"
|
|
]
|
|
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_COT_METRICS]
|
|
if not metrics:
|
|
return jsonify({"error": f"No valid metrics. Allowed: {sorted(analytics.ALLOWED_COT_METRICS)}"}), 400
|
|
|
|
start_date = request.args.get("start_date")
|
|
end_date = request.args.get("end_date")
|
|
limit = min(int(request.args.get("limit", 260)), 2000)
|
|
|
|
data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit)
|
|
return jsonify({"cftc_commodity_code": code, "metrics": metrics, "data": data})
|
|
|
|
|
|
@bp.route("/commodities/<code>/positioning/latest")
|
|
@api_key_required(scopes=["read"])
|
|
async def commodity_positioning_latest(code: str):
|
|
"""Latest week's full COT positioning snapshot for a commodity."""
|
|
data = await analytics.get_cot_positioning_latest(code)
|
|
if not data:
|
|
return jsonify({"error": "No positioning data found for this commodity"}), 404
|
|
return jsonify({"cftc_commodity_code": code, "data": data})
|
|
|
|
|
|
@bp.route("/commodities/<int:code>/metrics.csv")
|
|
@api_key_required(scopes=["read"])
|
|
async def commodity_metrics_csv(code: int):
|
|
"""CSV export of time series metrics."""
|
|
if g.plan == "free":
|
|
return jsonify({"error": "CSV export requires a Starter or Pro plan"}), 403
|
|
|
|
raw_metrics = request.args.getlist("metrics") or [
|
|
"Production", "Exports", "Imports", "Ending_Stocks", "Total_Distribution",
|
|
]
|
|
metrics = [m for m in raw_metrics if m in analytics.ALLOWED_METRICS]
|
|
if not metrics:
|
|
return jsonify({"error": "No valid metrics"}), 400
|
|
|
|
data = await analytics.get_global_time_series(code, metrics)
|
|
|
|
output = io.StringIO()
|
|
if data:
|
|
writer = csv.DictWriter(output, fieldnames=data[0].keys())
|
|
writer.writeheader()
|
|
writer.writerows(data)
|
|
|
|
return Response(
|
|
output.getvalue(),
|
|
mimetype="text/csv",
|
|
headers={"Content-Disposition": f"attachment; filename=commodity_{code}_metrics.csv"},
|
|
)
|