merge: CFTC COT combined (futures+options) report — extractor, transform, web toggle

This commit is contained in:
Deeman
2026-02-26 11:29:20 +01:00
12 changed files with 325 additions and 39 deletions

View File

@@ -10,6 +10,7 @@ dependencies = [
[project.scripts] [project.scripts]
extract_cot = "cftc_cot.execute:extract_cot_dataset" extract_cot = "cftc_cot.execute:extract_cot_dataset"
extract_cot_combined = "cftc_cot.execute:extract_cot_combined"
[build-system] [build-system]
requires = ["hatchling"] requires = ["hatchling"]

View File

@@ -1,11 +1,13 @@
"""CFTC COT Disaggregated Futures data extraction. """CFTC COT Disaggregated data extraction.
Downloads yearly ZIP files from CFTC and stores as gzip CSV in the landing Downloads yearly ZIP files from CFTC and stores as gzip CSV in the landing
directory. CFTC publishes one file per year that updates every Friday at directory. CFTC publishes one file per year that updates every Friday at
3:30 PM ET. On first run this backfills all years from 2006. On subsequent 3:30 PM ET. On first run this backfills all years from 2006. On subsequent
runs it skips files whose etag matches what is already on disk. runs it skips files whose etag matches what is already on disk.
Landing path: LANDING_DIR/cot/{year}/{etag}.csv.gzip Two report variants are supported:
- Futures-only: Landing path: LANDING_DIR/cot/{year}/{etag}.csv.gzip
- Combined (fut+options): Landing path: LANDING_DIR/cot_combined/{year}/{etag}.csv.gzip
""" """
import logging import logging
@@ -37,9 +39,10 @@ logger = logging.getLogger("CFTC COT Extractor")
LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing")) LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing"))
# CFTC publishes yearly ZIPs for the disaggregated futures-only report. # CFTC publishes yearly ZIPs for both variants of the disaggregated report.
# The file for the current year is updated each Friday at 3:30 PM ET. # The file for the current year is updated each Friday at 3:30 PM ET.
COT_URL_TEMPLATE = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip" COT_URL_FUTURES_ONLY = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip"
COT_URL_COMBINED = "https://www.cftc.gov/files/dea/history/com_disagg_txt_{year}.zip"
FIRST_YEAR = 2006 # Disaggregated report starts June 2006 FIRST_YEAR = 2006 # Disaggregated report starts June 2006
HTTP_TIMEOUT_SECONDS = 120 # COT ZIPs are up to ~30 MB HTTP_TIMEOUT_SECONDS = 120 # COT ZIPs are up to ~30 MB
@@ -60,12 +63,12 @@ def _synthetic_etag(year: int, headers: dict) -> str:
return etag return etag
def extract_cot_year(year: int, http_session: niquests.Session) -> int: def extract_cot_year(year: int, http_session: niquests.Session, url_template: str, landing_subdir: str) -> int:
"""Download and store COT data for a single year. """Download and store COT data for a single year.
Returns bytes_written (0 if skipped or unavailable). Returns bytes_written (0 if skipped or unavailable).
""" """
url = COT_URL_TEMPLATE.format(year=year) url = url_template.format(year=year)
logger.info(f"Checking COT data for {year}: {url}") logger.info(f"Checking COT data for {year}: {url}")
head = http_session.head(url, timeout=HTTP_TIMEOUT_SECONDS) head = http_session.head(url, timeout=HTTP_TIMEOUT_SECONDS)
@@ -79,7 +82,7 @@ def extract_cot_year(year: int, http_session: niquests.Session) -> int:
raw_etag = head.headers.get("etag", "") raw_etag = head.headers.get("etag", "")
etag = normalize_etag(raw_etag) if raw_etag else _synthetic_etag(year, head.headers) etag = normalize_etag(raw_etag) if raw_etag else _synthetic_etag(year, head.headers)
dest_dir = landing_path(LANDING_DIR, "cot", str(year)) dest_dir = landing_path(LANDING_DIR, landing_subdir, str(year))
local_file = dest_dir / f"{etag}.csv.gzip" local_file = dest_dir / f"{etag}.csv.gzip"
if local_file.exists(): if local_file.exists():
@@ -104,8 +107,8 @@ def extract_cot_year(year: int, http_session: niquests.Session) -> int:
return bytes_written return bytes_written
def extract_cot_dataset(): def _extract_cot(url_template: str, landing_subdir: str, extractor_name: str) -> None:
"""Extract all available CFTC COT disaggregated futures data. """Shared extraction loop for any COT report variant.
Downloads current year first (always re-checks for weekly Friday updates), Downloads current year first (always re-checks for weekly Friday updates),
then backfills historical years. Bounded to MAX_YEARS. Continues on then backfills historical years. Bounded to MAX_YEARS. Continues on
@@ -119,7 +122,7 @@ def extract_cot_dataset():
) )
conn = open_state_db(LANDING_DIR) conn = open_state_db(LANDING_DIR)
run_id = start_run(conn, "cftc_cot") run_id = start_run(conn, extractor_name)
files_written = 0 files_written = 0
files_skipped = 0 files_skipped = 0
bytes_written_total = 0 bytes_written_total = 0
@@ -127,7 +130,7 @@ def extract_cot_dataset():
with niquests.Session() as session: with niquests.Session() as session:
for year in years: for year in years:
try: try:
result = extract_cot_year(year, session) result = extract_cot_year(year, session, url_template, landing_subdir)
if result > 0: if result > 0:
files_written += 1 files_written += 1
bytes_written_total += result bytes_written_total += result
@@ -136,7 +139,7 @@ def extract_cot_dataset():
except Exception: except Exception:
logger.exception(f"Failed to extract COT data for {year}, continuing") logger.exception(f"Failed to extract COT data for {year}, continuing")
logger.info(f"COT extraction complete: {files_written} new file(s) downloaded") logger.info(f"COT extraction complete ({extractor_name}): {files_written} new file(s) downloaded")
end_run( end_run(
conn, run_id, status="success", conn, run_id, status="success",
files_written=files_written, files_skipped=files_skipped, files_written=files_written, files_skipped=files_skipped,
@@ -150,5 +153,15 @@ def extract_cot_dataset():
conn.close() conn.close()
def extract_cot_dataset():
"""Extract CFTC COT disaggregated futures-only report."""
_extract_cot(COT_URL_FUTURES_ONLY, "cot", "cftc_cot")
def extract_cot_combined():
"""Extract CFTC COT disaggregated combined (futures+options) report."""
_extract_cot(COT_URL_COMBINED, "cot_combined", "cftc_cot_combined")
if __name__ == "__main__": if __name__ == "__main__":
extract_cot_dataset() extract_cot_dataset()

View File

@@ -20,6 +20,10 @@ PIPELINES = {
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot"], "command": ["uv", "run", "--package", "cftc_cot", "extract_cot"],
"timeout_seconds": 1800, "timeout_seconds": 1800,
}, },
"extract_cot_combined": {
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot_combined"],
"timeout_seconds": 1800,
},
"extract_prices": { "extract_prices": {
"command": ["uv", "run", "--package", "coffee_prices", "extract_prices"], "command": ["uv", "run", "--package", "coffee_prices", "extract_prices"],
"timeout_seconds": 300, "timeout_seconds": 300,
@@ -49,7 +53,7 @@ PIPELINES = {
"timeout_seconds": 120, "timeout_seconds": 120,
}, },
"extract_all": { "extract_all": {
"command": ["meta", "extract", "extract_cot", "extract_prices", "extract_ice_all", "extract_weather"], "command": ["meta", "extract", "extract_cot", "extract_cot_combined", "extract_prices", "extract_ice_all", "extract_weather"],
"timeout_seconds": 6600, "timeout_seconds": 6600,
}, },
"transform": { "transform": {
@@ -68,7 +72,7 @@ PIPELINES = {
META_PIPELINES: dict[str, list[str]] = { META_PIPELINES: dict[str, list[str]] = {
"extract_all": ["extract", "extract_cot", "extract_prices", "extract_ice_all", "extract_weather"], "extract_all": ["extract", "extract_cot", "extract_cot_combined", "extract_prices", "extract_ice_all", "extract_weather"],
} }

View File

@@ -17,6 +17,13 @@ def cot_glob(evaluator) -> str:
return f"'{landing_dir}/cot/**/*.csv.gzip'" return f"'{landing_dir}/cot/**/*.csv.gzip'"
@macro()
def cot_combined_glob(evaluator) -> str:
"""Return a quoted glob path for all COT combined (futures+options) CSV gzip files under LANDING_DIR."""
landing_dir = evaluator.var("LANDING_DIR") or os.environ.get("LANDING_DIR", "data/landing")
return f"'{landing_dir}/cot_combined/**/*.csv.gzip'"
@macro() @macro()
def prices_glob(evaluator) -> str: def prices_glob(evaluator) -> str:
"""Return a quoted glob path for all coffee price CSV gzip files under LANDING_DIR.""" """Return a quoted glob path for all coffee price CSV gzip files under LANDING_DIR."""

View File

@@ -4,7 +4,7 @@ MODEL (
kind INCREMENTAL_BY_TIME_RANGE ( kind INCREMENTAL_BY_TIME_RANGE (
time_column report_date time_column report_date
), ),
grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date), grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date, report_type),
start '2006-06-13', start '2006-06-13',
cron '@daily' cron '@daily'
); );
@@ -21,6 +21,18 @@ WITH src AS (
all_varchar = TRUE, all_varchar = TRUE,
max_line_size = 10000000 max_line_size = 10000000
) )
UNION ALL BY NAME
SELECT
*
FROM READ_CSV(
@cot_combined_glob(),
compression = 'gzip',
header = TRUE,
union_by_name = TRUE,
filename = TRUE,
all_varchar = TRUE,
max_line_size = 10000000
)
), cast_and_clean AS ( ), cast_and_clean AS (
SELECT SELECT
TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */ TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */
@@ -28,6 +40,7 @@ WITH src AS (
TRIM(cftc_commodity_code) AS cftc_commodity_code, TRIM(cftc_commodity_code) AS cftc_commodity_code,
TRIM(cftc_contract_market_code) AS cftc_contract_market_code, TRIM(cftc_contract_market_code) AS cftc_contract_market_code,
TRIM(contract_units) AS contract_units, TRIM(contract_units) AS contract_units,
TRIM("FutOnly_or_Combined") AS report_type, /* 'FutOnly' or 'Combined' — discriminates the two CFTC report variants */
TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */ TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */
TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */ TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */
TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short, TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short,
@@ -66,12 +79,13 @@ WITH src AS (
cftc_commodity_code, cftc_commodity_code,
"Report_Date_as_YYYY-MM-DD", "Report_Date_as_YYYY-MM-DD",
cftc_contract_market_code, cftc_contract_market_code,
"FutOnly_or_Combined",
open_interest_all, open_interest_all,
m_money_positions_long_all, m_money_positions_long_all,
m_money_positions_short_all, m_money_positions_short_all,
prod_merc_positions_long_all, prod_merc_positions_long_all,
prod_merc_positions_short_all prod_merc_positions_short_all
) AS hkey /* Dedup key: hash of business grain + key metrics */ ) AS hkey /* Dedup key: hash of business grain + key metrics; includes report variant so fut-only and combined rows get distinct keys */
FROM src FROM src
/* Reject rows with null commodity code or malformed date */ /* Reject rows with null commodity code or malformed date */
WHERE WHERE
@@ -119,6 +133,7 @@ WITH src AS (
ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short, ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short,
ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread, ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread,
ANY_VALUE(ingest_date) AS ingest_date, ANY_VALUE(ingest_date) AS ingest_date,
ANY_VALUE(report_type) AS report_type,
hkey hkey
FROM cast_and_clean FROM cast_and_clean
GROUP BY GROUP BY

View File

@@ -20,6 +20,7 @@ WITH latest_revision AS (
ON f.cftc_commodity_code = d.cftc_commodity_code ON f.cftc_commodity_code = d.cftc_commodity_code
WHERE WHERE
d.commodity_name = 'Coffee, Green' d.commodity_name = 'Coffee, Green'
AND f.report_type = 'FutOnly'
AND f.report_date BETWEEN @start_ds AND @end_ds AND f.report_date BETWEEN @start_ds AND @end_ds
QUALIFY QUALIFY
ROW_NUMBER() OVER ( ROW_NUMBER() OVER (

View File

@@ -0,0 +1,148 @@
/* Serving mart: COT positioning (combined futures+options) for Coffee C futures. */ /* Same analytics as serving.cot_positioning, but filtered to the combined */ /* report variant (FutOnly_or_Combined = 'Combined'). Positions include */ /* options delta-equivalent exposure, showing total directional market bet. */ /* Grain: one row per report_date for Coffee C futures. */ /* Latest revision per date: MAX(ingest_date) used to deduplicate CFTC corrections. */
MODEL (
name serving.cot_positioning_combined,
kind INCREMENTAL_BY_TIME_RANGE (
time_column report_date
),
grain (
report_date
),
start '2006-06-13',
cron '@daily'
);
WITH latest_revision AS (
/* Pick the most recently ingested row when CFTC issues corrections */
SELECT
f.*
FROM foundation.fct_cot_positioning AS f
INNER JOIN foundation.dim_commodity AS d
ON f.cftc_commodity_code = d.cftc_commodity_code
WHERE
d.commodity_name = 'Coffee, Green'
AND f.report_type = 'Combined'
AND f.report_date BETWEEN @start_ds AND @end_ds
QUALIFY
ROW_NUMBER() OVER (
PARTITION BY f.report_date, f.cftc_contract_market_code
ORDER BY f.ingest_date DESC
) = 1
), with_derived AS (
SELECT
report_date,
market_and_exchange_name,
cftc_commodity_code,
cftc_contract_market_code,
contract_units,
ingest_date,
open_interest, /* Absolute positions (contracts, delta-equivalent for options) */
managed_money_long,
managed_money_short,
managed_money_spread,
managed_money_net,
prod_merc_long,
prod_merc_short,
prod_merc_net,
swap_long,
swap_short,
swap_spread,
swap_net,
other_reportable_long,
other_reportable_short,
other_reportable_spread,
other_reportable_net,
nonreportable_long,
nonreportable_short,
nonreportable_net,
ROUND(managed_money_net::REAL / NULLIF(open_interest, 0) * 100, 2) AS managed_money_net_pct_of_oi, /* Normalized: managed money net as % of open interest */ /* Removes size effects and makes cross-period comparison meaningful */
ROUND(managed_money_long::REAL / NULLIF(managed_money_short, 0), 3) AS managed_money_long_short_ratio, /* Long/short ratio: >1 = more bulls than bears in managed money */
change_open_interest, /* Weekly changes */
change_managed_money_long,
change_managed_money_short,
change_managed_money_net,
change_prod_merc_long,
change_prod_merc_short,
managed_money_net /* Week-over-week momentum in managed money net (via LAG) */ - LAG(managed_money_net, 1) OVER (ORDER BY report_date) AS managed_money_net_wow,
concentration_top4_long_pct, /* Concentration */
concentration_top4_short_pct,
concentration_top8_long_pct,
concentration_top8_short_pct,
traders_total, /* Trader counts */
traders_managed_money_long,
traders_managed_money_short,
traders_managed_money_spread,
CASE
WHEN MAX(managed_money_net) OVER w26 = MIN(managed_money_net) OVER w26
THEN 50.0
ELSE ROUND(
(
managed_money_net - MIN(managed_money_net) OVER w26
)::REAL / (
MAX(managed_money_net) OVER w26 - MIN(managed_money_net) OVER w26
) * 100,
1
)
END AS cot_index_26w, /* COT Index (26-week): where is current net vs. trailing 26 weeks? */ /* 0 = most bearish extreme, 100 = most bullish extreme */ /* Includes options delta-equivalent exposure */
CASE
WHEN MAX(managed_money_net) OVER w52 = MIN(managed_money_net) OVER w52
THEN 50.0
ELSE ROUND(
(
managed_money_net - MIN(managed_money_net) OVER w52
)::REAL / (
MAX(managed_money_net) OVER w52 - MIN(managed_money_net) OVER w52
) * 100,
1
)
END AS cot_index_52w /* COT Index (52-week): longer-term positioning context */
FROM latest_revision
WINDOW w26 AS (ORDER BY report_date ROWS BETWEEN 25 PRECEDING AND CURRENT ROW), w52 AS (ORDER BY report_date ROWS BETWEEN 51 PRECEDING AND CURRENT ROW)
)
SELECT
report_date,
market_and_exchange_name,
cftc_commodity_code,
cftc_contract_market_code,
contract_units,
ingest_date,
open_interest,
managed_money_long,
managed_money_short,
managed_money_spread,
managed_money_net,
prod_merc_long,
prod_merc_short,
prod_merc_net,
swap_long,
swap_short,
swap_spread,
swap_net,
other_reportable_long,
other_reportable_short,
other_reportable_spread,
other_reportable_net,
nonreportable_long,
nonreportable_short,
nonreportable_net,
managed_money_net_pct_of_oi,
managed_money_long_short_ratio,
change_open_interest,
change_managed_money_long,
change_managed_money_short,
change_managed_money_net,
change_prod_merc_long,
change_prod_merc_short,
managed_money_net_wow,
concentration_top4_long_pct,
concentration_top4_short_pct,
concentration_top8_long_pct,
concentration_top8_short_pct,
traders_total,
traders_managed_money_long,
traders_managed_money_short,
traders_managed_money_spread,
cot_index_26w,
cot_index_52w
FROM with_derived
ORDER BY
report_date

View File

@@ -302,17 +302,23 @@ def _validate_cot_metrics(metrics: list[str]) -> list[str]:
return valid return valid
def _cot_table(combined: bool) -> str:
return "serving.cot_positioning_combined" if combined else "serving.cot_positioning"
async def get_cot_positioning_time_series( async def get_cot_positioning_time_series(
cftc_commodity_code: str, cftc_commodity_code: str,
metrics: list[str], metrics: list[str],
start_date: str | None = None, start_date: str | None = None,
end_date: str | None = None, end_date: str | None = None,
limit: int = 520, limit: int = 520,
combined: bool = False,
) -> list[dict]: ) -> list[dict]:
"""Weekly COT positioning time series. limit defaults to ~10 years of weekly data.""" """Weekly COT positioning time series. limit defaults to ~10 years of weekly data."""
assert 1 <= limit <= 2000, "limit must be between 1 and 2000" assert 1 <= limit <= 2000, "limit must be between 1 and 2000"
metrics = _validate_cot_metrics(metrics) metrics = _validate_cot_metrics(metrics)
cols = ", ".join(metrics) cols = ", ".join(metrics)
table = _cot_table(combined)
where_parts = ["cftc_commodity_code = ?"] where_parts = ["cftc_commodity_code = ?"]
params: list = [cftc_commodity_code] params: list = [cftc_commodity_code]
@@ -329,7 +335,7 @@ async def get_cot_positioning_time_series(
return await fetch_analytics( return await fetch_analytics(
f""" f"""
SELECT report_date, {cols} SELECT report_date, {cols}
FROM serving.cot_positioning FROM {table}
WHERE {where_clause} WHERE {where_clause}
ORDER BY report_date ASC ORDER BY report_date ASC
LIMIT ? LIMIT ?
@@ -338,12 +344,13 @@ async def get_cot_positioning_time_series(
) )
async def get_cot_positioning_latest(cftc_commodity_code: str) -> dict | None: async def get_cot_positioning_latest(cftc_commodity_code: str, combined: bool = False) -> dict | None:
"""Latest week's full COT positioning snapshot.""" """Latest week's full COT positioning snapshot."""
table = _cot_table(combined)
rows = await fetch_analytics( rows = await fetch_analytics(
""" f"""
SELECT * SELECT *
FROM serving.cot_positioning FROM {table}
WHERE cftc_commodity_code = ? WHERE cftc_commodity_code = ?
ORDER BY report_date DESC ORDER BY report_date DESC
LIMIT 1 LIMIT 1
@@ -356,14 +363,16 @@ async def get_cot_positioning_latest(cftc_commodity_code: str) -> dict | None:
async def get_cot_index_trend( async def get_cot_index_trend(
cftc_commodity_code: str, cftc_commodity_code: str,
weeks: int = 104, weeks: int = 104,
combined: bool = False,
) -> list[dict]: ) -> list[dict]:
"""COT Index time series (26w and 52w) for the trailing N weeks.""" """COT Index time series (26w and 52w) for the trailing N weeks."""
assert 1 <= weeks <= 1040, "weeks must be between 1 and 1040" assert 1 <= weeks <= 1040, "weeks must be between 1 and 1040"
table = _cot_table(combined)
return await fetch_analytics( return await fetch_analytics(
""" f"""
SELECT report_date, cot_index_26w, cot_index_52w, SELECT report_date, cot_index_26w, cot_index_52w,
managed_money_net, managed_money_net_pct_of_oi managed_money_net, managed_money_net_pct_of_oi
FROM serving.cot_positioning FROM {table}
WHERE cftc_commodity_code = ? WHERE cftc_commodity_code = ?
ORDER BY report_date DESC ORDER BY report_date DESC
LIMIT ? LIMIT ?
@@ -372,6 +381,30 @@ async def get_cot_index_trend(
) )
async def get_cot_options_delta(cftc_commodity_code: str) -> dict | None:
"""Latest managed_money_net difference between combined and futures-only reports.
Shows whether the options book is reinforcing (same direction) or hedging
(opposite direction) the futures position. Returns None if either table
has no data.
"""
rows = await fetch_analytics(
"""
SELECT f.report_date,
f.managed_money_net AS fut_net,
c.managed_money_net AS combined_net,
c.managed_money_net - f.managed_money_net AS options_delta
FROM serving.cot_positioning f
JOIN serving.cot_positioning_combined c USING (report_date)
WHERE f.cftc_commodity_code = ?
ORDER BY f.report_date DESC
LIMIT 1
""",
[cftc_commodity_code],
)
return rows[0] if rows else None
# ============================================================================= # =============================================================================
# Coffee Prices Queries # Coffee Prices Queries
# ============================================================================= # =============================================================================

View File

@@ -172,6 +172,7 @@ async def commodity_positioning(code: str):
start_date — ISO date filter (YYYY-MM-DD) start_date — ISO date filter (YYYY-MM-DD)
end_date — ISO date filter (YYYY-MM-DD) end_date — ISO date filter (YYYY-MM-DD)
limit — max rows returned (default 260, max 2000) limit — max rows returned (default 260, max 2000)
type — report variant: "fut" (futures-only, default) or "combined" (futures+options)
""" """
raw_metrics = request.args.getlist("metrics") or [ raw_metrics = request.args.getlist("metrics") or [
"managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w" "managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w"
@@ -183,19 +184,27 @@ async def commodity_positioning(code: str):
start_date = request.args.get("start_date") start_date = request.args.get("start_date")
end_date = request.args.get("end_date") end_date = request.args.get("end_date")
limit = min(int(request.args.get("limit", 260)), 2000) limit = min(int(request.args.get("limit", 260)), 2000)
cot_type = request.args.get("type", "fut")
combined = cot_type == "combined"
data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit) data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit, combined=combined)
return jsonify({"cftc_commodity_code": code, "metrics": metrics, "data": data}) return jsonify({"cftc_commodity_code": code, "type": cot_type, "metrics": metrics, "data": data})
@bp.route("/commodities/<code>/positioning/latest") @bp.route("/commodities/<code>/positioning/latest")
@api_key_required(scopes=["read"]) @api_key_required(scopes=["read"])
async def commodity_positioning_latest(code: str): async def commodity_positioning_latest(code: str):
"""Latest week's full COT positioning snapshot for a commodity.""" """Latest week's full COT positioning snapshot for a commodity.
data = await analytics.get_cot_positioning_latest(code)
Query params:
type — report variant: "fut" (futures-only, default) or "combined" (futures+options)
"""
cot_type = request.args.get("type", "fut")
combined = cot_type == "combined"
data = await analytics.get_cot_positioning_latest(code, combined=combined)
if not data: if not data:
return jsonify({"error": "No positioning data found for this commodity"}), 404 return jsonify({"error": "No positioning data found for this commodity"}), 404
return jsonify({"cftc_commodity_code": code, "data": data}) return jsonify({"cftc_commodity_code": code, "type": cot_type, "data": data})
@bp.route("/commodities/<code>/prices") @bp.route("/commodities/<code>/prices")

View File

@@ -257,30 +257,45 @@ async def positioning():
if range_key not in RANGE_MAP: if range_key not in RANGE_MAP:
range_key = "1y" range_key = "1y"
cot_type = request.args.get("type", "fut")
if cot_type not in ("fut", "combined"):
cot_type = "fut"
combined = cot_type == "combined"
rng = RANGE_MAP[range_key] rng = RANGE_MAP[range_key]
price_limit = rng["days"] price_limit = rng["days"]
cot_weeks = rng["weeks"] cot_weeks = rng["weeks"]
options_delta = None
if analytics._db_path: if analytics._db_path:
results = await asyncio.gather( gather_coros = [
analytics.get_price_latest(analytics.COFFEE_TICKER), analytics.get_price_latest(analytics.COFFEE_TICKER),
analytics.get_price_time_series(analytics.COFFEE_TICKER, limit=price_limit), analytics.get_price_time_series(analytics.COFFEE_TICKER, limit=price_limit),
analytics.get_cot_positioning_latest(analytics.COFFEE_CFTC_CODE), analytics.get_cot_positioning_latest(analytics.COFFEE_CFTC_CODE, combined=combined),
analytics.get_cot_index_trend(analytics.COFFEE_CFTC_CODE, weeks=cot_weeks), analytics.get_cot_index_trend(analytics.COFFEE_CFTC_CODE, weeks=cot_weeks, combined=combined),
return_exceptions=True, ]
) if combined:
defaults = [None, [], None, []] gather_coros.append(analytics.get_cot_options_delta(analytics.COFFEE_CFTC_CODE))
price_latest, price_series, cot_latest, cot_trend = _safe(results, defaults)
results = await asyncio.gather(*gather_coros, return_exceptions=True)
defaults = [None, [], None, [], None] if combined else [None, [], None, []]
safe_results = _safe(results, defaults)
price_latest, price_series, cot_latest, cot_trend = safe_results[:4]
if combined:
options_delta = safe_results[4]
else: else:
price_latest, price_series, cot_latest, cot_trend = None, [], None, [] price_latest, price_series, cot_latest, cot_trend = None, [], None, []
ctx = dict( ctx = dict(
plan=plan, plan=plan,
range_key=range_key, range_key=range_key,
cot_type=cot_type,
price_latest=price_latest, price_latest=price_latest,
price_series=price_series, price_series=price_series,
cot_latest=cot_latest, cot_latest=cot_latest,
cot_trend=cot_trend, cot_trend=cot_trend,
options_delta=options_delta,
) )
if request.headers.get("HX-Request"): if request.headers.get("HX-Request"):

View File

@@ -37,6 +37,15 @@
{% endfor %} {% endfor %}
</div> </div>
<!-- Report type toggle -->
<div class="filter-pills" id="type-pills" style="margin-left: 1.5rem; border-left: 1px solid var(--color-parchment); padding-left: 1.5rem;">
{% for val, label in [("fut", "Futures"), ("combined", "F+O Combined")] %}
<button type="button"
class="filter-pill {{ 'active' if cot_type == val }}"
onclick="setType('{{ val }}')">{{ label }}</button>
{% endfor %}
</div>
<!-- MA toggles (client-side only) --> <!-- MA toggles (client-side only) -->
<label class="filter-check"> <label class="filter-check">
<input type="checkbox" id="ma20-toggle" checked onchange="toggleMA('sma20')"> 20d MA <input type="checkbox" id="ma20-toggle" checked onchange="toggleMA('sma20')"> 20d MA
@@ -57,15 +66,32 @@
<script> <script>
var POSITIONING_URL = '{{ url_for("dashboard.positioning") }}'; var POSITIONING_URL = '{{ url_for("dashboard.positioning") }}';
var currentRange = {{ range_key | tojson }}; var currentRange = {{ range_key | tojson }};
var currentType = {{ cot_type | tojson }};
function _positioningUrl() {
return POSITIONING_URL + '?range=' + currentRange + '&type=' + currentType;
}
function setRange(val) { function setRange(val) {
currentRange = val; currentRange = val;
var url = POSITIONING_URL + '?range=' + currentRange; var url = _positioningUrl();
window.history.pushState({}, '', url); window.history.pushState({}, '', url);
document.getElementById('positioning-canvas').classList.add('canvas-loading'); document.getElementById('positioning-canvas').classList.add('canvas-loading');
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' }); htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
} }
function setType(val) {
currentType = val;
var url = _positioningUrl();
window.history.pushState({}, '', url);
document.getElementById('positioning-canvas').classList.add('canvas-loading');
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
// Sync type pill active state immediately (canvas swap will also re-sync)
document.querySelectorAll('#type-pills .filter-pill').forEach(function (btn) {
btn.classList.toggle('active', btn.textContent.trim() === (val === 'combined' ? 'F+O Combined' : 'Futures'));
});
}
// MA toggles: client-side only — update Chart.js dataset visibility // MA toggles: client-side only — update Chart.js dataset visibility
function toggleMA(key) { function toggleMA(key) {
var chart = Chart.getChart('priceChart'); var chart = Chart.getChart('priceChart');
@@ -93,7 +119,8 @@ document.addEventListener('htmx:afterSwap', function (e) {
window.addEventListener('popstate', function () { window.addEventListener('popstate', function () {
var p = new URLSearchParams(window.location.search); var p = new URLSearchParams(window.location.search);
currentRange = p.get('range') || '1y'; currentRange = p.get('range') || '1y';
var url = POSITIONING_URL + '?range=' + currentRange; currentType = p.get('type') || 'fut';
var url = _positioningUrl();
document.getElementById('positioning-canvas').classList.add('canvas-loading'); document.getElementById('positioning-canvas').classList.add('canvas-loading');
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' }); htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
}); });

View File

@@ -19,11 +19,16 @@
</div> </div>
<div class="metric-card"> <div class="metric-card">
<div class="metric-label">MM Net Position</div> <div class="metric-label">MM Net Position{% if cot_type == "combined" %} <span style="font-size:0.7em;opacity:0.7">F+O</span>{% endif %}</div>
<div class="metric-value {% if cot_latest and cot_latest.managed_money_net > 0 %}text-bean-green{% elif cot_latest and cot_latest.managed_money_net < 0 %}text-danger{% endif %}"> <div class="metric-value {% if cot_latest and cot_latest.managed_money_net > 0 %}text-bean-green{% elif cot_latest and cot_latest.managed_money_net < 0 %}text-danger{% endif %}">
{% if cot_latest %}{{ "{:+,d}".format(cot_latest.managed_money_net | int) }}{% else %}--{% endif %} {% if cot_latest %}{{ "{:+,d}".format(cot_latest.managed_money_net | int) }}{% else %}--{% endif %}
</div> </div>
<div class="metric-sub">contracts (long short)</div> <div class="metric-sub">contracts (long short)</div>
{% if options_delta %}
<div class="metric-sub" style="font-family: 'Commit Mono', ui-monospace, monospace; margin-top: 0.25rem; {{ 'color: var(--color-bean-green)' if options_delta.options_delta > 0 else 'color: var(--color-copper)' }}">
Opt &Delta;: {{ "{:+,d}".format(options_delta.options_delta | int) }}
</div>
{% endif %}
</div> </div>
<div class="metric-card"> <div class="metric-card">
@@ -64,8 +69,8 @@
<div class="cc-chart-card"> <div class="cc-chart-card">
<div class="cc-chart-top"> <div class="cc-chart-top">
<div> <div>
<div class="cc-chart-title">Managed Money Net Position + COT Index</div> <div class="cc-chart-title">Managed Money Net Position{{ " (F+O Combined)" if cot_type == "combined" else "" }} + COT Index</div>
<div class="cc-chart-meta">CFTC Commitment of Traders · weekly · area = net contracts · dashed = COT index (0100)</div> <div class="cc-chart-meta">CFTC Commitment of Traders · weekly · {{ "futures + options delta-adjusted" if cot_type == "combined" else "futures only" }} · area = net contracts · dashed = COT index (0100)</div>
</div> </div>
</div> </div>
<div class="cc-chart-body"> <div class="cc-chart-body">
@@ -94,13 +99,21 @@
<script> <script>
(function () { (function () {
var range = {{ range_key | tojson }}; var range = {{ range_key | tojson }};
var cotType = {{ cot_type | tojson }};
// Sync range pills // Sync range pills
document.querySelectorAll('#range-pills .filter-pill').forEach(function (btn) { document.querySelectorAll('#range-pills .filter-pill').forEach(function (btn) {
btn.classList.toggle('active', btn.textContent.trim().toLowerCase() === range); btn.classList.toggle('active', btn.textContent.trim().toLowerCase() === range);
}); });
// Sync type pills
document.querySelectorAll('#type-pills .filter-pill').forEach(function (btn) {
var label = cotType === 'combined' ? 'F+O Combined' : 'Futures';
btn.classList.toggle('active', btn.textContent.trim() === label);
});
if (typeof currentRange !== 'undefined') currentRange = range; if (typeof currentRange !== 'undefined') currentRange = range;
if (typeof currentType !== 'undefined') currentType = cotType;
var C = { var C = {
copper: '#B45309', green: '#15803D', roast: '#4A2C1A', stone: '#78716C', copper: '#B45309', green: '#15803D', roast: '#4A2C1A', stone: '#78716C',