merge: CFTC COT combined (futures+options) report — extractor, transform, web toggle
This commit is contained in:
@@ -10,6 +10,7 @@ dependencies = [
|
|||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
extract_cot = "cftc_cot.execute:extract_cot_dataset"
|
extract_cot = "cftc_cot.execute:extract_cot_dataset"
|
||||||
|
extract_cot_combined = "cftc_cot.execute:extract_cot_combined"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["hatchling"]
|
requires = ["hatchling"]
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
"""CFTC COT Disaggregated Futures data extraction.
|
"""CFTC COT Disaggregated data extraction.
|
||||||
|
|
||||||
Downloads yearly ZIP files from CFTC and stores as gzip CSV in the landing
|
Downloads yearly ZIP files from CFTC and stores as gzip CSV in the landing
|
||||||
directory. CFTC publishes one file per year that updates every Friday at
|
directory. CFTC publishes one file per year that updates every Friday at
|
||||||
3:30 PM ET. On first run this backfills all years from 2006. On subsequent
|
3:30 PM ET. On first run this backfills all years from 2006. On subsequent
|
||||||
runs it skips files whose etag matches what is already on disk.
|
runs it skips files whose etag matches what is already on disk.
|
||||||
|
|
||||||
Landing path: LANDING_DIR/cot/{year}/{etag}.csv.gzip
|
Two report variants are supported:
|
||||||
|
- Futures-only: Landing path: LANDING_DIR/cot/{year}/{etag}.csv.gzip
|
||||||
|
- Combined (fut+options): Landing path: LANDING_DIR/cot_combined/{year}/{etag}.csv.gzip
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
@@ -37,9 +39,10 @@ logger = logging.getLogger("CFTC COT Extractor")
|
|||||||
|
|
||||||
LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing"))
|
LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing"))
|
||||||
|
|
||||||
# CFTC publishes yearly ZIPs for the disaggregated futures-only report.
|
# CFTC publishes yearly ZIPs for both variants of the disaggregated report.
|
||||||
# The file for the current year is updated each Friday at 3:30 PM ET.
|
# The file for the current year is updated each Friday at 3:30 PM ET.
|
||||||
COT_URL_TEMPLATE = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip"
|
COT_URL_FUTURES_ONLY = "https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip"
|
||||||
|
COT_URL_COMBINED = "https://www.cftc.gov/files/dea/history/com_disagg_txt_{year}.zip"
|
||||||
|
|
||||||
FIRST_YEAR = 2006 # Disaggregated report starts June 2006
|
FIRST_YEAR = 2006 # Disaggregated report starts June 2006
|
||||||
HTTP_TIMEOUT_SECONDS = 120 # COT ZIPs are up to ~30 MB
|
HTTP_TIMEOUT_SECONDS = 120 # COT ZIPs are up to ~30 MB
|
||||||
@@ -60,12 +63,12 @@ def _synthetic_etag(year: int, headers: dict) -> str:
|
|||||||
return etag
|
return etag
|
||||||
|
|
||||||
|
|
||||||
def extract_cot_year(year: int, http_session: niquests.Session) -> int:
|
def extract_cot_year(year: int, http_session: niquests.Session, url_template: str, landing_subdir: str) -> int:
|
||||||
"""Download and store COT data for a single year.
|
"""Download and store COT data for a single year.
|
||||||
|
|
||||||
Returns bytes_written (0 if skipped or unavailable).
|
Returns bytes_written (0 if skipped or unavailable).
|
||||||
"""
|
"""
|
||||||
url = COT_URL_TEMPLATE.format(year=year)
|
url = url_template.format(year=year)
|
||||||
logger.info(f"Checking COT data for {year}: {url}")
|
logger.info(f"Checking COT data for {year}: {url}")
|
||||||
|
|
||||||
head = http_session.head(url, timeout=HTTP_TIMEOUT_SECONDS)
|
head = http_session.head(url, timeout=HTTP_TIMEOUT_SECONDS)
|
||||||
@@ -79,7 +82,7 @@ def extract_cot_year(year: int, http_session: niquests.Session) -> int:
|
|||||||
raw_etag = head.headers.get("etag", "")
|
raw_etag = head.headers.get("etag", "")
|
||||||
etag = normalize_etag(raw_etag) if raw_etag else _synthetic_etag(year, head.headers)
|
etag = normalize_etag(raw_etag) if raw_etag else _synthetic_etag(year, head.headers)
|
||||||
|
|
||||||
dest_dir = landing_path(LANDING_DIR, "cot", str(year))
|
dest_dir = landing_path(LANDING_DIR, landing_subdir, str(year))
|
||||||
local_file = dest_dir / f"{etag}.csv.gzip"
|
local_file = dest_dir / f"{etag}.csv.gzip"
|
||||||
|
|
||||||
if local_file.exists():
|
if local_file.exists():
|
||||||
@@ -104,8 +107,8 @@ def extract_cot_year(year: int, http_session: niquests.Session) -> int:
|
|||||||
return bytes_written
|
return bytes_written
|
||||||
|
|
||||||
|
|
||||||
def extract_cot_dataset():
|
def _extract_cot(url_template: str, landing_subdir: str, extractor_name: str) -> None:
|
||||||
"""Extract all available CFTC COT disaggregated futures data.
|
"""Shared extraction loop for any COT report variant.
|
||||||
|
|
||||||
Downloads current year first (always re-checks for weekly Friday updates),
|
Downloads current year first (always re-checks for weekly Friday updates),
|
||||||
then backfills historical years. Bounded to MAX_YEARS. Continues on
|
then backfills historical years. Bounded to MAX_YEARS. Continues on
|
||||||
@@ -119,7 +122,7 @@ def extract_cot_dataset():
|
|||||||
)
|
)
|
||||||
|
|
||||||
conn = open_state_db(LANDING_DIR)
|
conn = open_state_db(LANDING_DIR)
|
||||||
run_id = start_run(conn, "cftc_cot")
|
run_id = start_run(conn, extractor_name)
|
||||||
files_written = 0
|
files_written = 0
|
||||||
files_skipped = 0
|
files_skipped = 0
|
||||||
bytes_written_total = 0
|
bytes_written_total = 0
|
||||||
@@ -127,7 +130,7 @@ def extract_cot_dataset():
|
|||||||
with niquests.Session() as session:
|
with niquests.Session() as session:
|
||||||
for year in years:
|
for year in years:
|
||||||
try:
|
try:
|
||||||
result = extract_cot_year(year, session)
|
result = extract_cot_year(year, session, url_template, landing_subdir)
|
||||||
if result > 0:
|
if result > 0:
|
||||||
files_written += 1
|
files_written += 1
|
||||||
bytes_written_total += result
|
bytes_written_total += result
|
||||||
@@ -136,7 +139,7 @@ def extract_cot_dataset():
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(f"Failed to extract COT data for {year}, continuing")
|
logger.exception(f"Failed to extract COT data for {year}, continuing")
|
||||||
|
|
||||||
logger.info(f"COT extraction complete: {files_written} new file(s) downloaded")
|
logger.info(f"COT extraction complete ({extractor_name}): {files_written} new file(s) downloaded")
|
||||||
end_run(
|
end_run(
|
||||||
conn, run_id, status="success",
|
conn, run_id, status="success",
|
||||||
files_written=files_written, files_skipped=files_skipped,
|
files_written=files_written, files_skipped=files_skipped,
|
||||||
@@ -150,5 +153,15 @@ def extract_cot_dataset():
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_cot_dataset():
|
||||||
|
"""Extract CFTC COT disaggregated futures-only report."""
|
||||||
|
_extract_cot(COT_URL_FUTURES_ONLY, "cot", "cftc_cot")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_cot_combined():
|
||||||
|
"""Extract CFTC COT disaggregated combined (futures+options) report."""
|
||||||
|
_extract_cot(COT_URL_COMBINED, "cot_combined", "cftc_cot_combined")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
extract_cot_dataset()
|
extract_cot_dataset()
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ PIPELINES = {
|
|||||||
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot"],
|
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot"],
|
||||||
"timeout_seconds": 1800,
|
"timeout_seconds": 1800,
|
||||||
},
|
},
|
||||||
|
"extract_cot_combined": {
|
||||||
|
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot_combined"],
|
||||||
|
"timeout_seconds": 1800,
|
||||||
|
},
|
||||||
"extract_prices": {
|
"extract_prices": {
|
||||||
"command": ["uv", "run", "--package", "coffee_prices", "extract_prices"],
|
"command": ["uv", "run", "--package", "coffee_prices", "extract_prices"],
|
||||||
"timeout_seconds": 300,
|
"timeout_seconds": 300,
|
||||||
@@ -49,7 +53,7 @@ PIPELINES = {
|
|||||||
"timeout_seconds": 120,
|
"timeout_seconds": 120,
|
||||||
},
|
},
|
||||||
"extract_all": {
|
"extract_all": {
|
||||||
"command": ["meta", "extract", "extract_cot", "extract_prices", "extract_ice_all", "extract_weather"],
|
"command": ["meta", "extract", "extract_cot", "extract_cot_combined", "extract_prices", "extract_ice_all", "extract_weather"],
|
||||||
"timeout_seconds": 6600,
|
"timeout_seconds": 6600,
|
||||||
},
|
},
|
||||||
"transform": {
|
"transform": {
|
||||||
@@ -68,7 +72,7 @@ PIPELINES = {
|
|||||||
|
|
||||||
|
|
||||||
META_PIPELINES: dict[str, list[str]] = {
|
META_PIPELINES: dict[str, list[str]] = {
|
||||||
"extract_all": ["extract", "extract_cot", "extract_prices", "extract_ice_all", "extract_weather"],
|
"extract_all": ["extract", "extract_cot", "extract_cot_combined", "extract_prices", "extract_ice_all", "extract_weather"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,13 @@ def cot_glob(evaluator) -> str:
|
|||||||
return f"'{landing_dir}/cot/**/*.csv.gzip'"
|
return f"'{landing_dir}/cot/**/*.csv.gzip'"
|
||||||
|
|
||||||
|
|
||||||
|
@macro()
|
||||||
|
def cot_combined_glob(evaluator) -> str:
|
||||||
|
"""Return a quoted glob path for all COT combined (futures+options) CSV gzip files under LANDING_DIR."""
|
||||||
|
landing_dir = evaluator.var("LANDING_DIR") or os.environ.get("LANDING_DIR", "data/landing")
|
||||||
|
return f"'{landing_dir}/cot_combined/**/*.csv.gzip'"
|
||||||
|
|
||||||
|
|
||||||
@macro()
|
@macro()
|
||||||
def prices_glob(evaluator) -> str:
|
def prices_glob(evaluator) -> str:
|
||||||
"""Return a quoted glob path for all coffee price CSV gzip files under LANDING_DIR."""
|
"""Return a quoted glob path for all coffee price CSV gzip files under LANDING_DIR."""
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ MODEL (
|
|||||||
kind INCREMENTAL_BY_TIME_RANGE (
|
kind INCREMENTAL_BY_TIME_RANGE (
|
||||||
time_column report_date
|
time_column report_date
|
||||||
),
|
),
|
||||||
grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date),
|
grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date, report_type),
|
||||||
start '2006-06-13',
|
start '2006-06-13',
|
||||||
cron '@daily'
|
cron '@daily'
|
||||||
);
|
);
|
||||||
@@ -21,6 +21,18 @@ WITH src AS (
|
|||||||
all_varchar = TRUE,
|
all_varchar = TRUE,
|
||||||
max_line_size = 10000000
|
max_line_size = 10000000
|
||||||
)
|
)
|
||||||
|
UNION ALL BY NAME
|
||||||
|
SELECT
|
||||||
|
*
|
||||||
|
FROM READ_CSV(
|
||||||
|
@cot_combined_glob(),
|
||||||
|
compression = 'gzip',
|
||||||
|
header = TRUE,
|
||||||
|
union_by_name = TRUE,
|
||||||
|
filename = TRUE,
|
||||||
|
all_varchar = TRUE,
|
||||||
|
max_line_size = 10000000
|
||||||
|
)
|
||||||
), cast_and_clean AS (
|
), cast_and_clean AS (
|
||||||
SELECT
|
SELECT
|
||||||
TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */
|
TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */
|
||||||
@@ -28,6 +40,7 @@ WITH src AS (
|
|||||||
TRIM(cftc_commodity_code) AS cftc_commodity_code,
|
TRIM(cftc_commodity_code) AS cftc_commodity_code,
|
||||||
TRIM(cftc_contract_market_code) AS cftc_contract_market_code,
|
TRIM(cftc_contract_market_code) AS cftc_contract_market_code,
|
||||||
TRIM(contract_units) AS contract_units,
|
TRIM(contract_units) AS contract_units,
|
||||||
|
TRIM("FutOnly_or_Combined") AS report_type, /* 'FutOnly' or 'Combined' — discriminates the two CFTC report variants */
|
||||||
TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */
|
TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */
|
||||||
TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */
|
TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */
|
||||||
TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short,
|
TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short,
|
||||||
@@ -66,12 +79,13 @@ WITH src AS (
|
|||||||
cftc_commodity_code,
|
cftc_commodity_code,
|
||||||
"Report_Date_as_YYYY-MM-DD",
|
"Report_Date_as_YYYY-MM-DD",
|
||||||
cftc_contract_market_code,
|
cftc_contract_market_code,
|
||||||
|
"FutOnly_or_Combined",
|
||||||
open_interest_all,
|
open_interest_all,
|
||||||
m_money_positions_long_all,
|
m_money_positions_long_all,
|
||||||
m_money_positions_short_all,
|
m_money_positions_short_all,
|
||||||
prod_merc_positions_long_all,
|
prod_merc_positions_long_all,
|
||||||
prod_merc_positions_short_all
|
prod_merc_positions_short_all
|
||||||
) AS hkey /* Dedup key: hash of business grain + key metrics */
|
) AS hkey /* Dedup key: hash of business grain + key metrics; includes report variant so fut-only and combined rows get distinct keys */
|
||||||
FROM src
|
FROM src
|
||||||
/* Reject rows with null commodity code or malformed date */
|
/* Reject rows with null commodity code or malformed date */
|
||||||
WHERE
|
WHERE
|
||||||
@@ -119,6 +133,7 @@ WITH src AS (
|
|||||||
ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short,
|
ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short,
|
||||||
ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread,
|
ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread,
|
||||||
ANY_VALUE(ingest_date) AS ingest_date,
|
ANY_VALUE(ingest_date) AS ingest_date,
|
||||||
|
ANY_VALUE(report_type) AS report_type,
|
||||||
hkey
|
hkey
|
||||||
FROM cast_and_clean
|
FROM cast_and_clean
|
||||||
GROUP BY
|
GROUP BY
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ WITH latest_revision AS (
|
|||||||
ON f.cftc_commodity_code = d.cftc_commodity_code
|
ON f.cftc_commodity_code = d.cftc_commodity_code
|
||||||
WHERE
|
WHERE
|
||||||
d.commodity_name = 'Coffee, Green'
|
d.commodity_name = 'Coffee, Green'
|
||||||
|
AND f.report_type = 'FutOnly'
|
||||||
AND f.report_date BETWEEN @start_ds AND @end_ds
|
AND f.report_date BETWEEN @start_ds AND @end_ds
|
||||||
QUALIFY
|
QUALIFY
|
||||||
ROW_NUMBER() OVER (
|
ROW_NUMBER() OVER (
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
/* Serving mart: COT positioning (combined futures+options) for Coffee C futures. */ /* Same analytics as serving.cot_positioning, but filtered to the combined */ /* report variant (FutOnly_or_Combined = 'Combined'). Positions include */ /* options delta-equivalent exposure, showing total directional market bet. */ /* Grain: one row per report_date for Coffee C futures. */ /* Latest revision per date: MAX(ingest_date) used to deduplicate CFTC corrections. */
|
||||||
|
MODEL (
|
||||||
|
name serving.cot_positioning_combined,
|
||||||
|
kind INCREMENTAL_BY_TIME_RANGE (
|
||||||
|
time_column report_date
|
||||||
|
),
|
||||||
|
grain (
|
||||||
|
report_date
|
||||||
|
),
|
||||||
|
start '2006-06-13',
|
||||||
|
cron '@daily'
|
||||||
|
);
|
||||||
|
|
||||||
|
WITH latest_revision AS (
|
||||||
|
/* Pick the most recently ingested row when CFTC issues corrections */
|
||||||
|
SELECT
|
||||||
|
f.*
|
||||||
|
FROM foundation.fct_cot_positioning AS f
|
||||||
|
INNER JOIN foundation.dim_commodity AS d
|
||||||
|
ON f.cftc_commodity_code = d.cftc_commodity_code
|
||||||
|
WHERE
|
||||||
|
d.commodity_name = 'Coffee, Green'
|
||||||
|
AND f.report_type = 'Combined'
|
||||||
|
AND f.report_date BETWEEN @start_ds AND @end_ds
|
||||||
|
QUALIFY
|
||||||
|
ROW_NUMBER() OVER (
|
||||||
|
PARTITION BY f.report_date, f.cftc_contract_market_code
|
||||||
|
ORDER BY f.ingest_date DESC
|
||||||
|
) = 1
|
||||||
|
), with_derived AS (
|
||||||
|
SELECT
|
||||||
|
report_date,
|
||||||
|
market_and_exchange_name,
|
||||||
|
cftc_commodity_code,
|
||||||
|
cftc_contract_market_code,
|
||||||
|
contract_units,
|
||||||
|
ingest_date,
|
||||||
|
open_interest, /* Absolute positions (contracts, delta-equivalent for options) */
|
||||||
|
managed_money_long,
|
||||||
|
managed_money_short,
|
||||||
|
managed_money_spread,
|
||||||
|
managed_money_net,
|
||||||
|
prod_merc_long,
|
||||||
|
prod_merc_short,
|
||||||
|
prod_merc_net,
|
||||||
|
swap_long,
|
||||||
|
swap_short,
|
||||||
|
swap_spread,
|
||||||
|
swap_net,
|
||||||
|
other_reportable_long,
|
||||||
|
other_reportable_short,
|
||||||
|
other_reportable_spread,
|
||||||
|
other_reportable_net,
|
||||||
|
nonreportable_long,
|
||||||
|
nonreportable_short,
|
||||||
|
nonreportable_net,
|
||||||
|
ROUND(managed_money_net::REAL / NULLIF(open_interest, 0) * 100, 2) AS managed_money_net_pct_of_oi, /* Normalized: managed money net as % of open interest */ /* Removes size effects and makes cross-period comparison meaningful */
|
||||||
|
ROUND(managed_money_long::REAL / NULLIF(managed_money_short, 0), 3) AS managed_money_long_short_ratio, /* Long/short ratio: >1 = more bulls than bears in managed money */
|
||||||
|
change_open_interest, /* Weekly changes */
|
||||||
|
change_managed_money_long,
|
||||||
|
change_managed_money_short,
|
||||||
|
change_managed_money_net,
|
||||||
|
change_prod_merc_long,
|
||||||
|
change_prod_merc_short,
|
||||||
|
managed_money_net /* Week-over-week momentum in managed money net (via LAG) */ - LAG(managed_money_net, 1) OVER (ORDER BY report_date) AS managed_money_net_wow,
|
||||||
|
concentration_top4_long_pct, /* Concentration */
|
||||||
|
concentration_top4_short_pct,
|
||||||
|
concentration_top8_long_pct,
|
||||||
|
concentration_top8_short_pct,
|
||||||
|
traders_total, /* Trader counts */
|
||||||
|
traders_managed_money_long,
|
||||||
|
traders_managed_money_short,
|
||||||
|
traders_managed_money_spread,
|
||||||
|
CASE
|
||||||
|
WHEN MAX(managed_money_net) OVER w26 = MIN(managed_money_net) OVER w26
|
||||||
|
THEN 50.0
|
||||||
|
ELSE ROUND(
|
||||||
|
(
|
||||||
|
managed_money_net - MIN(managed_money_net) OVER w26
|
||||||
|
)::REAL / (
|
||||||
|
MAX(managed_money_net) OVER w26 - MIN(managed_money_net) OVER w26
|
||||||
|
) * 100,
|
||||||
|
1
|
||||||
|
)
|
||||||
|
END AS cot_index_26w, /* COT Index (26-week): where is current net vs. trailing 26 weeks? */ /* 0 = most bearish extreme, 100 = most bullish extreme */ /* Includes options delta-equivalent exposure */
|
||||||
|
CASE
|
||||||
|
WHEN MAX(managed_money_net) OVER w52 = MIN(managed_money_net) OVER w52
|
||||||
|
THEN 50.0
|
||||||
|
ELSE ROUND(
|
||||||
|
(
|
||||||
|
managed_money_net - MIN(managed_money_net) OVER w52
|
||||||
|
)::REAL / (
|
||||||
|
MAX(managed_money_net) OVER w52 - MIN(managed_money_net) OVER w52
|
||||||
|
) * 100,
|
||||||
|
1
|
||||||
|
)
|
||||||
|
END AS cot_index_52w /* COT Index (52-week): longer-term positioning context */
|
||||||
|
FROM latest_revision
|
||||||
|
WINDOW w26 AS (ORDER BY report_date ROWS BETWEEN 25 PRECEDING AND CURRENT ROW), w52 AS (ORDER BY report_date ROWS BETWEEN 51 PRECEDING AND CURRENT ROW)
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
report_date,
|
||||||
|
market_and_exchange_name,
|
||||||
|
cftc_commodity_code,
|
||||||
|
cftc_contract_market_code,
|
||||||
|
contract_units,
|
||||||
|
ingest_date,
|
||||||
|
open_interest,
|
||||||
|
managed_money_long,
|
||||||
|
managed_money_short,
|
||||||
|
managed_money_spread,
|
||||||
|
managed_money_net,
|
||||||
|
prod_merc_long,
|
||||||
|
prod_merc_short,
|
||||||
|
prod_merc_net,
|
||||||
|
swap_long,
|
||||||
|
swap_short,
|
||||||
|
swap_spread,
|
||||||
|
swap_net,
|
||||||
|
other_reportable_long,
|
||||||
|
other_reportable_short,
|
||||||
|
other_reportable_spread,
|
||||||
|
other_reportable_net,
|
||||||
|
nonreportable_long,
|
||||||
|
nonreportable_short,
|
||||||
|
nonreportable_net,
|
||||||
|
managed_money_net_pct_of_oi,
|
||||||
|
managed_money_long_short_ratio,
|
||||||
|
change_open_interest,
|
||||||
|
change_managed_money_long,
|
||||||
|
change_managed_money_short,
|
||||||
|
change_managed_money_net,
|
||||||
|
change_prod_merc_long,
|
||||||
|
change_prod_merc_short,
|
||||||
|
managed_money_net_wow,
|
||||||
|
concentration_top4_long_pct,
|
||||||
|
concentration_top4_short_pct,
|
||||||
|
concentration_top8_long_pct,
|
||||||
|
concentration_top8_short_pct,
|
||||||
|
traders_total,
|
||||||
|
traders_managed_money_long,
|
||||||
|
traders_managed_money_short,
|
||||||
|
traders_managed_money_spread,
|
||||||
|
cot_index_26w,
|
||||||
|
cot_index_52w
|
||||||
|
FROM with_derived
|
||||||
|
ORDER BY
|
||||||
|
report_date
|
||||||
@@ -302,17 +302,23 @@ def _validate_cot_metrics(metrics: list[str]) -> list[str]:
|
|||||||
return valid
|
return valid
|
||||||
|
|
||||||
|
|
||||||
|
def _cot_table(combined: bool) -> str:
|
||||||
|
return "serving.cot_positioning_combined" if combined else "serving.cot_positioning"
|
||||||
|
|
||||||
|
|
||||||
async def get_cot_positioning_time_series(
|
async def get_cot_positioning_time_series(
|
||||||
cftc_commodity_code: str,
|
cftc_commodity_code: str,
|
||||||
metrics: list[str],
|
metrics: list[str],
|
||||||
start_date: str | None = None,
|
start_date: str | None = None,
|
||||||
end_date: str | None = None,
|
end_date: str | None = None,
|
||||||
limit: int = 520,
|
limit: int = 520,
|
||||||
|
combined: bool = False,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Weekly COT positioning time series. limit defaults to ~10 years of weekly data."""
|
"""Weekly COT positioning time series. limit defaults to ~10 years of weekly data."""
|
||||||
assert 1 <= limit <= 2000, "limit must be between 1 and 2000"
|
assert 1 <= limit <= 2000, "limit must be between 1 and 2000"
|
||||||
metrics = _validate_cot_metrics(metrics)
|
metrics = _validate_cot_metrics(metrics)
|
||||||
cols = ", ".join(metrics)
|
cols = ", ".join(metrics)
|
||||||
|
table = _cot_table(combined)
|
||||||
|
|
||||||
where_parts = ["cftc_commodity_code = ?"]
|
where_parts = ["cftc_commodity_code = ?"]
|
||||||
params: list = [cftc_commodity_code]
|
params: list = [cftc_commodity_code]
|
||||||
@@ -329,7 +335,7 @@ async def get_cot_positioning_time_series(
|
|||||||
return await fetch_analytics(
|
return await fetch_analytics(
|
||||||
f"""
|
f"""
|
||||||
SELECT report_date, {cols}
|
SELECT report_date, {cols}
|
||||||
FROM serving.cot_positioning
|
FROM {table}
|
||||||
WHERE {where_clause}
|
WHERE {where_clause}
|
||||||
ORDER BY report_date ASC
|
ORDER BY report_date ASC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
@@ -338,12 +344,13 @@ async def get_cot_positioning_time_series(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def get_cot_positioning_latest(cftc_commodity_code: str) -> dict | None:
|
async def get_cot_positioning_latest(cftc_commodity_code: str, combined: bool = False) -> dict | None:
|
||||||
"""Latest week's full COT positioning snapshot."""
|
"""Latest week's full COT positioning snapshot."""
|
||||||
|
table = _cot_table(combined)
|
||||||
rows = await fetch_analytics(
|
rows = await fetch_analytics(
|
||||||
"""
|
f"""
|
||||||
SELECT *
|
SELECT *
|
||||||
FROM serving.cot_positioning
|
FROM {table}
|
||||||
WHERE cftc_commodity_code = ?
|
WHERE cftc_commodity_code = ?
|
||||||
ORDER BY report_date DESC
|
ORDER BY report_date DESC
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
@@ -356,14 +363,16 @@ async def get_cot_positioning_latest(cftc_commodity_code: str) -> dict | None:
|
|||||||
async def get_cot_index_trend(
|
async def get_cot_index_trend(
|
||||||
cftc_commodity_code: str,
|
cftc_commodity_code: str,
|
||||||
weeks: int = 104,
|
weeks: int = 104,
|
||||||
|
combined: bool = False,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""COT Index time series (26w and 52w) for the trailing N weeks."""
|
"""COT Index time series (26w and 52w) for the trailing N weeks."""
|
||||||
assert 1 <= weeks <= 1040, "weeks must be between 1 and 1040"
|
assert 1 <= weeks <= 1040, "weeks must be between 1 and 1040"
|
||||||
|
table = _cot_table(combined)
|
||||||
return await fetch_analytics(
|
return await fetch_analytics(
|
||||||
"""
|
f"""
|
||||||
SELECT report_date, cot_index_26w, cot_index_52w,
|
SELECT report_date, cot_index_26w, cot_index_52w,
|
||||||
managed_money_net, managed_money_net_pct_of_oi
|
managed_money_net, managed_money_net_pct_of_oi
|
||||||
FROM serving.cot_positioning
|
FROM {table}
|
||||||
WHERE cftc_commodity_code = ?
|
WHERE cftc_commodity_code = ?
|
||||||
ORDER BY report_date DESC
|
ORDER BY report_date DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
@@ -372,6 +381,30 @@ async def get_cot_index_trend(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def get_cot_options_delta(cftc_commodity_code: str) -> dict | None:
|
||||||
|
"""Latest managed_money_net difference between combined and futures-only reports.
|
||||||
|
|
||||||
|
Shows whether the options book is reinforcing (same direction) or hedging
|
||||||
|
(opposite direction) the futures position. Returns None if either table
|
||||||
|
has no data.
|
||||||
|
"""
|
||||||
|
rows = await fetch_analytics(
|
||||||
|
"""
|
||||||
|
SELECT f.report_date,
|
||||||
|
f.managed_money_net AS fut_net,
|
||||||
|
c.managed_money_net AS combined_net,
|
||||||
|
c.managed_money_net - f.managed_money_net AS options_delta
|
||||||
|
FROM serving.cot_positioning f
|
||||||
|
JOIN serving.cot_positioning_combined c USING (report_date)
|
||||||
|
WHERE f.cftc_commodity_code = ?
|
||||||
|
ORDER BY f.report_date DESC
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
[cftc_commodity_code],
|
||||||
|
)
|
||||||
|
return rows[0] if rows else None
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Coffee Prices Queries
|
# Coffee Prices Queries
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
@@ -172,6 +172,7 @@ async def commodity_positioning(code: str):
|
|||||||
start_date — ISO date filter (YYYY-MM-DD)
|
start_date — ISO date filter (YYYY-MM-DD)
|
||||||
end_date — ISO date filter (YYYY-MM-DD)
|
end_date — ISO date filter (YYYY-MM-DD)
|
||||||
limit — max rows returned (default 260, max 2000)
|
limit — max rows returned (default 260, max 2000)
|
||||||
|
type — report variant: "fut" (futures-only, default) or "combined" (futures+options)
|
||||||
"""
|
"""
|
||||||
raw_metrics = request.args.getlist("metrics") or [
|
raw_metrics = request.args.getlist("metrics") or [
|
||||||
"managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w"
|
"managed_money_net", "prod_merc_net", "open_interest", "cot_index_26w"
|
||||||
@@ -183,19 +184,27 @@ async def commodity_positioning(code: str):
|
|||||||
start_date = request.args.get("start_date")
|
start_date = request.args.get("start_date")
|
||||||
end_date = request.args.get("end_date")
|
end_date = request.args.get("end_date")
|
||||||
limit = min(int(request.args.get("limit", 260)), 2000)
|
limit = min(int(request.args.get("limit", 260)), 2000)
|
||||||
|
cot_type = request.args.get("type", "fut")
|
||||||
|
combined = cot_type == "combined"
|
||||||
|
|
||||||
data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit)
|
data = await analytics.get_cot_positioning_time_series(code, metrics, start_date, end_date, limit, combined=combined)
|
||||||
return jsonify({"cftc_commodity_code": code, "metrics": metrics, "data": data})
|
return jsonify({"cftc_commodity_code": code, "type": cot_type, "metrics": metrics, "data": data})
|
||||||
|
|
||||||
|
|
||||||
@bp.route("/commodities/<code>/positioning/latest")
|
@bp.route("/commodities/<code>/positioning/latest")
|
||||||
@api_key_required(scopes=["read"])
|
@api_key_required(scopes=["read"])
|
||||||
async def commodity_positioning_latest(code: str):
|
async def commodity_positioning_latest(code: str):
|
||||||
"""Latest week's full COT positioning snapshot for a commodity."""
|
"""Latest week's full COT positioning snapshot for a commodity.
|
||||||
data = await analytics.get_cot_positioning_latest(code)
|
|
||||||
|
Query params:
|
||||||
|
type — report variant: "fut" (futures-only, default) or "combined" (futures+options)
|
||||||
|
"""
|
||||||
|
cot_type = request.args.get("type", "fut")
|
||||||
|
combined = cot_type == "combined"
|
||||||
|
data = await analytics.get_cot_positioning_latest(code, combined=combined)
|
||||||
if not data:
|
if not data:
|
||||||
return jsonify({"error": "No positioning data found for this commodity"}), 404
|
return jsonify({"error": "No positioning data found for this commodity"}), 404
|
||||||
return jsonify({"cftc_commodity_code": code, "data": data})
|
return jsonify({"cftc_commodity_code": code, "type": cot_type, "data": data})
|
||||||
|
|
||||||
|
|
||||||
@bp.route("/commodities/<code>/prices")
|
@bp.route("/commodities/<code>/prices")
|
||||||
|
|||||||
@@ -257,30 +257,45 @@ async def positioning():
|
|||||||
if range_key not in RANGE_MAP:
|
if range_key not in RANGE_MAP:
|
||||||
range_key = "1y"
|
range_key = "1y"
|
||||||
|
|
||||||
|
cot_type = request.args.get("type", "fut")
|
||||||
|
if cot_type not in ("fut", "combined"):
|
||||||
|
cot_type = "fut"
|
||||||
|
combined = cot_type == "combined"
|
||||||
|
|
||||||
rng = RANGE_MAP[range_key]
|
rng = RANGE_MAP[range_key]
|
||||||
price_limit = rng["days"]
|
price_limit = rng["days"]
|
||||||
cot_weeks = rng["weeks"]
|
cot_weeks = rng["weeks"]
|
||||||
|
|
||||||
|
options_delta = None
|
||||||
if analytics._db_path:
|
if analytics._db_path:
|
||||||
results = await asyncio.gather(
|
gather_coros = [
|
||||||
analytics.get_price_latest(analytics.COFFEE_TICKER),
|
analytics.get_price_latest(analytics.COFFEE_TICKER),
|
||||||
analytics.get_price_time_series(analytics.COFFEE_TICKER, limit=price_limit),
|
analytics.get_price_time_series(analytics.COFFEE_TICKER, limit=price_limit),
|
||||||
analytics.get_cot_positioning_latest(analytics.COFFEE_CFTC_CODE),
|
analytics.get_cot_positioning_latest(analytics.COFFEE_CFTC_CODE, combined=combined),
|
||||||
analytics.get_cot_index_trend(analytics.COFFEE_CFTC_CODE, weeks=cot_weeks),
|
analytics.get_cot_index_trend(analytics.COFFEE_CFTC_CODE, weeks=cot_weeks, combined=combined),
|
||||||
return_exceptions=True,
|
]
|
||||||
)
|
if combined:
|
||||||
defaults = [None, [], None, []]
|
gather_coros.append(analytics.get_cot_options_delta(analytics.COFFEE_CFTC_CODE))
|
||||||
price_latest, price_series, cot_latest, cot_trend = _safe(results, defaults)
|
|
||||||
|
results = await asyncio.gather(*gather_coros, return_exceptions=True)
|
||||||
|
defaults = [None, [], None, [], None] if combined else [None, [], None, []]
|
||||||
|
safe_results = _safe(results, defaults)
|
||||||
|
|
||||||
|
price_latest, price_series, cot_latest, cot_trend = safe_results[:4]
|
||||||
|
if combined:
|
||||||
|
options_delta = safe_results[4]
|
||||||
else:
|
else:
|
||||||
price_latest, price_series, cot_latest, cot_trend = None, [], None, []
|
price_latest, price_series, cot_latest, cot_trend = None, [], None, []
|
||||||
|
|
||||||
ctx = dict(
|
ctx = dict(
|
||||||
plan=plan,
|
plan=plan,
|
||||||
range_key=range_key,
|
range_key=range_key,
|
||||||
|
cot_type=cot_type,
|
||||||
price_latest=price_latest,
|
price_latest=price_latest,
|
||||||
price_series=price_series,
|
price_series=price_series,
|
||||||
cot_latest=cot_latest,
|
cot_latest=cot_latest,
|
||||||
cot_trend=cot_trend,
|
cot_trend=cot_trend,
|
||||||
|
options_delta=options_delta,
|
||||||
)
|
)
|
||||||
|
|
||||||
if request.headers.get("HX-Request"):
|
if request.headers.get("HX-Request"):
|
||||||
|
|||||||
@@ -37,6 +37,15 @@
|
|||||||
{% endfor %}
|
{% endfor %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Report type toggle -->
|
||||||
|
<div class="filter-pills" id="type-pills" style="margin-left: 1.5rem; border-left: 1px solid var(--color-parchment); padding-left: 1.5rem;">
|
||||||
|
{% for val, label in [("fut", "Futures"), ("combined", "F+O Combined")] %}
|
||||||
|
<button type="button"
|
||||||
|
class="filter-pill {{ 'active' if cot_type == val }}"
|
||||||
|
onclick="setType('{{ val }}')">{{ label }}</button>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- MA toggles (client-side only) -->
|
<!-- MA toggles (client-side only) -->
|
||||||
<label class="filter-check">
|
<label class="filter-check">
|
||||||
<input type="checkbox" id="ma20-toggle" checked onchange="toggleMA('sma20')"> 20d MA
|
<input type="checkbox" id="ma20-toggle" checked onchange="toggleMA('sma20')"> 20d MA
|
||||||
@@ -57,15 +66,32 @@
|
|||||||
<script>
|
<script>
|
||||||
var POSITIONING_URL = '{{ url_for("dashboard.positioning") }}';
|
var POSITIONING_URL = '{{ url_for("dashboard.positioning") }}';
|
||||||
var currentRange = {{ range_key | tojson }};
|
var currentRange = {{ range_key | tojson }};
|
||||||
|
var currentType = {{ cot_type | tojson }};
|
||||||
|
|
||||||
|
function _positioningUrl() {
|
||||||
|
return POSITIONING_URL + '?range=' + currentRange + '&type=' + currentType;
|
||||||
|
}
|
||||||
|
|
||||||
function setRange(val) {
|
function setRange(val) {
|
||||||
currentRange = val;
|
currentRange = val;
|
||||||
var url = POSITIONING_URL + '?range=' + currentRange;
|
var url = _positioningUrl();
|
||||||
window.history.pushState({}, '', url);
|
window.history.pushState({}, '', url);
|
||||||
document.getElementById('positioning-canvas').classList.add('canvas-loading');
|
document.getElementById('positioning-canvas').classList.add('canvas-loading');
|
||||||
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
|
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setType(val) {
|
||||||
|
currentType = val;
|
||||||
|
var url = _positioningUrl();
|
||||||
|
window.history.pushState({}, '', url);
|
||||||
|
document.getElementById('positioning-canvas').classList.add('canvas-loading');
|
||||||
|
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
|
||||||
|
// Sync type pill active state immediately (canvas swap will also re-sync)
|
||||||
|
document.querySelectorAll('#type-pills .filter-pill').forEach(function (btn) {
|
||||||
|
btn.classList.toggle('active', btn.textContent.trim() === (val === 'combined' ? 'F+O Combined' : 'Futures'));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// MA toggles: client-side only — update Chart.js dataset visibility
|
// MA toggles: client-side only — update Chart.js dataset visibility
|
||||||
function toggleMA(key) {
|
function toggleMA(key) {
|
||||||
var chart = Chart.getChart('priceChart');
|
var chart = Chart.getChart('priceChart');
|
||||||
@@ -93,7 +119,8 @@ document.addEventListener('htmx:afterSwap', function (e) {
|
|||||||
window.addEventListener('popstate', function () {
|
window.addEventListener('popstate', function () {
|
||||||
var p = new URLSearchParams(window.location.search);
|
var p = new URLSearchParams(window.location.search);
|
||||||
currentRange = p.get('range') || '1y';
|
currentRange = p.get('range') || '1y';
|
||||||
var url = POSITIONING_URL + '?range=' + currentRange;
|
currentType = p.get('type') || 'fut';
|
||||||
|
var url = _positioningUrl();
|
||||||
document.getElementById('positioning-canvas').classList.add('canvas-loading');
|
document.getElementById('positioning-canvas').classList.add('canvas-loading');
|
||||||
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
|
htmx.ajax('GET', url, { target: '#positioning-canvas', swap: 'innerHTML' });
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,11 +19,16 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="metric-card">
|
<div class="metric-card">
|
||||||
<div class="metric-label">MM Net Position</div>
|
<div class="metric-label">MM Net Position{% if cot_type == "combined" %} <span style="font-size:0.7em;opacity:0.7">F+O</span>{% endif %}</div>
|
||||||
<div class="metric-value {% if cot_latest and cot_latest.managed_money_net > 0 %}text-bean-green{% elif cot_latest and cot_latest.managed_money_net < 0 %}text-danger{% endif %}">
|
<div class="metric-value {% if cot_latest and cot_latest.managed_money_net > 0 %}text-bean-green{% elif cot_latest and cot_latest.managed_money_net < 0 %}text-danger{% endif %}">
|
||||||
{% if cot_latest %}{{ "{:+,d}".format(cot_latest.managed_money_net | int) }}{% else %}--{% endif %}
|
{% if cot_latest %}{{ "{:+,d}".format(cot_latest.managed_money_net | int) }}{% else %}--{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="metric-sub">contracts (long − short)</div>
|
<div class="metric-sub">contracts (long − short)</div>
|
||||||
|
{% if options_delta %}
|
||||||
|
<div class="metric-sub" style="font-family: 'Commit Mono', ui-monospace, monospace; margin-top: 0.25rem; {{ 'color: var(--color-bean-green)' if options_delta.options_delta > 0 else 'color: var(--color-copper)' }}">
|
||||||
|
Opt Δ: {{ "{:+,d}".format(options_delta.options_delta | int) }}
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="metric-card">
|
<div class="metric-card">
|
||||||
@@ -64,8 +69,8 @@
|
|||||||
<div class="cc-chart-card">
|
<div class="cc-chart-card">
|
||||||
<div class="cc-chart-top">
|
<div class="cc-chart-top">
|
||||||
<div>
|
<div>
|
||||||
<div class="cc-chart-title">Managed Money Net Position + COT Index</div>
|
<div class="cc-chart-title">Managed Money Net Position{{ " (F+O Combined)" if cot_type == "combined" else "" }} + COT Index</div>
|
||||||
<div class="cc-chart-meta">CFTC Commitment of Traders · weekly · area = net contracts · dashed = COT index (0–100)</div>
|
<div class="cc-chart-meta">CFTC Commitment of Traders · weekly · {{ "futures + options delta-adjusted" if cot_type == "combined" else "futures only" }} · area = net contracts · dashed = COT index (0–100)</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="cc-chart-body">
|
<div class="cc-chart-body">
|
||||||
@@ -94,13 +99,21 @@
|
|||||||
<script>
|
<script>
|
||||||
(function () {
|
(function () {
|
||||||
var range = {{ range_key | tojson }};
|
var range = {{ range_key | tojson }};
|
||||||
|
var cotType = {{ cot_type | tojson }};
|
||||||
|
|
||||||
// Sync range pills
|
// Sync range pills
|
||||||
document.querySelectorAll('#range-pills .filter-pill').forEach(function (btn) {
|
document.querySelectorAll('#range-pills .filter-pill').forEach(function (btn) {
|
||||||
btn.classList.toggle('active', btn.textContent.trim().toLowerCase() === range);
|
btn.classList.toggle('active', btn.textContent.trim().toLowerCase() === range);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Sync type pills
|
||||||
|
document.querySelectorAll('#type-pills .filter-pill').forEach(function (btn) {
|
||||||
|
var label = cotType === 'combined' ? 'F+O Combined' : 'Futures';
|
||||||
|
btn.classList.toggle('active', btn.textContent.trim() === label);
|
||||||
|
});
|
||||||
|
|
||||||
if (typeof currentRange !== 'undefined') currentRange = range;
|
if (typeof currentRange !== 'undefined') currentRange = range;
|
||||||
|
if (typeof currentType !== 'undefined') currentType = cotType;
|
||||||
|
|
||||||
var C = {
|
var C = {
|
||||||
copper: '#B45309', green: '#15803D', roast: '#4A2C1A', stone: '#78716C',
|
copper: '#B45309', green: '#15803D', roast: '#4A2C1A', stone: '#78716C',
|
||||||
|
|||||||
Reference in New Issue
Block a user