merge: CFTC COT combined (futures+options) report — extractor, transform, web toggle

transform/sqlmesh_materia/macros/__init__.py

@@ -17,6 +17,13 @@ def cot_glob(evaluator) -> str:
     return f"'{landing_dir}/cot/**/*.csv.gzip'"
 
 
+@macro()
+def cot_combined_glob(evaluator) -> str:
+    """Return a quoted glob path for all COT combined (futures+options) CSV gzip files under LANDING_DIR."""
+    landing_dir = evaluator.var("LANDING_DIR") or os.environ.get("LANDING_DIR", "data/landing")
+    return f"'{landing_dir}/cot_combined/**/*.csv.gzip'"
+
+
 @macro()
 def prices_glob(evaluator) -> str:
     """Return a quoted glob path for all coffee price CSV gzip files under LANDING_DIR."""

transform/sqlmesh_materia/models/foundation/fct_cot_positioning.sql

@@ -4,7 +4,7 @@ MODEL (
   kind INCREMENTAL_BY_TIME_RANGE (
     time_column report_date
   ),
-  grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date),
+  grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date, report_type),
   start '2006-06-13',
   cron '@daily'
 );
@@ -21,6 +21,18 @@ WITH src AS (
     all_varchar = TRUE,
     max_line_size = 10000000
   )
+  UNION ALL BY NAME
+  SELECT
+    *
+  FROM READ_CSV(
+    @cot_combined_glob(),
+    compression = 'gzip',
+    header = TRUE,
+    union_by_name = TRUE,
+    filename = TRUE,
+    all_varchar = TRUE,
+    max_line_size = 10000000
+  )
 ), cast_and_clean AS (
   SELECT
     TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */
@@ -28,6 +40,7 @@ WITH src AS (
     TRIM(cftc_commodity_code) AS cftc_commodity_code,
     TRIM(cftc_contract_market_code) AS cftc_contract_market_code,
     TRIM(contract_units) AS contract_units,
+    TRIM("FutOnly_or_Combined") AS report_type, /* 'FutOnly' or 'Combined' — discriminates the two CFTC report variants */
     TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */
     TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */
     TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short,
@@ -66,12 +79,13 @@ WITH src AS (
       cftc_commodity_code,
       "Report_Date_as_YYYY-MM-DD",
       cftc_contract_market_code,
+      "FutOnly_or_Combined",
       open_interest_all,
       m_money_positions_long_all,
       m_money_positions_short_all,
       prod_merc_positions_long_all,
       prod_merc_positions_short_all
-    ) AS hkey /* Dedup key: hash of business grain + key metrics */
+    ) AS hkey /* Dedup key: hash of business grain + key metrics; includes report variant so fut-only and combined rows get distinct keys */
   FROM src
   /* Reject rows with null commodity code or malformed date */
   WHERE
@@ -119,6 +133,7 @@ WITH src AS (
     ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short,
     ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread,
     ANY_VALUE(ingest_date) AS ingest_date,
+    ANY_VALUE(report_type) AS report_type,
     hkey
   FROM cast_and_clean
   GROUP BY

transform/sqlmesh_materia/models/serving/obt_cot_positioning.sql

@@ -20,6 +20,7 @@ WITH latest_revision AS (
     ON f.cftc_commodity_code = d.cftc_commodity_code
   WHERE
     d.commodity_name = 'Coffee, Green'
+    AND f.report_type = 'FutOnly'
     AND f.report_date BETWEEN @start_ds AND @end_ds
   QUALIFY
     ROW_NUMBER() OVER (

transform/sqlmesh_materia/models/serving/obt_cot_positioning_combined.sql

@@ -0,0 +1,148 @@
+/* Serving mart: COT positioning (combined futures+options) for Coffee C futures. */ /* Same analytics as serving.cot_positioning, but filtered to the combined */ /* report variant (FutOnly_or_Combined = 'Combined'). Positions include */ /* options delta-equivalent exposure, showing total directional market bet. */ /* Grain: one row per report_date for Coffee C futures. */ /* Latest revision per date: MAX(ingest_date) used to deduplicate CFTC corrections. */
+MODEL (
+  name serving.cot_positioning_combined,
+  kind INCREMENTAL_BY_TIME_RANGE (
+    time_column report_date
+  ),
+  grain (
+    report_date
+  ),
+  start '2006-06-13',
+  cron '@daily'
+);
+
+WITH latest_revision AS (
+  /* Pick the most recently ingested row when CFTC issues corrections */
+  SELECT
+    f.*
+  FROM foundation.fct_cot_positioning AS f
+  INNER JOIN foundation.dim_commodity AS d
+    ON f.cftc_commodity_code = d.cftc_commodity_code
+  WHERE
+    d.commodity_name = 'Coffee, Green'
+    AND f.report_type = 'Combined'
+    AND f.report_date BETWEEN @start_ds AND @end_ds
+  QUALIFY
+    ROW_NUMBER() OVER (
+      PARTITION BY f.report_date, f.cftc_contract_market_code
+      ORDER BY f.ingest_date DESC
+    ) = 1
+), with_derived AS (
+  SELECT
+    report_date,
+    market_and_exchange_name,
+    cftc_commodity_code,
+    cftc_contract_market_code,
+    contract_units,
+    ingest_date,
+    open_interest, /* Absolute positions (contracts, delta-equivalent for options) */
+    managed_money_long,
+    managed_money_short,
+    managed_money_spread,
+    managed_money_net,
+    prod_merc_long,
+    prod_merc_short,
+    prod_merc_net,
+    swap_long,
+    swap_short,
+    swap_spread,
+    swap_net,
+    other_reportable_long,
+    other_reportable_short,
+    other_reportable_spread,
+    other_reportable_net,
+    nonreportable_long,
+    nonreportable_short,
+    nonreportable_net,
+    ROUND(managed_money_net::REAL / NULLIF(open_interest, 0) * 100, 2) AS managed_money_net_pct_of_oi, /* Normalized: managed money net as % of open interest */ /* Removes size effects and makes cross-period comparison meaningful */
+    ROUND(managed_money_long::REAL / NULLIF(managed_money_short, 0), 3) AS managed_money_long_short_ratio, /* Long/short ratio: >1 = more bulls than bears in managed money */
+    change_open_interest, /* Weekly changes */
+    change_managed_money_long,
+    change_managed_money_short,
+    change_managed_money_net,
+    change_prod_merc_long,
+    change_prod_merc_short,
+    managed_money_net /* Week-over-week momentum in managed money net (via LAG) */ - LAG(managed_money_net, 1) OVER (ORDER BY report_date) AS managed_money_net_wow,
+    concentration_top4_long_pct, /* Concentration */
+    concentration_top4_short_pct,
+    concentration_top8_long_pct,
+    concentration_top8_short_pct,
+    traders_total, /* Trader counts */
+    traders_managed_money_long,
+    traders_managed_money_short,
+    traders_managed_money_spread,
+    CASE
+      WHEN MAX(managed_money_net) OVER w26 = MIN(managed_money_net) OVER w26
+      THEN 50.0
+      ELSE ROUND(
+        (
+          managed_money_net - MIN(managed_money_net) OVER w26
+        )::REAL / (
+          MAX(managed_money_net) OVER w26 - MIN(managed_money_net) OVER w26
+        ) * 100,
+        1
+      )
+    END AS cot_index_26w, /* COT Index (26-week): where is current net vs. trailing 26 weeks? */ /* 0 = most bearish extreme, 100 = most bullish extreme */ /* Includes options delta-equivalent exposure */
+    CASE
+      WHEN MAX(managed_money_net) OVER w52 = MIN(managed_money_net) OVER w52
+      THEN 50.0
+      ELSE ROUND(
+        (
+          managed_money_net - MIN(managed_money_net) OVER w52
+        )::REAL / (
+          MAX(managed_money_net) OVER w52 - MIN(managed_money_net) OVER w52
+        ) * 100,
+        1
+      )
+    END AS cot_index_52w /* COT Index (52-week): longer-term positioning context */
+  FROM latest_revision
+  WINDOW w26 AS (ORDER BY report_date ROWS BETWEEN 25 PRECEDING AND CURRENT ROW), w52 AS (ORDER BY report_date ROWS BETWEEN 51 PRECEDING AND CURRENT ROW)
+)
+SELECT
+  report_date,
+  market_and_exchange_name,
+  cftc_commodity_code,
+  cftc_contract_market_code,
+  contract_units,
+  ingest_date,
+  open_interest,
+  managed_money_long,
+  managed_money_short,
+  managed_money_spread,
+  managed_money_net,
+  prod_merc_long,
+  prod_merc_short,
+  prod_merc_net,
+  swap_long,
+  swap_short,
+  swap_spread,
+  swap_net,
+  other_reportable_long,
+  other_reportable_short,
+  other_reportable_spread,
+  other_reportable_net,
+  nonreportable_long,
+  nonreportable_short,
+  nonreportable_net,
+  managed_money_net_pct_of_oi,
+  managed_money_long_short_ratio,
+  change_open_interest,
+  change_managed_money_long,
+  change_managed_money_short,
+  change_managed_money_net,
+  change_prod_merc_long,
+  change_prod_merc_short,
+  managed_money_net_wow,
+  concentration_top4_long_pct,
+  concentration_top4_short_pct,
+  concentration_top8_long_pct,
+  concentration_top8_short_pct,
+  traders_total,
+  traders_managed_money_long,
+  traders_managed_money_short,
+  traders_managed_money_spread,
+  cot_index_26w,
+  cot_index_52w
+FROM with_derived
+ORDER BY
+  report_date