Add CFTC COT data integration with foundation data model layer
- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization - SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart) - dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology - fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities - obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios - Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest] - Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart - 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
85
transform/sqlmesh_materia/models/raw/cot_disaggregated.sql
Normal file
85
transform/sqlmesh_materia/models/raw/cot_disaggregated.sql
Normal file
@@ -0,0 +1,85 @@
|
||||
-- Raw CFTC Commitment of Traders — Disaggregated Futures Only.
|
||||
--
|
||||
-- Technical ingestion layer only: reads gzip CSVs from the landing directory
|
||||
-- and surfaces the columns needed by downstream foundation models.
|
||||
-- All values are varchar; casting happens in foundation.
|
||||
--
|
||||
-- Source: CFTC yearly ZIPs at
|
||||
-- https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip
|
||||
-- Coverage: June 2006 – present (new file every Friday at 3:30 PM ET)
|
||||
|
||||
MODEL (
|
||||
name raw.cot_disaggregated,
|
||||
kind FULL,
|
||||
grain (cftc_commodity_code, report_date_as_yyyy_mm_dd, cftc_contract_market_code),
|
||||
start '2006-06-13',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
SELECT
|
||||
-- Identifiers
|
||||
"Market_and_Exchange_Names" AS market_and_exchange_names,
|
||||
"Report_Date_as_YYYY-MM-DD" AS report_date_as_yyyy_mm_dd,
|
||||
"CFTC_Commodity_Code" AS cftc_commodity_code,
|
||||
"CFTC_Contract_Market_Code" AS cftc_contract_market_code,
|
||||
"Contract_Units" AS contract_units,
|
||||
|
||||
-- Open interest
|
||||
"Open_Interest_All" AS open_interest_all,
|
||||
|
||||
-- Producer / Merchant / Processor / User (commercial hedgers)
|
||||
"Prod_Merc_Positions_Long_All" AS prod_merc_positions_long_all,
|
||||
"Prod_Merc_Positions_Short_All" AS prod_merc_positions_short_all,
|
||||
|
||||
-- Swap dealers
|
||||
"Swap_Positions_Long_All" AS swap_positions_long_all,
|
||||
"Swap__Positions_Short_All" AS swap_positions_short_all,
|
||||
"Swap__Positions_Spread_All" AS swap_positions_spread_all,
|
||||
|
||||
-- Managed money (hedge funds, CTAs — key speculative signal)
|
||||
"M_Money_Positions_Long_All" AS m_money_positions_long_all,
|
||||
"M_Money_Positions_Short_All" AS m_money_positions_short_all,
|
||||
"M_Money_Positions_Spread_All" AS m_money_positions_spread_all,
|
||||
|
||||
-- Other reportables
|
||||
"Other_Rept_Positions_Long_All" AS other_rept_positions_long_all,
|
||||
"Other_Rept_Positions_Short_All" AS other_rept_positions_short_all,
|
||||
"Other_Rept_Positions_Spread_All" AS other_rept_positions_spread_all,
|
||||
|
||||
-- Non-reportable (small speculators)
|
||||
"NonRept_Positions_Long_All" AS nonrept_positions_long_all,
|
||||
"NonRept_Positions_Short_All" AS nonrept_positions_short_all,
|
||||
|
||||
-- Week-over-week changes
|
||||
"Change_in_Open_Interest_All" AS change_in_open_interest_all,
|
||||
"Change_in_M_Money_Long_All" AS change_in_m_money_long_all,
|
||||
"Change_in_M_Money_Short_All" AS change_in_m_money_short_all,
|
||||
"Change_in_Prod_Merc_Long_All" AS change_in_prod_merc_long_all,
|
||||
"Change_in_Prod_Merc_Short_All" AS change_in_prod_merc_short_all,
|
||||
|
||||
-- Concentration (% of OI held by top 4 and top 8 traders)
|
||||
"Conc_Gross_LE_4_TDR_Long_All" AS conc_gross_le_4_tdr_long_all,
|
||||
"Conc_Gross_LE_4_TDR_Short_All" AS conc_gross_le_4_tdr_short_all,
|
||||
"Conc_Gross_LE_8_TDR_Long_All" AS conc_gross_le_8_tdr_long_all,
|
||||
"Conc_Gross_LE_8_TDR_Short_All" AS conc_gross_le_8_tdr_short_all,
|
||||
|
||||
-- Trader counts
|
||||
"Traders_Tot_All" AS traders_tot_all,
|
||||
"Traders_M_Money_Long_All" AS traders_m_money_long_all,
|
||||
"Traders_M_Money_Short_All" AS traders_m_money_short_all,
|
||||
"Traders_M_Money_Spread_All" AS traders_m_money_spread_all,
|
||||
|
||||
-- Lineage
|
||||
filename
|
||||
FROM read_csv(
|
||||
@cot_glob(),
|
||||
delim = ',',
|
||||
encoding = 'utf-8',
|
||||
compression = 'gzip',
|
||||
header = true,
|
||||
union_by_name = true,
|
||||
filename = true,
|
||||
all_varchar = true,
|
||||
max_line_size = 10000000,
|
||||
ignore_errors = true
|
||||
)
|
||||
Reference in New Issue
Block a user