deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CFTC COT data integration with foundation data model layer

Browse Source 
- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs
  from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization
- SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart)
- dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology
- fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities
- obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios
- Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest]
- Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart
- 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
2026-02-20 21:57:04 +01:00
parent d09ba91023
commit 0a83b2cb74
19 changed files with 1111 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
transform/sqlmesh_materia/tests/test_cot_foundation.yaml Normal file
View File

@@ -0,0 +1,99 @@
test_fct_cot_positioning_types_and_net_positions:
model: foundation.fct_cot_positioning
inputs:
raw.cot_disaggregated:
rows:
- market_and_exchange_names: "COFFEE C - ICE FUTURES U.S."
report_date_as_yyyy_mm_dd: "2024-01-02"
cftc_commodity_code: "083731"
cftc_contract_market_code: "083731"
contract_units: "37,500 POUNDS"
open_interest_all: "250000"
prod_merc_positions_long_all: "80000"
prod_merc_positions_short_all: "90000"
swap_positions_long_all: "30000"
swap_positions_short_all: "35000"
swap_positions_spread_all: "10000"
m_money_positions_long_all: "60000"
m_money_positions_short_all: "40000"
m_money_positions_spread_all: "15000"
other_rept_positions_long_all: "20000"
other_rept_positions_short_all: "18000"
other_rept_positions_spread_all: "5000"
nonrept_positions_long_all: "12000"
nonrept_positions_short_all: "14000"
change_in_open_interest_all: "5000"
change_in_m_money_long_all: "2000"
change_in_m_money_short_all: "-1000"
change_in_prod_merc_long_all: "1000"
change_in_prod_merc_short_all: "500"
conc_gross_le_4_tdr_long_all: "35.5"
conc_gross_le_4_tdr_short_all: "28.3"
conc_gross_le_8_tdr_long_all: "52.1"
conc_gross_le_8_tdr_short_all: "44.7"
traders_tot_all: "450"
traders_m_money_long_all: "85"
traders_m_money_short_all: "62"
traders_m_money_spread_all: "20"
filename: "data/landing/cot/2024/abc123.csv.gzip"
expected:
rows:
- report_date: "2024-01-02"
cftc_commodity_code: "083731"
open_interest: 250000
managed_money_long: 60000
managed_money_short: 40000
managed_money_net: 20000
prod_merc_long: 80000
prod_merc_short: 90000
prod_merc_net: -10000
swap_long: 30000
swap_short: 35000
swap_net: -5000
nonreportable_long: 12000
nonreportable_short: 14000
nonreportable_net: -2000
change_managed_money_net: 3000
traders_managed_money_long: 85
traders_managed_money_short: 62
test_fct_cot_positioning_rejects_null_commodity:
model: foundation.fct_cot_positioning
inputs:
raw.cot_disaggregated:
rows:
- market_and_exchange_names: "SOME OTHER CONTRACT"
report_date_as_yyyy_mm_dd: "2024-01-02"
cftc_commodity_code: ""
cftc_contract_market_code: "999999"
contract_units: "N/A"
open_interest_all: "1000"
prod_merc_positions_long_all: "500"
prod_merc_positions_short_all: "500"
swap_positions_long_all: "0"
swap_positions_short_all: "0"
swap_positions_spread_all: "0"
m_money_positions_long_all: "0"
m_money_positions_short_all: "0"
m_money_positions_spread_all: "0"
other_rept_positions_long_all: "0"
other_rept_positions_short_all: "0"
other_rept_positions_spread_all: "0"
nonrept_positions_long_all: "0"
nonrept_positions_short_all: "0"
change_in_open_interest_all: "0"
change_in_m_money_long_all: "0"
change_in_m_money_short_all: "0"
change_in_prod_merc_long_all: "0"
change_in_prod_merc_short_all: "0"
conc_gross_le_4_tdr_long_all: "0"
conc_gross_le_4_tdr_short_all: "0"
conc_gross_le_8_tdr_long_all: "0"
conc_gross_le_8_tdr_short_all: "0"
traders_tot_all: "10"
traders_m_money_long_all: "0"
traders_m_money_short_all: "0"
traders_m_money_spread_all: "0"
filename: "data/landing/cot/2024/abc123.csv.gzip"
expected:
rows: []