0a83b2cb74
- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization - SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart) - dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology - fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities - obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios - Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest] - Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart - 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
86 lines
3.3 KiB
SQL
86 lines
3.3 KiB
SQL
-- Raw CFTC Commitment of Traders — Disaggregated Futures Only.
|
||
--
|
||
-- Technical ingestion layer only: reads gzip CSVs from the landing directory
|
||
-- and surfaces the columns needed by downstream foundation models.
|
||
-- All values are varchar; casting happens in foundation.
|
||
--
|
||
-- Source: CFTC yearly ZIPs at
|
||
-- https://www.cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip
|
||
-- Coverage: June 2006 – present (new file every Friday at 3:30 PM ET)
|
||
|
||
MODEL (
|
||
name raw.cot_disaggregated,
|
||
kind FULL,
|
||
grain (cftc_commodity_code, report_date_as_yyyy_mm_dd, cftc_contract_market_code),
|
||
start '2006-06-13',
|
||
cron '@daily'
|
||
);
|
||
|
||
SELECT
|
||
-- Identifiers
|
||
"Market_and_Exchange_Names" AS market_and_exchange_names,
|
||
"Report_Date_as_YYYY-MM-DD" AS report_date_as_yyyy_mm_dd,
|
||
"CFTC_Commodity_Code" AS cftc_commodity_code,
|
||
"CFTC_Contract_Market_Code" AS cftc_contract_market_code,
|
||
"Contract_Units" AS contract_units,
|
||
|
||
-- Open interest
|
||
"Open_Interest_All" AS open_interest_all,
|
||
|
||
-- Producer / Merchant / Processor / User (commercial hedgers)
|
||
"Prod_Merc_Positions_Long_All" AS prod_merc_positions_long_all,
|
||
"Prod_Merc_Positions_Short_All" AS prod_merc_positions_short_all,
|
||
|
||
-- Swap dealers
|
||
"Swap_Positions_Long_All" AS swap_positions_long_all,
|
||
"Swap__Positions_Short_All" AS swap_positions_short_all,
|
||
"Swap__Positions_Spread_All" AS swap_positions_spread_all,
|
||
|
||
-- Managed money (hedge funds, CTAs — key speculative signal)
|
||
"M_Money_Positions_Long_All" AS m_money_positions_long_all,
|
||
"M_Money_Positions_Short_All" AS m_money_positions_short_all,
|
||
"M_Money_Positions_Spread_All" AS m_money_positions_spread_all,
|
||
|
||
-- Other reportables
|
||
"Other_Rept_Positions_Long_All" AS other_rept_positions_long_all,
|
||
"Other_Rept_Positions_Short_All" AS other_rept_positions_short_all,
|
||
"Other_Rept_Positions_Spread_All" AS other_rept_positions_spread_all,
|
||
|
||
-- Non-reportable (small speculators)
|
||
"NonRept_Positions_Long_All" AS nonrept_positions_long_all,
|
||
"NonRept_Positions_Short_All" AS nonrept_positions_short_all,
|
||
|
||
-- Week-over-week changes
|
||
"Change_in_Open_Interest_All" AS change_in_open_interest_all,
|
||
"Change_in_M_Money_Long_All" AS change_in_m_money_long_all,
|
||
"Change_in_M_Money_Short_All" AS change_in_m_money_short_all,
|
||
"Change_in_Prod_Merc_Long_All" AS change_in_prod_merc_long_all,
|
||
"Change_in_Prod_Merc_Short_All" AS change_in_prod_merc_short_all,
|
||
|
||
-- Concentration (% of OI held by top 4 and top 8 traders)
|
||
"Conc_Gross_LE_4_TDR_Long_All" AS conc_gross_le_4_tdr_long_all,
|
||
"Conc_Gross_LE_4_TDR_Short_All" AS conc_gross_le_4_tdr_short_all,
|
||
"Conc_Gross_LE_8_TDR_Long_All" AS conc_gross_le_8_tdr_long_all,
|
||
"Conc_Gross_LE_8_TDR_Short_All" AS conc_gross_le_8_tdr_short_all,
|
||
|
||
-- Trader counts
|
||
"Traders_Tot_All" AS traders_tot_all,
|
||
"Traders_M_Money_Long_All" AS traders_m_money_long_all,
|
||
"Traders_M_Money_Short_All" AS traders_m_money_short_all,
|
||
"Traders_M_Money_Spread_All" AS traders_m_money_spread_all,
|
||
|
||
-- Lineage
|
||
filename
|
||
FROM read_csv(
|
||
@cot_glob(),
|
||
delim = ',',
|
||
encoding = 'utf-8',
|
||
compression = 'gzip',
|
||
header = true,
|
||
union_by_name = true,
|
||
filename = true,
|
||
all_varchar = true,
|
||
max_line_size = 10000000,
|
||
ignore_errors = true
|
||
)
|