Fix COT pipeline: TRY_CAST nulls, dim_commodity leading zeros, correct CFTC codes
- config.yaml: remove ambiguousorinvalidcolumn linter rule (false positives on read_csv TVFs) - fct_cot_positioning: use TRY_CAST throughout — CFTC uses '.' as null in many columns - raw/cot_disaggregated: add columns() declaration for 33 varchar cols - dim_commodity: switch from SEED to FULL model with SQL VALUES to preserve leading zeros Pandas auto-converts '083' → 83 even with varchar column declarations in SEED models - seeds/dim_commodity.csv: correct cftc_commodity_code from '083731' (contract market code) to '083' (3-digit CFTC commodity code); add CSV quoting - test_cot_foundation.yaml: fix output key name, vars for time range, partial: true, and correct cftc_commodity_code to '083' - analytics.py: COFFEE_CFTC_CODE '083731' → '083' to match actual data Result: serving.cot_positioning has 685 rows (2013-01-08 to 2026-02-17), 23/23 tests pass. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
test_fct_cot_positioning_types_and_net_positions:
|
||||
model: foundation.fct_cot_positioning
|
||||
vars:
|
||||
start: "2024-01-01"
|
||||
end: "2024-01-31"
|
||||
inputs:
|
||||
raw.cot_disaggregated:
|
||||
rows:
|
||||
- market_and_exchange_names: "COFFEE C - ICE FUTURES U.S."
|
||||
report_date_as_yyyy_mm_dd: "2024-01-02"
|
||||
cftc_commodity_code: "083731"
|
||||
cftc_commodity_code: "083"
|
||||
cftc_contract_market_code: "083731"
|
||||
contract_units: "37,500 POUNDS"
|
||||
open_interest_all: "250000"
|
||||
@@ -36,29 +39,34 @@ test_fct_cot_positioning_types_and_net_positions:
|
||||
traders_m_money_short_all: "62"
|
||||
traders_m_money_spread_all: "20"
|
||||
filename: "data/landing/cot/2024/abc123.csv.gzip"
|
||||
expected:
|
||||
rows:
|
||||
- report_date: "2024-01-02"
|
||||
cftc_commodity_code: "083731"
|
||||
open_interest: 250000
|
||||
managed_money_long: 60000
|
||||
managed_money_short: 40000
|
||||
managed_money_net: 20000
|
||||
prod_merc_long: 80000
|
||||
prod_merc_short: 90000
|
||||
prod_merc_net: -10000
|
||||
swap_long: 30000
|
||||
swap_short: 35000
|
||||
swap_net: -5000
|
||||
nonreportable_long: 12000
|
||||
nonreportable_short: 14000
|
||||
nonreportable_net: -2000
|
||||
change_managed_money_net: 3000
|
||||
traders_managed_money_long: 85
|
||||
traders_managed_money_short: 62
|
||||
outputs:
|
||||
partial: true
|
||||
query:
|
||||
rows:
|
||||
- report_date: "2024-01-02"
|
||||
cftc_commodity_code: "083"
|
||||
open_interest: 250000
|
||||
managed_money_long: 60000
|
||||
managed_money_short: 40000
|
||||
managed_money_net: 20000
|
||||
prod_merc_long: 80000
|
||||
prod_merc_short: 90000
|
||||
prod_merc_net: -10000
|
||||
swap_long: 30000
|
||||
swap_short: 35000
|
||||
swap_net: -5000
|
||||
nonreportable_long: 12000
|
||||
nonreportable_short: 14000
|
||||
nonreportable_net: -2000
|
||||
change_managed_money_net: 3000
|
||||
traders_managed_money_long: 85
|
||||
traders_managed_money_short: 62
|
||||
|
||||
test_fct_cot_positioning_rejects_null_commodity:
|
||||
model: foundation.fct_cot_positioning
|
||||
vars:
|
||||
start: "2024-01-01"
|
||||
end: "2024-01-31"
|
||||
inputs:
|
||||
raw.cot_disaggregated:
|
||||
rows:
|
||||
@@ -95,5 +103,6 @@ test_fct_cot_positioning_rejects_null_commodity:
|
||||
traders_m_money_short_all: "0"
|
||||
traders_m_money_spread_all: "0"
|
||||
filename: "data/landing/cot/2024/abc123.csv.gzip"
|
||||
expected:
|
||||
rows: []
|
||||
outputs:
|
||||
query:
|
||||
rows: []
|
||||
|
||||
Reference in New Issue
Block a user