423fb8c619
extract: wrap response.content in BytesIO before passing to
normalize_zipped_csv, and call .read() on the returned BytesIO before
write_bytes (two bugs: wrong type in, wrong type out)
sqlmesh: {{ var() }} inside SQL string literals is not substituted by
SQLMesh's Jinja (SQL parser treats them as opaque strings). Replace with
a @psd_glob() macro that evaluates LANDING_DIR at render time and returns
a quoted glob path string.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
25 lines
1.0 KiB
SQL
25 lines
1.0 KiB
SQL
MODEL (
|
|
name raw.psd_alldata,
|
|
kind FULL,
|
|
grain ( commodity_code, country_code, market_year, calendar_year, month, attribute_id,unit_id ),
|
|
start '2006-08-01',
|
|
cron '@daily',
|
|
columns (
|
|
commodity_code varchar,
|
|
commodity_description varchar,
|
|
country_code varchar,
|
|
country_name varchar,
|
|
market_year varchar,
|
|
calendar_year varchar,
|
|
month varchar,
|
|
attribute_id varchar,
|
|
attribute_description varchar,
|
|
unit_id varchar,
|
|
unit_description varchar,
|
|
value varchar,
|
|
filename varchar
|
|
)
|
|
);
|
|
select *
|
|
FROM read_csv(@psd_glob(), delim=',', encoding='utf-8', compression='gzip', max_line_size=10000000, header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)
|