deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update SQLMesh for R2 data access & Convert psd data to gzip

Browse Source
This commit is contained in:
Hendrik Dreesmann
2025-11-02 00:26:01 +01:00
parent fc27d5f887
commit b702e6565a
26 changed files with 3553 additions and 1786 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
transform/sqlmesh_materia/models/raw/psd_data.sql
View File

@@ -20,5 +20,5 @@ MODEL (
filename varchar
)
);
SELECT *
FROM read_csv('zip://extract/psdonline/src/psdonline/data/*.zip/*.csv', header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)
select *
FROM read_csv('extract/psdonline/src/psdonline/data/*.csv.gzip', delim=',', encoding='utf-8', compression='gzip', max_line_size=10000000, header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], all_varchar=true)