fix seeds; update models
This commit is contained in:
26
transform/sqlmesh_materia/models/raw/psd_data.sql
Normal file
26
transform/sqlmesh_materia/models/raw/psd_data.sql
Normal file
@@ -0,0 +1,26 @@
|
||||
MODEL (
|
||||
name raw.psd_alldata,
|
||||
kind FULL,
|
||||
start '2006-08-01',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
SELECT
|
||||
*
|
||||
--format('{}-{}-01',split(filename, '/')[-4],split(filename, '/')[-3])::date as ingest_date
|
||||
FROM read_csv('zip:///home/deeman/projects/materia/extract/psdonline/src/psdonline/data/**/*.zip/*.csv', header=true, union_by_name=true, filename=true, names = ['commodity_code', 'commodity_description', 'country_code', 'country_name', 'market_year', 'calendar_year', 'month', 'attribute_id', 'attribute_description', 'unit_id', 'unit_description', 'value'], types=
|
||||
{
|
||||
'commodity_code' : 'VARCHAR',
|
||||
'commodity_description' :'VARCHAR',
|
||||
'country_code' : 'VARCHAR',
|
||||
'country_name' : 'VARCHAR',
|
||||
'market_year' : 'BIGINT' ,
|
||||
'calendar_year' : 'BIGINT' ,
|
||||
'month' : 'VARCHAR',
|
||||
'attribute_id' : 'VARCHAR',
|
||||
'attribute_description' :'VARCHAR',
|
||||
'unit_id' : 'VARCHAR',
|
||||
'unit_description' : 'VARCHAR',
|
||||
'value' : 'DOUBLE'
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user