diff --git a/transform/sqlmesh_materia/models/raw/raw_psd_data.sql b/transform/sqlmesh_materia/models/raw/raw_psd_data.sql new file mode 100644 index 0000000..01b1996 --- /dev/null +++ b/transform/sqlmesh_materia/models/raw/raw_psd_data.sql @@ -0,0 +1,7 @@ +MODEL ( +name raw.psd_data, +kind SEED ( +path '../../../../extract/psdonline/src/psdonline/data/**/*.csv' +), +cron '@daily', +); diff --git a/transform/sqlmesh_materia/schema.yml b/transform/sqlmesh_materia/schema.yml new file mode 100644 index 0000000..b7ddf8f --- /dev/null +++ b/transform/sqlmesh_materia/schema.yml @@ -0,0 +1,21 @@ +model: + name: raw.raw_psd_alldata # schema.table_name format + kind: EXTERNAL # This is the key part! + path: ../../../../extract/psdonline/src/psdonline/data/**/*.zip # Path relative to the project root + format: csv # Specify the file format + dialect: duckdb # The dialect that will read this file + options: + auto_detect: true + columns: + commodity_code: string + commodity_description: string + country_code: string + country_name: string + market_year: int + calendar_year: int + month: int + attribute_id: string + attribute_description: string + unit_id: string + unit_description: string + value: float