diff --git a/transform/sqlmesh_materia/models/raw/raw_psd_data.sql b/transform/sqlmesh_materia/models/raw/raw_psd_data.sql deleted file mode 100644 index 0e4e0ba..0000000 --- a/transform/sqlmesh_materia/models/raw/raw_psd_data.sql +++ /dev/null @@ -1,21 +0,0 @@ -MODEL ( - name raw.psd_data, - kind FULL, - start '2006-08-01', - cron '@daily' -- This cron determines how often SQLMesh checks for new data. -); - -SELECT - Commodity_Code::VARCHAR as commodity_code, - Commodity_Description::VARCHAR as commodity_description, - Country_Code::VARCHAR as country_code, - Country_Name::VARCHAR as country_name, - Market_Year::BIGINT as market_year, - Calendar_Year::BIGINT as calendar_year, - Month::VARCHAR as month, - Attribute_ID::VARCHAR as attribute_id, - Attribute_Description::VARCHAR as attribute_description, - Unit_ID::VARCHAR as unit_id, - Unit_Description::VARCHAR as unit_description, - Value::DOUBLE as value, - FROM read_csv_auto('zip:///home/deeman/projects/materia/extract/psdonline/src/psdonline/data/**/*.zip/*.csv', header=true, union_by_name=true) diff --git a/transform/sqlmesh_materia/schema.yml b/transform/sqlmesh_materia/schema.yml index b7ddf8f..957c320 100644 --- a/transform/sqlmesh_materia/schema.yml +++ b/transform/sqlmesh_materia/schema.yml @@ -1,7 +1,6 @@ model: - name: raw.raw_psd_alldata # schema.table_name format + name: raw.psd_alldata # schema.table_name format kind: EXTERNAL # This is the key part! - path: ../../../../extract/psdonline/src/psdonline/data/**/*.zip # Path relative to the project root format: csv # Specify the file format dialect: duckdb # The dialect that will read this file options: