raw ingest model

transform/sqlmesh_materia/config.yaml

@@ -7,7 +7,7 @@ gateways:
       # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection
       # https://sqlmesh.readthedocs.io/en/stable/integrations/engines/duckdb/#connection-options
       type: duckdb
-      database: materia_dev_db.db
+      database: materia_dev.db
       extensions:
         - name: zipfs
 
@@ -17,7 +17,7 @@ gateways:
       # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection
       # https://sqlmesh.readthedocs.io/en/stable/integrations/engines/duckdb/#connection-options
       type: duckdb
-      database: materia_prod_db.db
+      database: materia_prod.db
       extensions:
         - name: zipfs
      

transform/sqlmesh_materia/models/raw/raw_psd_data.sql

@@ -1,7 +1,21 @@
 MODEL (
-name raw.psd_data,
-kind SEED (
-path '../../../../extract/psdonline/src/psdonline/data/**/*.csv'
-),
-cron '@daily',
+  name raw.psd_data,
+  kind FULL,
+  start '2006-08-01',
+  cron '@daily' -- This cron determines how often SQLMesh checks for new data.
 );
+
+SELECT
+	Commodity_Code::VARCHAR as commodity_code,
+	Commodity_Description::VARCHAR as commodity_description,
+	Country_Code::VARCHAR as country_code,
+	Country_Name::VARCHAR as country_name,
+	Market_Year::BIGINT as market_year,
+	Calendar_Year::BIGINT as calendar_year,
+	Month::VARCHAR as month,
+	Attribute_ID::VARCHAR as attribute_id,
+	Attribute_Description::VARCHAR as attribute_description,
+	Unit_ID::VARCHAR as unit_id,
+	Unit_Description::VARCHAR as unit_description,
+	Value::DOUBLE as value,
+  FROM read_csv_auto('zip:///home/deeman/projects/materia/extract/psdonline/src/psdonline/data/**/*.zip/*.csv', header=true, union_by_name=true)

transform/sqlmesh_materia/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 ]
 requires-python = ">=3.13"
 dependencies = [
-    "sqlmesh>=0.200.0",
+    "sqlmesh[duckdb,lsp]>=0.200.0",
 ]
 
 

transform/sqlmesh_materia/tests/test_full_model.yaml

@@ -1,19 +0,0 @@
-test_example_full_model:
-  model: sqlmesh_example.full_model
-  inputs:
-    sqlmesh_example.incremental_model:
-      rows:
-      - id: 1
-        item_id: 1
-      - id: 2
-        item_id: 1
-      - id: 3
-        item_id: 2
-  outputs:
-    query:
-      rows:
-      - item_id: 1
-        num_orders: 2
-      - item_id: 2
-        num_orders: 1
-