deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add reference data

Browse Source
This commit is contained in:
Deeman
2025-07-27 18:28:30 +02:00
parent ff283b62ff
commit c0d8f60d1c
2 changed files with 1 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
transform/sqlmesh_materia/models/raw/raw_psd_data.sql
View File

@@ -1,21 +0,0 @@
MODEL (
name raw.psd_data,
kind FULL,
start '2006-08-01',
cron '@daily' -- This cron determines how often SQLMesh checks for new data.
);
SELECT
Commodity_Code::VARCHAR as commodity_code,
Commodity_Description::VARCHAR as commodity_description,
Country_Code::VARCHAR as country_code,
Country_Name::VARCHAR as country_name,
Market_Year::BIGINT as market_year,
Calendar_Year::BIGINT as calendar_year,
Month::VARCHAR as month,
Attribute_ID::VARCHAR as attribute_id,
Attribute_Description::VARCHAR as attribute_description,
Unit_ID::VARCHAR as unit_id,
Unit_Description::VARCHAR as unit_description,
Value::DOUBLE as value,
FROM read_csv_auto('zip:///home/deeman/projects/materia/extract/psdonline/src/psdonline/data/**/*.zip/*.csv', header=true, union_by_name=true)

3
transform/sqlmesh_materia/schema.yml
View File

@@ -1,7 +1,6 @@
model: model:
name: raw.raw_psd_alldata # schema.table_name format name: raw.psd_alldata # schema.table_name format
kind: EXTERNAL # This is the key part! kind: EXTERNAL # This is the key part!
path: ../../../../extract/psdonline/src/psdonline/data/**/*.zip # Path relative to the project root
format: csv # Specify the file format format: csv # Specify the file format
dialect: duckdb # The dialect that will read this file dialect: duckdb # The dialect that will read this file
options: options: