MODEL (
  name staging.psdalldata__commodity,
  kind INCREMENTAL_BY_TIME_RANGE (
    time_column ingest_date
  ),
  start '2006-08-01',
  cron '@daily'
);
with cast_dtypes as (
  SELECT
	raw.psd_alldata.commodity_code::int as commodity_code,
	coalesce(commodity_name, commodity_description) as commodity_name,
	country_code::varchar(3) as country_code,
	country_name,
	market_year::int as market_year,
	calendar_year::int as calendar_year,
	month::int as month,
	raw.psd_alldata.attribute_id::int as attribute_id,
	coalesce(attribute_name, attribute_description) as attribute_name,
	raw.psd_alldata.unit_id::int as unit_id,
	coalesce(unit_name, unit_description) as unit_name,
	value::float as value,
	filename
	FROM raw.psd_alldata
  left join raw.psd_commodity_codes on raw.psd_commodity_codes.commodity_code = raw.psd_alldata.commodity_code::int
  left join raw.psd_unit_of_measure_codes on raw.psd_unit_of_measure_codes.unit_id = raw.psd_alldata.unit_id::int 
  left join raw.psd_attribute_codes on raw.psd_attribute_codes.attribute_id = raw.psd_alldata.attribute_id::int 
),
metadata_and_deduplication as (
select 
    any_value(commodity_code) as commodity_code,
    any_value(commodity_name) as commodity_name,
    any_value(country_code) as country_code,
    any_value(country_name) as country_name,
    any_value(market_year) as market_year,
    any_value(calendar_year) as calendar_year,
    any_value(month) as month,
    any_value(attribute_id) as attribute_id,
    any_value(attribute_name) as attribute_name,
    any_value(unit_id) as unit_id,
    any_value(unit_name) as unit_name,
    any_value(value) as value, 
    hash(commodity_code, commodity_name, country_code, country_name, market_year, calendar_year, month, attribute_id, attribute_name, unit_id, unit_name, value) as hkey,
    any_value(make_date(split(filename, '/')[-3]::int, split(filename, '/')[-2]::int, 1)) as ingest_date,
    any_value(if(month!=0,last_day(make_date(market_year, month, 1)),null)) as market_date_month_end,
  from cast_dtypes
  group by hkey
)
select  hkey,
 	commodity_code,
	commodity_name,
	country_code,
	country_name,
	market_year,
	calendar_year,
	month,
	attribute_id,
	attribute_name,
	unit_id,
	unit_name,
	value,
	ingest_date,
from metadata_and_deduplication
where ingest_date between @start_ds and @end_ds;
