MODEL (
  name staging.psdalldata__commodity,
  kind INCREMENTAL_BY_TIME_RANGE (
    time_column ingest_date
  ),
  start '2006-08-01',
  cron '@daily'
);

WITH cast_dtypes AS (
  SELECT
    src.commodity_code::INT AS commodity_code,
    COALESCE(commodity_name, commodity_description) AS commodity_name,
    country_code::TEXT AS country_code,
    country_name,
    market_year::INT AS market_year,
    calendar_year::INT AS calendar_year,
    month::INT AS month,
    src.attribute_id::INT AS attribute_id,
    COALESCE(attribute_name, attribute_description) AS attribute_name,
    src.unit_id::INT AS unit_id,
    COALESCE(unit_name, unit_description) AS unit_name,
    value::REAL AS value,
    filename
  FROM READ_CSV(
    @psd_glob(),
    compression = 'gzip',
    header = TRUE,
    union_by_name = TRUE,
    filename = TRUE,
    all_varchar = TRUE,
    max_line_size = 10000000
  ) AS src
  LEFT JOIN seeds.psd_commodity_codes
    ON seeds.psd_commodity_codes.commodity_code = src.commodity_code::INT
  LEFT JOIN seeds.psd_unit_of_measure_codes
    ON seeds.psd_unit_of_measure_codes.unit_id = src.unit_id::INT
  LEFT JOIN seeds.psd_attribute_codes
    ON seeds.psd_attribute_codes.attribute_id = src.attribute_id::INT
), metadata_and_deduplication AS (
  SELECT
    ANY_VALUE(commodity_code) AS commodity_code,
    ANY_VALUE(commodity_name) AS commodity_name,
    ANY_VALUE(country_code) AS country_code,
    ANY_VALUE(country_name) AS country_name,
    ANY_VALUE(market_year) AS market_year,
    ANY_VALUE(calendar_year) AS calendar_year,
    ANY_VALUE(month) AS month,
    ANY_VALUE(attribute_id) AS attribute_id,
    ANY_VALUE(attribute_name) AS attribute_name,
    ANY_VALUE(unit_id) AS unit_id,
    ANY_VALUE(unit_name) AS unit_name,
    ANY_VALUE(value) AS value,
    HASH(
      commodity_code,
      commodity_name,
      country_code,
      country_name,
      market_year,
      calendar_year,
      month,
      attribute_id,
      attribute_name,
      unit_id,
      unit_name,
      value
    ) AS hkey,
    ANY_VALUE(
      MAKE_DATE(STR_SPLIT(filename, '/')[-3]::INT, STR_SPLIT(filename, '/')[-2]::INT, 1)
    ) AS ingest_date,
    ANY_VALUE(
      CASE WHEN month <> 0 THEN LAST_DAY(MAKE_DATE(market_year, month, 1)) ELSE NULL END
    ) AS market_date_month_end
  FROM cast_dtypes
  GROUP BY
    hkey
)
SELECT
  hkey,
  commodity_code,
  commodity_name,
  country_code,
  country_name,
  market_year,
  calendar_year,
  month,
  attribute_id,
  attribute_name,
  unit_id,
  unit_name,
  value,
  ingest_date
FROM metadata_and_deduplication
WHERE
  ingest_date BETWEEN @start_ds AND @end_ds