/* Foundation fact: ICE historical end-of-month Coffee C certified warehouse stocks by port. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* Covers November 1996 to present (30-year history). */ /* Grain: one row per report_date (end-of-month). */ /* Port columns are in bags (60kg). */
MODEL (
  name foundation.fct_ice_warehouse_stocks_by_port,
  kind INCREMENTAL_BY_TIME_RANGE (
    time_column report_date
  ),
  grain (
    report_date
  ),
  start '1996-11-01',
  cron '@daily'
);

WITH src AS (
  SELECT
    *
  FROM READ_CSV(
    @ice_stocks_by_port_glob(),
    compression = 'gzip',
    header = TRUE,
    union_by_name = TRUE,
    filename = TRUE,
    all_varchar = TRUE
  )
), cast_and_clean AS (
  SELECT
    TRY_CAST(report_date AS DATE) AS report_date,
    TRY_CAST(new_york_bags AS BIGINT) AS new_york_bags,
    TRY_CAST(new_orleans_bags AS BIGINT) AS new_orleans_bags,
    TRY_CAST(houston_bags AS BIGINT) AS houston_bags,
    TRY_CAST(miami_bags AS BIGINT) AS miami_bags,
    TRY_CAST(antwerp_bags AS BIGINT) AS antwerp_bags,
    TRY_CAST(hamburg_bremen_bags AS BIGINT) AS hamburg_bremen_bags,
    TRY_CAST(barcelona_bags AS BIGINT) AS barcelona_bags,
    TRY_CAST(virginia_bags AS BIGINT) AS virginia_bags,
    TRY_CAST(total_bags AS BIGINT) AS total_bags,
    filename AS source_file,
    HASH(report_date, total_bags) AS hkey
  FROM src
  WHERE
    NOT TRY_CAST(report_date AS DATE) IS NULL
    AND NOT TRY_CAST(total_bags AS BIGINT) IS NULL
), deduplicated AS (
  SELECT
    ANY_VALUE(report_date) AS report_date,
    ANY_VALUE(new_york_bags) AS new_york_bags,
    ANY_VALUE(new_orleans_bags) AS new_orleans_bags,
    ANY_VALUE(houston_bags) AS houston_bags,
    ANY_VALUE(miami_bags) AS miami_bags,
    ANY_VALUE(antwerp_bags) AS antwerp_bags,
    ANY_VALUE(hamburg_bremen_bags) AS hamburg_bremen_bags,
    ANY_VALUE(barcelona_bags) AS barcelona_bags,
    ANY_VALUE(virginia_bags) AS virginia_bags,
    ANY_VALUE(total_bags) AS total_bags,
    ANY_VALUE(source_file) AS source_file,
    hkey
  FROM cast_and_clean
  GROUP BY
    hkey
)
SELECT
  *
FROM deduplicated
WHERE
  report_date BETWEEN @start_ds AND @end_ds