/* Foundation fact: ICE certified Coffee C (Arabica) warehouse stocks. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* "Certified" means Coffee C graded and stamped as delivery-eligible */ /* against ICE futures contracts — a key physical supply indicator. */ /* Grain: one row per report_date. */
MODEL (
  name foundation.fct_ice_warehouse_stocks,
  kind INCREMENTAL_BY_TIME_RANGE (
    time_column report_date
  ),
  grain (
    report_date
  ),
  start '2000-01-01',
  cron '@daily'
);

WITH src AS (
  SELECT
    *
  FROM READ_CSV(
    @ice_stocks_glob(),
    compression = 'gzip',
    header = TRUE,
    union_by_name = TRUE,
    filename = TRUE,
    all_varchar = TRUE
  )
), cast_and_clean AS (
  SELECT
    TRY_CAST(report_date AS DATE) AS report_date,
    TRY_CAST(total_certified_bags AS BIGINT) AS total_certified_bags,
    TRY_CAST(pending_grading_bags AS BIGINT) AS pending_grading_bags,
    filename AS source_file,
    HASH(report_date, total_certified_bags) AS hkey /* Dedup key: report date + total bags */
  FROM src
  WHERE
    NOT TRY_CAST(report_date AS DATE) IS NULL
    AND NOT TRY_CAST(total_certified_bags AS BIGINT) IS NULL
), deduplicated AS (
  SELECT
    ANY_VALUE(report_date) AS report_date,
    ANY_VALUE(total_certified_bags) AS total_certified_bags,
    ANY_VALUE(pending_grading_bags) AS pending_grading_bags,
    ANY_VALUE(source_file) AS source_file,
    hkey
  FROM cast_and_clean
  GROUP BY
    hkey
)
SELECT
  *
FROM deduplicated
WHERE
  report_date BETWEEN @start_ds AND @end_ds