refactor: rename materia → beanflows throughout codebase
- Rename src/materia/ → src/beanflows/ (Python package)
- Rename transform/sqlmesh_materia/ → transform/sqlmesh_beanflows/
- Rename infra/supervisor/materia-supervisor.service → beanflows-supervisor.service
- Rename infra/backup/materia-backup.{service,timer} → beanflows-backup.{service,timer}
- Update all path strings: /opt/materia → /opt/beanflows, /data/materia → /data/beanflows
- Update pyproject.toml: project name, CLI entrypoint, workspace source key
- Update all internal imports from materia.* → beanflows.*
- Update infra scripts: REPO_DIR, service names, systemctl references
- Fix docker-compose.prod.yml: /data/materia → /data/beanflows (bind mount path)
Intentionally left unchanged: Pulumi stack name (materia-infrastructure) and
Hetzner resource names ("materia-key", "managed_by: materia") — these reference
live cloud infrastructure and require separate cloud-side renames.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
@@ -0,0 +1,15 @@
|
||||
/* Commodity dimension: conforms identifiers across source systems. */ /* This is the ontology. Each row is a commodity tracked by BeanFlows. */ /* As new sources are added (ICO, futures prices, satellite), their */ /* commodity identifiers are added as columns here — not as separate tables. */ /* As new commodities are added (cocoa, sugar), rows are added here. */ /* References: */ /* usda_commodity_code → staging.psdalldata__commodity.commodity_code (numeric string, e.g. '0711100') */ /* cftc_commodity_code → foundation.fct_cot_positioning.cftc_commodity_code (3-char, e.g. '083') */ /* NOTE: Defined as FULL model (not SEED) to guarantee leading-zero preservation. */ /* Pandas CSV loading converts '083' → 83 even with varchar column declarations. */
|
||||
MODEL (
|
||||
name foundation.dim_commodity,
|
||||
kind FULL
|
||||
);
|
||||
|
||||
SELECT
|
||||
usda_commodity_code,
|
||||
cftc_commodity_code,
|
||||
ticker,
|
||||
ice_stock_report_code,
|
||||
commodity_name,
|
||||
commodity_group
|
||||
FROM (VALUES
|
||||
('0711100', '083', 'KC=F', 'COFFEE-C', 'Coffee, Green', 'Softs')) AS t(usda_commodity_code, cftc_commodity_code, ticker, ice_stock_report_code, commodity_name, commodity_group)
|
||||
@@ -0,0 +1,58 @@
|
||||
/* Foundation fact: daily KC=F Coffee C futures prices. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* Covers all available history from the landing directory. */ /* Grain: one row per trade_date. */ /* Dedup: hash of (trade_date, close) — if Yahoo Finance corrects a price, */ /* the new hash triggers a re-ingest on the next incremental run. */
|
||||
MODEL (
|
||||
name foundation.fct_coffee_prices,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column trade_date
|
||||
),
|
||||
grain (
|
||||
trade_date
|
||||
),
|
||||
start '1971-08-16',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@prices_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE
|
||||
)
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
TRY_CAST(Date AS DATE) AS trade_date,
|
||||
TRY_CAST(Open AS DOUBLE) AS open,
|
||||
TRY_CAST(High AS DOUBLE) AS high,
|
||||
TRY_CAST(Low AS DOUBLE) AS low,
|
||||
TRY_CAST(Close AS DOUBLE) AS close,
|
||||
TRY_CAST("Adj Close" AS DOUBLE) AS adj_close,
|
||||
TRY_CAST(Volume AS BIGINT) AS volume,
|
||||
filename AS source_file, /* Filename encodes the content hash — use as ingest identifier */
|
||||
HASH(Date, Close) AS hkey /* Dedup key: trade date + close price */
|
||||
FROM src
|
||||
WHERE
|
||||
NOT TRY_CAST(Date AS DATE) IS NULL AND NOT TRY_CAST(Close AS DOUBLE) IS NULL
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(trade_date) AS trade_date,
|
||||
ANY_VALUE(open) AS open,
|
||||
ANY_VALUE(high) AS high,
|
||||
ANY_VALUE(low) AS low,
|
||||
ANY_VALUE(close) AS close,
|
||||
ANY_VALUE(adj_close) AS adj_close,
|
||||
ANY_VALUE(volume) AS volume,
|
||||
ANY_VALUE(source_file) AS source_file,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM deduplicated
|
||||
WHERE
|
||||
trade_date BETWEEN @start_ds AND @end_ds
|
||||
@@ -0,0 +1,146 @@
|
||||
/* Foundation fact: CFTC COT positioning, weekly grain, all commodities. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* cleans column names, computes net positions (long - short) per trader category, */ /* and deduplicates via hash key. Covers all commodities — filtering to */ /* a specific commodity happens in the serving layer. */ /* Grain: one row per (cftc_commodity_code, report_date, cftc_contract_market_code) */ /* History: revisions appear as new rows with a later ingest_date. */ /* Serving layer picks max(ingest_date) per grain for latest view. */
|
||||
MODEL (
|
||||
name foundation.fct_cot_positioning,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column report_date
|
||||
),
|
||||
grain (cftc_commodity_code, report_date, cftc_contract_market_code, ingest_date, report_type),
|
||||
start '2006-06-13',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@cot_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE,
|
||||
max_line_size = 10000000
|
||||
)
|
||||
UNION ALL BY NAME
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@cot_combined_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE,
|
||||
max_line_size = 10000000
|
||||
)
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
TRIM(market_and_exchange_names) AS market_and_exchange_name, /* Identifiers */
|
||||
"Report_Date_as_YYYY-MM-DD"::DATE AS report_date,
|
||||
TRIM(cftc_commodity_code) AS cftc_commodity_code,
|
||||
TRIM(cftc_contract_market_code) AS cftc_contract_market_code,
|
||||
TRIM(contract_units) AS contract_units,
|
||||
TRIM("FutOnly_or_Combined") AS report_type, /* 'FutOnly' or 'Combined' — discriminates the two CFTC report variants */
|
||||
TRY_CAST(open_interest_all AS INT) AS open_interest, /* Open interest */ /* CFTC uses '.' as null for any field — use TRY_CAST throughout */
|
||||
TRY_CAST(prod_merc_positions_long_all AS INT) AS prod_merc_long, /* Producer / Merchant (commercial hedgers: exporters, processors) */
|
||||
TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_short,
|
||||
TRY_CAST(swap_positions_long_all AS INT) AS swap_long, /* Swap dealers */
|
||||
TRY_CAST("Swap__Positions_Short_All" AS INT) AS swap_short,
|
||||
TRY_CAST("Swap__Positions_Spread_All" AS INT) AS swap_spread,
|
||||
TRY_CAST(m_money_positions_long_all AS INT) AS managed_money_long, /* Managed money (hedge funds, CTAs — the primary speculative signal) */
|
||||
TRY_CAST(m_money_positions_short_all AS INT) AS managed_money_short,
|
||||
TRY_CAST(m_money_positions_spread_all AS INT) AS managed_money_spread,
|
||||
TRY_CAST(other_rept_positions_long_all AS INT) AS other_reportable_long, /* Other reportables */
|
||||
TRY_CAST(other_rept_positions_short_all AS INT) AS other_reportable_short,
|
||||
TRY_CAST(other_rept_positions_spread_all AS INT) AS other_reportable_spread,
|
||||
TRY_CAST(nonrept_positions_long_all AS INT) AS nonreportable_long, /* Non-reportable (small speculators, below reporting threshold) */
|
||||
TRY_CAST(nonrept_positions_short_all AS INT) AS nonreportable_short,
|
||||
TRY_CAST(prod_merc_positions_long_all AS INT) /* Net positions (long minus short per category) */ - TRY_CAST(prod_merc_positions_short_all AS INT) AS prod_merc_net,
|
||||
TRY_CAST(m_money_positions_long_all AS INT) - TRY_CAST(m_money_positions_short_all AS INT) AS managed_money_net,
|
||||
TRY_CAST(swap_positions_long_all AS INT) - TRY_CAST("Swap__Positions_Short_All" AS INT) AS swap_net,
|
||||
TRY_CAST(other_rept_positions_long_all AS INT) - TRY_CAST(other_rept_positions_short_all AS INT) AS other_reportable_net,
|
||||
TRY_CAST(nonrept_positions_long_all AS INT) - TRY_CAST(nonrept_positions_short_all AS INT) AS nonreportable_net,
|
||||
TRY_CAST(change_in_open_interest_all AS INT) AS change_open_interest, /* Week-over-week changes */
|
||||
TRY_CAST(change_in_m_money_long_all AS INT) AS change_managed_money_long,
|
||||
TRY_CAST(change_in_m_money_short_all AS INT) AS change_managed_money_short,
|
||||
TRY_CAST(change_in_m_money_long_all AS INT) - TRY_CAST(change_in_m_money_short_all AS INT) AS change_managed_money_net,
|
||||
TRY_CAST(change_in_prod_merc_long_all AS INT) AS change_prod_merc_long,
|
||||
TRY_CAST(change_in_prod_merc_short_all AS INT) AS change_prod_merc_short,
|
||||
TRY_CAST(conc_gross_le_4_tdr_long_all AS REAL) AS concentration_top4_long_pct, /* Concentration ratios (% of OI held by top 4 / top 8 traders) */
|
||||
TRY_CAST(conc_gross_le_4_tdr_short_all AS REAL) AS concentration_top4_short_pct,
|
||||
TRY_CAST(conc_gross_le_8_tdr_long_all AS REAL) AS concentration_top8_long_pct,
|
||||
TRY_CAST(conc_gross_le_8_tdr_short_all AS REAL) AS concentration_top8_short_pct,
|
||||
TRY_CAST(traders_tot_all AS INT) AS traders_total, /* Trader counts */
|
||||
TRY_CAST(traders_m_money_long_all AS INT) AS traders_managed_money_long,
|
||||
TRY_CAST(traders_m_money_short_all AS INT) AS traders_managed_money_short,
|
||||
TRY_CAST(traders_m_money_spread_all AS INT) AS traders_managed_money_spread,
|
||||
MAKE_DATE(STR_SPLIT(filename, '/')[-2]::INT, 1, 1) AS ingest_date, /* Ingest date: derived from landing path year directory */ /* Path: .../cot/{year}/{etag}.csv.gzip → extract year from [-2] */
|
||||
HASH(
|
||||
cftc_commodity_code,
|
||||
"Report_Date_as_YYYY-MM-DD",
|
||||
cftc_contract_market_code,
|
||||
"FutOnly_or_Combined",
|
||||
open_interest_all,
|
||||
m_money_positions_long_all,
|
||||
m_money_positions_short_all,
|
||||
prod_merc_positions_long_all,
|
||||
prod_merc_positions_short_all
|
||||
) AS hkey /* Dedup key: hash of business grain + key metrics; includes report variant so fut-only and combined rows get distinct keys */
|
||||
FROM src
|
||||
/* Reject rows with null commodity code or malformed date */
|
||||
WHERE
|
||||
NOT TRIM(cftc_commodity_code) IS NULL
|
||||
AND LENGTH(TRIM(cftc_commodity_code)) > 0
|
||||
AND NOT "Report_Date_as_YYYY-MM-DD"::DATE IS NULL
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(market_and_exchange_name) AS market_and_exchange_name,
|
||||
ANY_VALUE(report_date) AS report_date,
|
||||
ANY_VALUE(cftc_commodity_code) AS cftc_commodity_code,
|
||||
ANY_VALUE(cftc_contract_market_code) AS cftc_contract_market_code,
|
||||
ANY_VALUE(contract_units) AS contract_units,
|
||||
ANY_VALUE(open_interest) AS open_interest,
|
||||
ANY_VALUE(prod_merc_long) AS prod_merc_long,
|
||||
ANY_VALUE(prod_merc_short) AS prod_merc_short,
|
||||
ANY_VALUE(prod_merc_net) AS prod_merc_net,
|
||||
ANY_VALUE(swap_long) AS swap_long,
|
||||
ANY_VALUE(swap_short) AS swap_short,
|
||||
ANY_VALUE(swap_spread) AS swap_spread,
|
||||
ANY_VALUE(swap_net) AS swap_net,
|
||||
ANY_VALUE(managed_money_long) AS managed_money_long,
|
||||
ANY_VALUE(managed_money_short) AS managed_money_short,
|
||||
ANY_VALUE(managed_money_spread) AS managed_money_spread,
|
||||
ANY_VALUE(managed_money_net) AS managed_money_net,
|
||||
ANY_VALUE(other_reportable_long) AS other_reportable_long,
|
||||
ANY_VALUE(other_reportable_short) AS other_reportable_short,
|
||||
ANY_VALUE(other_reportable_spread) AS other_reportable_spread,
|
||||
ANY_VALUE(other_reportable_net) AS other_reportable_net,
|
||||
ANY_VALUE(nonreportable_long) AS nonreportable_long,
|
||||
ANY_VALUE(nonreportable_short) AS nonreportable_short,
|
||||
ANY_VALUE(nonreportable_net) AS nonreportable_net,
|
||||
ANY_VALUE(change_open_interest) AS change_open_interest,
|
||||
ANY_VALUE(change_managed_money_long) AS change_managed_money_long,
|
||||
ANY_VALUE(change_managed_money_short) AS change_managed_money_short,
|
||||
ANY_VALUE(change_managed_money_net) AS change_managed_money_net,
|
||||
ANY_VALUE(change_prod_merc_long) AS change_prod_merc_long,
|
||||
ANY_VALUE(change_prod_merc_short) AS change_prod_merc_short,
|
||||
ANY_VALUE(concentration_top4_long_pct) AS concentration_top4_long_pct,
|
||||
ANY_VALUE(concentration_top4_short_pct) AS concentration_top4_short_pct,
|
||||
ANY_VALUE(concentration_top8_long_pct) AS concentration_top8_long_pct,
|
||||
ANY_VALUE(concentration_top8_short_pct) AS concentration_top8_short_pct,
|
||||
ANY_VALUE(traders_total) AS traders_total,
|
||||
ANY_VALUE(traders_managed_money_long) AS traders_managed_money_long,
|
||||
ANY_VALUE(traders_managed_money_short) AS traders_managed_money_short,
|
||||
ANY_VALUE(traders_managed_money_spread) AS traders_managed_money_spread,
|
||||
ANY_VALUE(ingest_date) AS ingest_date,
|
||||
ANY_VALUE(report_type) AS report_type,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM deduplicated
|
||||
WHERE
|
||||
report_date BETWEEN @start_ds AND @end_ds
|
||||
@@ -0,0 +1,62 @@
|
||||
/* Foundation fact: ICE certified Coffee C (Arabica) aging report. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* Grain: one row per (report_date, age_bucket). */ /* Age buckets represent how long coffee has been in certified storage. */ /* Port columns are in bags (60kg). */
|
||||
MODEL (
|
||||
name foundation.fct_ice_aging_stocks,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column report_date
|
||||
),
|
||||
grain (report_date, age_bucket),
|
||||
start '2020-01-01',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@ice_aging_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE
|
||||
)
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
TRY_CAST(report_date AS DATE) AS report_date,
|
||||
age_bucket,
|
||||
TRY_CAST(antwerp_bags AS BIGINT) AS antwerp_bags,
|
||||
TRY_CAST(hamburg_bremen_bags AS BIGINT) AS hamburg_bremen_bags,
|
||||
TRY_CAST(houston_bags AS BIGINT) AS houston_bags,
|
||||
TRY_CAST(miami_bags AS BIGINT) AS miami_bags,
|
||||
TRY_CAST(new_orleans_bags AS BIGINT) AS new_orleans_bags,
|
||||
TRY_CAST(new_york_bags AS BIGINT) AS new_york_bags,
|
||||
TRY_CAST(total_bags AS BIGINT) AS total_bags,
|
||||
filename AS source_file,
|
||||
HASH(report_date, age_bucket, total_bags) AS hkey
|
||||
FROM src
|
||||
WHERE
|
||||
NOT TRY_CAST(report_date AS DATE) IS NULL
|
||||
AND NOT age_bucket IS NULL
|
||||
AND age_bucket <> ''
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(report_date) AS report_date,
|
||||
ANY_VALUE(age_bucket) AS age_bucket,
|
||||
ANY_VALUE(antwerp_bags) AS antwerp_bags,
|
||||
ANY_VALUE(hamburg_bremen_bags) AS hamburg_bremen_bags,
|
||||
ANY_VALUE(houston_bags) AS houston_bags,
|
||||
ANY_VALUE(miami_bags) AS miami_bags,
|
||||
ANY_VALUE(new_orleans_bags) AS new_orleans_bags,
|
||||
ANY_VALUE(new_york_bags) AS new_york_bags,
|
||||
ANY_VALUE(total_bags) AS total_bags,
|
||||
ANY_VALUE(source_file) AS source_file,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM deduplicated
|
||||
WHERE
|
||||
report_date BETWEEN @start_ds AND @end_ds
|
||||
@@ -0,0 +1,51 @@
|
||||
/* Foundation fact: ICE certified Coffee C (Arabica) warehouse stocks. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* "Certified" means Coffee C graded and stamped as delivery-eligible */ /* against ICE futures contracts — a key physical supply indicator. */ /* Grain: one row per report_date. */
|
||||
MODEL (
|
||||
name foundation.fct_ice_warehouse_stocks,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column report_date
|
||||
),
|
||||
grain (
|
||||
report_date
|
||||
),
|
||||
start '2000-01-01',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@ice_stocks_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE
|
||||
)
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
TRY_CAST(report_date AS DATE) AS report_date,
|
||||
TRY_CAST(total_certified_bags AS BIGINT) AS total_certified_bags,
|
||||
TRY_CAST(pending_grading_bags AS BIGINT) AS pending_grading_bags,
|
||||
filename AS source_file,
|
||||
HASH(report_date, total_certified_bags) AS hkey /* Dedup key: report date + total bags */
|
||||
FROM src
|
||||
WHERE
|
||||
NOT TRY_CAST(report_date AS DATE) IS NULL
|
||||
AND NOT TRY_CAST(total_certified_bags AS BIGINT) IS NULL
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(report_date) AS report_date,
|
||||
ANY_VALUE(total_certified_bags) AS total_certified_bags,
|
||||
ANY_VALUE(pending_grading_bags) AS pending_grading_bags,
|
||||
ANY_VALUE(source_file) AS source_file,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM deduplicated
|
||||
WHERE
|
||||
report_date BETWEEN @start_ds AND @end_ds
|
||||
@@ -0,0 +1,65 @@
|
||||
/* Foundation fact: ICE historical end-of-month Coffee C certified warehouse stocks by port. */ /* Reads directly from the landing zone, casts varchar columns to proper types, */ /* and deduplicates via hash key. */ /* Covers November 1996 to present (30-year history). */ /* Grain: one row per report_date (end-of-month). */ /* Port columns are in bags (60kg). */
|
||||
MODEL (
|
||||
name foundation.fct_ice_warehouse_stocks_by_port,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column report_date
|
||||
),
|
||||
grain (
|
||||
report_date
|
||||
),
|
||||
start '1996-11-01',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
SELECT
|
||||
*
|
||||
FROM READ_CSV(
|
||||
@ice_stocks_by_port_glob(),
|
||||
compression = 'gzip',
|
||||
header = TRUE,
|
||||
union_by_name = TRUE,
|
||||
filename = TRUE,
|
||||
all_varchar = TRUE
|
||||
)
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
TRY_CAST(report_date AS DATE) AS report_date,
|
||||
TRY_CAST(new_york_bags AS BIGINT) AS new_york_bags,
|
||||
TRY_CAST(new_orleans_bags AS BIGINT) AS new_orleans_bags,
|
||||
TRY_CAST(houston_bags AS BIGINT) AS houston_bags,
|
||||
TRY_CAST(miami_bags AS BIGINT) AS miami_bags,
|
||||
TRY_CAST(antwerp_bags AS BIGINT) AS antwerp_bags,
|
||||
TRY_CAST(hamburg_bremen_bags AS BIGINT) AS hamburg_bremen_bags,
|
||||
TRY_CAST(barcelona_bags AS BIGINT) AS barcelona_bags,
|
||||
TRY_CAST(virginia_bags AS BIGINT) AS virginia_bags,
|
||||
TRY_CAST(total_bags AS BIGINT) AS total_bags,
|
||||
filename AS source_file,
|
||||
HASH(report_date, total_bags) AS hkey
|
||||
FROM src
|
||||
WHERE
|
||||
NOT TRY_CAST(report_date AS DATE) IS NULL
|
||||
AND NOT TRY_CAST(total_bags AS BIGINT) IS NULL
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(report_date) AS report_date,
|
||||
ANY_VALUE(new_york_bags) AS new_york_bags,
|
||||
ANY_VALUE(new_orleans_bags) AS new_orleans_bags,
|
||||
ANY_VALUE(houston_bags) AS houston_bags,
|
||||
ANY_VALUE(miami_bags) AS miami_bags,
|
||||
ANY_VALUE(antwerp_bags) AS antwerp_bags,
|
||||
ANY_VALUE(hamburg_bremen_bags) AS hamburg_bremen_bags,
|
||||
ANY_VALUE(barcelona_bags) AS barcelona_bags,
|
||||
ANY_VALUE(virginia_bags) AS virginia_bags,
|
||||
ANY_VALUE(total_bags) AS total_bags,
|
||||
ANY_VALUE(source_file) AS source_file,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM deduplicated
|
||||
WHERE
|
||||
report_date BETWEEN @start_ds AND @end_ds
|
||||
@@ -0,0 +1,125 @@
|
||||
/* Foundation fact: daily weather observations for 12 coffee-growing regions. */
|
||||
/* Source: Open-Meteo (ERA5 reanalysis archive + forecast model for recent days) */
|
||||
/* Landing: LANDING_DIR/weather/{location_id}/{year}/{date}.json.gz */
|
||||
/* One file per (location_id, date). Content: flat Open-Meteo JSON per day. */
|
||||
/* Open-Meteo returns parallel arrays; execute.py splits them into per-day files. */
|
||||
/* Grain: (location_id, observation_date) — one row per location per day. */
|
||||
/* Dedup key: hash(location_id, date) — past weather is immutable. */
|
||||
/* location_id is parsed from filename: split(filename, '/')[-3] */
|
||||
/* Path structure: .../weather/{location_id}/{year}/{date}.json.gz */
|
||||
/* Crop stress flags: */
|
||||
/* is_frost — temp_min_c < 2.0°C (ICO Arabica frost damage threshold) */
|
||||
/* is_heat_stress — temp_max_c > 35.0°C (photosynthesis impairment) */
|
||||
/* is_drought — precipitation_mm < 1.0 (agronomic dry day) */
|
||||
/* is_high_vpd — vpd_max_kpa > 1.5 (significant plant water stress) */
|
||||
/* in_growing_season — simplified month-range flag by variety */
|
||||
MODEL (
|
||||
name foundation.fct_weather_daily,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column observation_date
|
||||
),
|
||||
grain (location_id, observation_date),
|
||||
start '2020-01-01',
|
||||
cron '@daily'
|
||||
);
|
||||
|
||||
WITH src AS (
|
||||
/* Open-Meteo files are flat JSON: all variables at top level (no nested structs). */
|
||||
/* read_json(format='auto') infers column types directly from the numeric values. */
|
||||
SELECT
|
||||
*
|
||||
FROM READ_JSON(@weather_glob(), format = 'auto', compression = 'gzip', filename = TRUE)
|
||||
), located AS (
|
||||
SELECT
|
||||
src.*,
|
||||
STR_SPLIT(filename, '/')[-3] AS location_id, /* location_id is the 3rd-from-last path segment: */
|
||||
/* e.g. .../weather/brazil_minas_gerais/2024/2024-01-15.json.gz → 'brazil_minas_gerais' */
|
||||
TRY_CAST(src."date" AS DATE) AS observation_date
|
||||
FROM src
|
||||
), cast_and_clean AS (
|
||||
SELECT
|
||||
location_id,
|
||||
observation_date,
|
||||
/* Temperature (°C) */
|
||||
TRY_CAST(located.temperature_2m_min AS DOUBLE) AS temp_min_c,
|
||||
TRY_CAST(located.temperature_2m_max AS DOUBLE) AS temp_max_c,
|
||||
TRY_CAST(located.temperature_2m_mean AS DOUBLE) AS temp_mean_c,
|
||||
/* Precipitation (mm total for the day) */
|
||||
COALESCE(TRY_CAST(located.precipitation_sum AS DOUBLE), 0.0) AS precipitation_mm,
|
||||
/* Humidity (% — daily max) */
|
||||
TRY_CAST(located.relative_humidity_2m_max AS DOUBLE) AS humidity_max_pct,
|
||||
/* Cloud cover (% — daily mean) */
|
||||
TRY_CAST(located.cloud_cover_mean AS DOUBLE) AS cloud_cover_mean_pct,
|
||||
/* Wind (m/s max — Open-Meteo requested with wind_speed_unit=ms) */
|
||||
TRY_CAST(located.wind_speed_10m_max AS DOUBLE) AS wind_max_speed_ms,
|
||||
/* ET₀ (mm/day — FAO Penman-Monteith; direct crop water demand signal) */
|
||||
TRY_CAST(located.et0_fao_evapotranspiration AS DOUBLE) AS et0_mm,
|
||||
/* VPD (kPa — max; >1.5 kPa = significant plant water stress) */
|
||||
TRY_CAST(located.vapour_pressure_deficit_max AS DOUBLE) AS vpd_max_kpa,
|
||||
/* Crop stress flags */
|
||||
TRY_CAST(located.temperature_2m_min AS DOUBLE) < 2.0 AS is_frost,
|
||||
TRY_CAST(located.temperature_2m_max AS DOUBLE) > 35.0 AS is_heat_stress,
|
||||
COALESCE(TRY_CAST(located.precipitation_sum AS DOUBLE), 0.0) < 1.0 AS is_drought,
|
||||
TRY_CAST(located.vapour_pressure_deficit_max AS DOUBLE) > 1.5 AS is_high_vpd,
|
||||
HASH(location_id, located."date") AS hkey,
|
||||
filename
|
||||
FROM located
|
||||
WHERE
|
||||
NOT observation_date IS NULL AND NOT location_id IS NULL AND location_id <> ''
|
||||
), deduplicated AS (
|
||||
SELECT
|
||||
ANY_VALUE(location_id) AS location_id,
|
||||
ANY_VALUE(observation_date) AS observation_date,
|
||||
ANY_VALUE(temp_min_c) AS temp_min_c,
|
||||
ANY_VALUE(temp_max_c) AS temp_max_c,
|
||||
ANY_VALUE(temp_mean_c) AS temp_mean_c,
|
||||
ANY_VALUE(precipitation_mm) AS precipitation_mm,
|
||||
ANY_VALUE(humidity_max_pct) AS humidity_max_pct,
|
||||
ANY_VALUE(cloud_cover_mean_pct) AS cloud_cover_mean_pct,
|
||||
ANY_VALUE(wind_max_speed_ms) AS wind_max_speed_ms,
|
||||
ANY_VALUE(et0_mm) AS et0_mm,
|
||||
ANY_VALUE(vpd_max_kpa) AS vpd_max_kpa,
|
||||
ANY_VALUE(is_frost) AS is_frost,
|
||||
ANY_VALUE(is_heat_stress) AS is_heat_stress,
|
||||
ANY_VALUE(is_drought) AS is_drought,
|
||||
ANY_VALUE(is_high_vpd) AS is_high_vpd,
|
||||
hkey
|
||||
FROM cast_and_clean
|
||||
GROUP BY
|
||||
hkey
|
||||
)
|
||||
SELECT
|
||||
d.observation_date,
|
||||
d.location_id,
|
||||
loc.name AS location_name,
|
||||
loc.country,
|
||||
loc.lat,
|
||||
loc.lon,
|
||||
loc.variety,
|
||||
d.temp_min_c,
|
||||
d.temp_max_c,
|
||||
d.temp_mean_c,
|
||||
d.precipitation_mm,
|
||||
d.humidity_max_pct,
|
||||
d.cloud_cover_mean_pct,
|
||||
d.wind_max_speed_ms,
|
||||
d.et0_mm,
|
||||
d.vpd_max_kpa,
|
||||
d.is_frost,
|
||||
d.is_heat_stress,
|
||||
d.is_drought,
|
||||
d.is_high_vpd,
|
||||
CASE loc.variety
|
||||
WHEN 'Arabica'
|
||||
THEN EXTRACT(MONTH FROM d.observation_date) BETWEEN 4 AND 10
|
||||
WHEN 'Robusta'
|
||||
THEN EXTRACT(MONTH FROM d.observation_date) BETWEEN 4 AND 11
|
||||
ELSE FALSE
|
||||
END AS in_growing_season /* Growing season: simplified month-range flag by variety. */
|
||||
/* Arabica: Apr–Oct (covers northern + southern hemisphere risk windows). */
|
||||
/* Robusta: Apr–Nov (Vietnam/Indonesia main cycle). */
|
||||
FROM deduplicated AS d
|
||||
LEFT JOIN seeds.weather_locations AS loc
|
||||
ON d.location_id = loc.location_id
|
||||
WHERE
|
||||
d.observation_date BETWEEN @start_ds AND @end_ds
|
||||
Reference in New Issue
Block a user