feat(extract): add OpenWeatherMap daily weather extractor
Adds extract/openweathermap package with daily weather extraction for 8
coffee-growing regions (Brazil, Vietnam, Colombia, Ethiopia, Honduras,
Guatemala, Indonesia). Feeds crop stress signal for commodity sentiment score.
Extractor:
- OWM One Call API 3.0 / Day Summary — one JSON.gz per (location, date)
- extract_weather: daily, fetches yesterday + today (16 calls max)
- extract_weather_backfill: fills 2020-01-01 to yesterday, capped at 500
calls/run with resume cursor '{location_id}:{date}' for crash safety
- Full idempotency via file existence check; state tracking via extract_core
SQLMesh:
- seeds.weather_locations (8 regions with lat/lon/variety)
- foundation.fct_weather_daily: INCREMENTAL_BY_TIME_RANGE, grain
(location_id, observation_date), dedup via hash key, crop stress flags:
is_frost (<2°C), is_heat_stress (>35°C), is_drought (<1mm), in_growing_season
Landing path: LANDING_DIR/weather/{location_id}/{year}/{date}.json.gz
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,19 +1,12 @@
|
||||
-- Serving mart: ICE certified Coffee C warehouse stocks, analytics-ready.
|
||||
--
|
||||
-- Adds 30-day rolling average, week-over-week change, and drawdown from
|
||||
-- 52-week high. Physical supply indicator used alongside S/D and positioning.
|
||||
--
|
||||
-- "Certified stocks" = coffee graded and stamped as eligible for delivery
|
||||
-- against ICE Coffee C futures — traders watch this as a squeeze indicator.
|
||||
--
|
||||
-- Grain: one row per report_date.
|
||||
|
||||
/* Serving mart: ICE certified Coffee C warehouse stocks, analytics-ready. */ /* Adds 30-day rolling average, week-over-week change, and drawdown from */ /* 52-week high. Physical supply indicator used alongside S/D and positioning. */ /* "Certified stocks" = coffee graded and stamped as eligible for delivery */ /* against ICE Coffee C futures — traders watch this as a squeeze indicator. */ /* Grain: one row per report_date. */
|
||||
MODEL (
|
||||
name serving.ice_warehouse_stocks,
|
||||
kind INCREMENTAL_BY_TIME_RANGE (
|
||||
time_column report_date
|
||||
),
|
||||
grain (report_date),
|
||||
grain (
|
||||
report_date
|
||||
),
|
||||
start '2000-01-01',
|
||||
cron '@daily'
|
||||
);
|
||||
@@ -23,45 +16,25 @@ WITH base AS (
|
||||
f.report_date,
|
||||
f.total_certified_bags,
|
||||
f.pending_grading_bags,
|
||||
|
||||
-- Week-over-week change (compare to 7 calendar days ago via LAG over ordered rows)
|
||||
-- Using LAG(1) since data is daily: compares to previous trading/reporting day
|
||||
f.total_certified_bags
|
||||
- LAG(f.total_certified_bags, 1) OVER (ORDER BY f.report_date) AS wow_change_bags,
|
||||
|
||||
-- 30-day rolling average (smooths daily noise)
|
||||
round(
|
||||
AVG(f.total_certified_bags::double) OVER (
|
||||
ORDER BY f.report_date ROWS BETWEEN 29 PRECEDING AND CURRENT ROW
|
||||
),
|
||||
f.total_certified_bags /* Week-over-week change (compare to 7 calendar days ago via LAG over ordered rows) */ /* Using LAG(1) since data is daily: compares to previous trading/reporting day */ - LAG(f.total_certified_bags, 1) OVER (ORDER BY f.report_date) AS wow_change_bags,
|
||||
ROUND(
|
||||
AVG(f.total_certified_bags::DOUBLE) OVER (ORDER BY f.report_date ROWS BETWEEN 29 PRECEDING AND CURRENT ROW),
|
||||
0
|
||||
) AS avg_30d_bags,
|
||||
|
||||
-- 52-week high (365 calendar days ≈ 252 trading days; use 365-row window as proxy)
|
||||
MAX(f.total_certified_bags) OVER (
|
||||
ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW
|
||||
) AS high_52w_bags,
|
||||
|
||||
-- Drawdown from 52-week high (pct below peak — squeeze indicator)
|
||||
round(
|
||||
(f.total_certified_bags::double
|
||||
- MAX(f.total_certified_bags) OVER (
|
||||
ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW
|
||||
)::double
|
||||
)
|
||||
/ NULLIF(
|
||||
MAX(f.total_certified_bags) OVER (
|
||||
ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW
|
||||
)::double,
|
||||
0
|
||||
) * 100,
|
||||
) AS avg_30d_bags, /* 30-day rolling average (smooths daily noise) */
|
||||
MAX(f.total_certified_bags) OVER (ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW) AS high_52w_bags, /* 52-week high (365 calendar days ≈ 252 trading days; use 365-row window as proxy) */
|
||||
ROUND(
|
||||
(
|
||||
f.total_certified_bags::DOUBLE - MAX(f.total_certified_bags) OVER (ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW)::DOUBLE
|
||||
) / NULLIF(
|
||||
MAX(f.total_certified_bags) OVER (ORDER BY f.report_date ROWS BETWEEN 364 PRECEDING AND CURRENT ROW)::DOUBLE,
|
||||
0
|
||||
) * 100,
|
||||
2
|
||||
) AS drawdown_from_52w_high_pct
|
||||
|
||||
FROM foundation.fct_ice_warehouse_stocks f
|
||||
WHERE f.report_date BETWEEN @start_ds AND @end_ds
|
||||
) AS drawdown_from_52w_high_pct /* Drawdown from 52-week high (pct below peak — squeeze indicator) */
|
||||
FROM foundation.fct_ice_warehouse_stocks AS f
|
||||
WHERE
|
||||
f.report_date BETWEEN @start_ds AND @end_ds
|
||||
)
|
||||
|
||||
SELECT
|
||||
b.report_date,
|
||||
d.commodity_name,
|
||||
@@ -72,7 +45,9 @@ SELECT
|
||||
b.avg_30d_bags,
|
||||
b.high_52w_bags,
|
||||
b.drawdown_from_52w_high_pct
|
||||
FROM base b
|
||||
CROSS JOIN foundation.dim_commodity d
|
||||
WHERE d.ice_stock_report_code = 'COFFEE-C'
|
||||
ORDER BY b.report_date
|
||||
FROM base AS b
|
||||
CROSS JOIN foundation.dim_commodity AS d
|
||||
WHERE
|
||||
d.ice_stock_report_code = 'COFFEE-C'
|
||||
ORDER BY
|
||||
b.report_date
|
||||
Reference in New Issue
Block a user