Add Phase 1A-C + ICE warehouse stocks: prices, methodology, pipeline automation
Phase 1A — KC=F Coffee Futures Prices: - New extract/coffee_prices/ package (yfinance): downloads KC=F daily OHLCV, stores as gzip CSV with SHA256-based idempotency - SQLMesh models: raw/coffee_prices → foundation/fct_coffee_prices → serving/coffee_prices (with 20d/50d SMA, 52-week high/low, daily return %) - Dashboard: 4 metric cards + dual-line chart (close, 20d MA, 50d MA) - API: GET /commodities/<ticker>/prices Phase 1B — Data Methodology Page: - New /methodology route with full-page template (base.html) - 6 anchored sections: USDA PSD, CFTC COT, KC=F price, ICE warehouse stocks, data quality model, update schedule table - "Methodology" link added to marketing footer Phase 1C — Automated Pipeline: - supervisor.sh updated: runs extract_cot, extract_prices, extract_ice in sequence before transform - Webhook failure alerting via ALERT_WEBHOOK_URL env var (ntfy/Slack/Telegram) ICE Warehouse Stocks: - New extract/ice_stocks/ package (niquests): normalizes ICE Report Center CSV to canonical schema, hash-based idempotency, soft-fail on 404 with guidance - SQLMesh models: raw/ice_warehouse_stocks → foundation/fct_ice_warehouse_stocks → serving/ice_warehouse_stocks (30d avg, WoW change, 52w drawdown) - Dashboard: 4 metric cards + line chart (certified bags + 30d avg) - API: GET /commodities/<code>/stocks Foundation: - dim_commodity: added ticker (KC=F) and ice_stock_report_code (COFFEE-C) columns - macros/__init__.py: added prices_glob() and ice_stocks_glob() - pipelines.py: added extract_prices and extract_ice entries Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Deeman
@@ -316,6 +316,79 @@ async def get_cot_index_trend(
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Coffee Prices Queries
|
||||
# =============================================================================
|
||||
|
||||
# KC=F Yahoo Finance ticker
|
||||
COFFEE_TICKER = "KC=F"
|
||||
|
||||
|
||||
async def get_price_time_series(ticker: str, limit: int = 504) -> list[dict]:
|
||||
"""Daily OHLCV + moving averages from serving.coffee_prices. Default ~2 years."""
|
||||
assert 1 <= limit <= 5000, "limit must be between 1 and 5000"
|
||||
return await fetch_analytics(
|
||||
"""
|
||||
SELECT trade_date, open, high, low, close, volume,
|
||||
daily_return_pct, sma_20d, sma_50d, high_52w, low_52w
|
||||
FROM serving.coffee_prices
|
||||
WHERE ticker = ?
|
||||
ORDER BY trade_date DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[ticker, limit],
|
||||
)
|
||||
|
||||
|
||||
async def get_price_latest(ticker: str) -> dict | None:
|
||||
"""Latest trading day's close price, daily return, and 52-week range."""
|
||||
rows = await fetch_analytics(
|
||||
"""
|
||||
SELECT trade_date, close, daily_return_pct, high_52w, low_52w, sma_20d, sma_50d
|
||||
FROM serving.coffee_prices
|
||||
WHERE ticker = ?
|
||||
ORDER BY trade_date DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
[ticker],
|
||||
)
|
||||
return rows[0] if rows else None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# ICE Warehouse Stocks Queries
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def get_ice_stocks_trend(days: int = 365) -> list[dict]:
|
||||
"""Daily ICE certified stocks over the trailing N days."""
|
||||
assert 1 <= days <= 3650, "days must be between 1 and 3650"
|
||||
return await fetch_analytics(
|
||||
"""
|
||||
SELECT report_date, total_certified_bags, pending_grading_bags,
|
||||
wow_change_bags, avg_30d_bags, high_52w_bags, drawdown_from_52w_high_pct
|
||||
FROM serving.ice_warehouse_stocks
|
||||
ORDER BY report_date DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[days],
|
||||
)
|
||||
|
||||
|
||||
async def get_ice_stocks_latest() -> dict | None:
|
||||
"""Latest ICE certified warehouse stock report."""
|
||||
rows = await fetch_analytics(
|
||||
"""
|
||||
SELECT report_date, total_certified_bags, pending_grading_bags,
|
||||
wow_change_bags, avg_30d_bags, drawdown_from_52w_high_pct
|
||||
FROM serving.ice_warehouse_stocks
|
||||
ORDER BY report_date DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
return rows[0] if rows else None
|
||||
|
||||
|
||||
async def get_country_comparison(
|
||||
commodity_code: int,
|
||||
country_codes: list[str],
|
||||
|
||||
@@ -198,6 +198,55 @@ async def commodity_positioning_latest(code: str):
|
||||
return jsonify({"cftc_commodity_code": code, "data": data})
|
||||
|
||||
|
||||
@bp.route("/commodities/<code>/prices")
|
||||
@api_key_required(scopes=["read"])
|
||||
async def commodity_prices(code: str):
|
||||
"""Daily OHLCV price time series for a commodity ticker (e.g. KC=F).
|
||||
|
||||
Query params:
|
||||
start_date — ISO date filter (YYYY-MM-DD)
|
||||
end_date — ISO date filter (YYYY-MM-DD)
|
||||
limit — max rows returned (default 504 ≈ 2 years, max 5000)
|
||||
"""
|
||||
limit = min(int(request.args.get("limit", 504)), 5000)
|
||||
start_date = request.args.get("start_date")
|
||||
end_date = request.args.get("end_date")
|
||||
|
||||
data = await analytics.get_price_time_series(code, limit=limit)
|
||||
|
||||
# Apply date filters in Python — simpler than adding optional params to the query
|
||||
if start_date:
|
||||
data = [r for r in data if str(r["trade_date"]) >= start_date]
|
||||
if end_date:
|
||||
data = [r for r in data if str(r["trade_date"]) <= end_date]
|
||||
|
||||
return jsonify({"ticker": code, "data": data})
|
||||
|
||||
|
||||
@bp.route("/commodities/<code>/stocks")
|
||||
@api_key_required(scopes=["read"])
|
||||
async def commodity_ice_stocks(code: str):
|
||||
"""ICE certified warehouse stock time series.
|
||||
|
||||
Query params:
|
||||
start_date — ISO date filter (YYYY-MM-DD)
|
||||
end_date — ISO date filter (YYYY-MM-DD)
|
||||
days — trailing days (default 365, max 3650)
|
||||
"""
|
||||
days = min(int(request.args.get("days", 365)), 3650)
|
||||
start_date = request.args.get("start_date")
|
||||
end_date = request.args.get("end_date")
|
||||
|
||||
data = await analytics.get_ice_stocks_trend(days=days)
|
||||
|
||||
if start_date:
|
||||
data = [r for r in data if str(r["report_date"]) >= start_date]
|
||||
if end_date:
|
||||
data = [r for r in data if str(r["report_date"]) <= end_date]
|
||||
|
||||
return jsonify({"commodity": code, "data": data})
|
||||
|
||||
|
||||
@bp.route("/commodities/<int:code>/metrics.csv")
|
||||
@api_key_required(scopes=["read"])
|
||||
async def commodity_metrics_csv(code: int):
|
||||
|
||||
@@ -100,7 +100,12 @@ async def index():
|
||||
|
||||
# Fetch all analytics data in parallel (empty lists/None if DB not available)
|
||||
if analytics._conn is not None:
|
||||
time_series, top_producers, stu_trend, balance, yoy, cot_latest, cot_trend = await asyncio.gather(
|
||||
(
|
||||
time_series, top_producers, stu_trend, balance, yoy,
|
||||
cot_latest, cot_trend,
|
||||
price_series, price_latest,
|
||||
ice_stocks_trend, ice_stocks_latest,
|
||||
) = await asyncio.gather(
|
||||
analytics.get_global_time_series(
|
||||
analytics.COFFEE_COMMODITY_CODE,
|
||||
["production", "exports", "imports", "ending_stocks", "total_distribution"],
|
||||
@@ -111,10 +116,16 @@ async def index():
|
||||
analytics.get_production_yoy_by_country(analytics.COFFEE_COMMODITY_CODE, limit=15),
|
||||
analytics.get_cot_positioning_latest(analytics.COFFEE_CFTC_CODE),
|
||||
analytics.get_cot_index_trend(analytics.COFFEE_CFTC_CODE, weeks=104),
|
||||
analytics.get_price_time_series(analytics.COFFEE_TICKER, limit=504),
|
||||
analytics.get_price_latest(analytics.COFFEE_TICKER),
|
||||
analytics.get_ice_stocks_trend(days=365),
|
||||
analytics.get_ice_stocks_latest(),
|
||||
)
|
||||
else:
|
||||
time_series, top_producers, stu_trend, balance, yoy = [], [], [], [], []
|
||||
cot_latest, cot_trend = None, []
|
||||
price_series, price_latest = [], None
|
||||
ice_stocks_trend, ice_stocks_latest = [], None
|
||||
|
||||
# Latest global snapshot for key metric cards
|
||||
latest = time_series[-1] if time_series else {}
|
||||
@@ -140,6 +151,10 @@ async def index():
|
||||
yoy=yoy,
|
||||
cot_latest=cot_latest,
|
||||
cot_trend=cot_trend,
|
||||
price_series=price_series,
|
||||
price_latest=price_latest,
|
||||
ice_stocks_trend=ice_stocks_trend,
|
||||
ice_stocks_latest=ice_stocks_latest,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -148,6 +148,80 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Coffee Futures Price (KC=F) -->
|
||||
{% if price_latest %}
|
||||
<div class="chart-container mb-8">
|
||||
<h2 class="text-xl mb-1">Coffee C Futures Price — KC=F</h2>
|
||||
<p class="text-muted mb-4">ICE Coffee C Arabica · Daily close price · Source: Yahoo Finance</p>
|
||||
<div class="grid-4 mb-4">
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">Latest Close</div>
|
||||
<div class="metric-value">{{ "{:.2f}".format(price_latest.close) }}</div>
|
||||
<div class="metric-sub">¢/lb · as of {{ price_latest.trade_date }}</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">Daily Change</div>
|
||||
<div class="metric-value {% if price_latest.daily_return_pct and price_latest.daily_return_pct > 0 %}text-green{% elif price_latest.daily_return_pct and price_latest.daily_return_pct < 0 %}text-red{% endif %}">
|
||||
{% if price_latest.daily_return_pct is not none %}
|
||||
{{ "{:+.2f}%".format(price_latest.daily_return_pct) }}
|
||||
{% else %}--{% endif %}
|
||||
</div>
|
||||
<div class="metric-sub">vs previous close</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">52-Week High</div>
|
||||
<div class="metric-value">{{ "{:.2f}".format(price_latest.high_52w) }}</div>
|
||||
<div class="metric-sub">¢/lb</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">52-Week Low</div>
|
||||
<div class="metric-value">{{ "{:.2f}".format(price_latest.low_52w) }}</div>
|
||||
<div class="metric-sub">¢/lb</div>
|
||||
</div>
|
||||
</div>
|
||||
<canvas id="priceChart"></canvas>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- ICE Certified Warehouse Stocks -->
|
||||
{% if ice_stocks_latest %}
|
||||
<div class="chart-container mb-8">
|
||||
<h2 class="text-xl mb-1">ICE Certified Warehouse Stocks</h2>
|
||||
<p class="text-muted mb-4">Physical Arabica certified for delivery against ICE Coffee C futures · as of {{ ice_stocks_latest.report_date }}</p>
|
||||
<div class="grid-4 mb-4">
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">Certified Stocks</div>
|
||||
<div class="metric-value">{{ "{:,.0f}".format(ice_stocks_latest.total_certified_bags) }}</div>
|
||||
<div class="metric-sub">60-kg bags</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">Week-over-Week</div>
|
||||
<div class="metric-value {% if ice_stocks_latest.wow_change_bags and ice_stocks_latest.wow_change_bags > 0 %}text-green{% elif ice_stocks_latest.wow_change_bags and ice_stocks_latest.wow_change_bags < 0 %}text-red{% endif %}">
|
||||
{% if ice_stocks_latest.wow_change_bags is not none %}
|
||||
{{ "{:+,d}".format(ice_stocks_latest.wow_change_bags | int) }}
|
||||
{% else %}--{% endif %}
|
||||
</div>
|
||||
<div class="metric-sub">bags vs previous day</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">30-Day Average</div>
|
||||
<div class="metric-value">{{ "{:,.0f}".format(ice_stocks_latest.avg_30d_bags) }}</div>
|
||||
<div class="metric-sub">60-kg bags</div>
|
||||
</div>
|
||||
<div class="metric-card">
|
||||
<div class="metric-label">Drawdown from 52w High</div>
|
||||
<div class="metric-value {% if ice_stocks_latest.drawdown_from_52w_high_pct and ice_stocks_latest.drawdown_from_52w_high_pct < -10 %}text-red{% endif %}">
|
||||
{% if ice_stocks_latest.drawdown_from_52w_high_pct is not none %}
|
||||
{{ "{:.1f}%".format(ice_stocks_latest.drawdown_from_52w_high_pct) }}
|
||||
{% else %}--{% endif %}
|
||||
</div>
|
||||
<div class="metric-sub">below 52-week peak</div>
|
||||
</div>
|
||||
</div>
|
||||
<canvas id="iceStocksChart"></canvas>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Quick Actions -->
|
||||
<div class="grid-3">
|
||||
<a href="{{ url_for('dashboard.countries') }}" class="btn-outline text-center">Country Comparison</a>
|
||||
@@ -286,6 +360,97 @@ if (cotRaw && cotRaw.length > 0) {
|
||||
});
|
||||
}
|
||||
|
||||
// -- Coffee Prices Chart (close + 20d MA + 50d MA) --
|
||||
const priceRaw = {{ price_series | tojson }};
|
||||
if (priceRaw && priceRaw.length > 0) {
|
||||
const priceData = [...priceRaw].reverse(); // query returns DESC, chart needs ASC
|
||||
new Chart(document.getElementById('priceChart'), {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: priceData.map(r => r.trade_date),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Close (¢/lb)',
|
||||
data: priceData.map(r => r.close),
|
||||
borderColor: CHART_COLORS.copper,
|
||||
backgroundColor: CHART_COLORS.copper + '18',
|
||||
fill: true,
|
||||
tension: 0.2,
|
||||
pointRadius: 0,
|
||||
yAxisID: 'y'
|
||||
},
|
||||
{
|
||||
label: '20d MA',
|
||||
data: priceData.map(r => r.sma_20d),
|
||||
borderColor: CHART_COLORS.beanGreen,
|
||||
borderDash: [4, 3],
|
||||
tension: 0.2,
|
||||
pointRadius: 0,
|
||||
yAxisID: 'y'
|
||||
},
|
||||
{
|
||||
label: '50d MA',
|
||||
data: priceData.map(r => r.sma_50d),
|
||||
borderColor: CHART_COLORS.roast,
|
||||
borderDash: [8, 4],
|
||||
tension: 0.2,
|
||||
pointRadius: 0,
|
||||
yAxisID: 'y'
|
||||
}
|
||||
]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
interaction: {mode: 'index', intersect: false},
|
||||
plugins: {legend: {position: 'bottom'}},
|
||||
scales: {
|
||||
x: {ticks: {maxTicksLimit: 12}},
|
||||
y: {title: {display: true, text: '¢/lb'}, beginAtZero: false}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -- ICE Warehouse Stocks Chart --
|
||||
const iceRaw = {{ ice_stocks_trend | tojson }};
|
||||
if (iceRaw && iceRaw.length > 0) {
|
||||
const iceData = [...iceRaw].reverse(); // query returns DESC, chart needs ASC
|
||||
new Chart(document.getElementById('iceStocksChart'), {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: iceData.map(r => r.report_date),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Certified Stocks (bags)',
|
||||
data: iceData.map(r => r.total_certified_bags),
|
||||
borderColor: CHART_COLORS.roast,
|
||||
backgroundColor: CHART_COLORS.roast + '18',
|
||||
fill: true,
|
||||
tension: 0.2,
|
||||
pointRadius: 0
|
||||
},
|
||||
{
|
||||
label: '30d Average',
|
||||
data: iceData.map(r => r.avg_30d_bags),
|
||||
borderColor: CHART_COLORS.stone,
|
||||
borderDash: [5, 4],
|
||||
tension: 0.2,
|
||||
pointRadius: 0
|
||||
}
|
||||
]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
interaction: {mode: 'index', intersect: false},
|
||||
plugins: {legend: {position: 'bottom'}},
|
||||
scales: {
|
||||
x: {ticks: {maxTicksLimit: 12}},
|
||||
y: {title: {display: true, text: '60-kg bags'}, beginAtZero: false}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -- Top Producers Horizontal Bar --
|
||||
const topData = {{ top_producers | tojson }};
|
||||
if (topData.length > 0) {
|
||||
|
||||
@@ -46,6 +46,12 @@ async def about():
|
||||
return await render_template("about.html")
|
||||
|
||||
|
||||
@bp.route("/methodology")
|
||||
async def methodology():
|
||||
"""Data methodology page — explains all data sources."""
|
||||
return await render_template("methodology.html")
|
||||
|
||||
|
||||
@bp.route("/feedback", methods=["POST"])
|
||||
@csrf_protect
|
||||
async def feedback():
|
||||
|
||||
230
web/src/beanflows/public/templates/methodology.html
Normal file
230
web/src/beanflows/public/templates/methodology.html
Normal file
@@ -0,0 +1,230 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Data Methodology — {{ config.APP_NAME }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<main>
|
||||
<!-- Hero -->
|
||||
<section class="hero">
|
||||
<div class="container-page">
|
||||
<h1 class="heading-display">Data Methodology</h1>
|
||||
<p>Every number on BeanFlows has a source, a frequency, and a known limitation. Here's exactly where the data comes from and how we process it.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Table of Contents -->
|
||||
<section class="container-page py-8 max-w-3xl mx-auto">
|
||||
<nav class="bg-latte rounded-lg p-6 mb-12">
|
||||
<h2 class="text-sm font-semibold text-espresso uppercase tracking-wide mb-3">On this page</h2>
|
||||
<ul class="list-none p-0 space-y-1.5 text-sm">
|
||||
<li><a href="#usda-psd" class="text-copper">USDA Production, Supply & Distribution</a></li>
|
||||
<li><a href="#cftc-cot" class="text-copper">CFTC Commitments of Traders</a></li>
|
||||
<li><a href="#kc-price" class="text-copper">Coffee Futures Price (KC=F)</a></li>
|
||||
<li><a href="#ice-stocks" class="text-copper">ICE Certified Warehouse Stocks</a></li>
|
||||
<li><a href="#data-quality" class="text-copper">Data Quality</a></li>
|
||||
<li><a href="#update-schedule" class="text-copper">Update Schedule</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<!-- USDA PSD -->
|
||||
<section id="usda-psd" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">USDA Production, Supply & Distribution</h2>
|
||||
<p class="text-stone mb-4">The USDA's <strong>Production, Supply and Distribution (PSD) Online</strong> database is the definitive public source for agricultural commodity supply and demand balances. It is maintained by the USDA Foreign Agricultural Service and covers 160+ countries and 50+ commodities going back to the 1960s for some crops.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">What we use</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>Commodity:</strong> Coffee, Green — USDA commodity code <code class="bg-parchment px-1 rounded">0711100</code></li>
|
||||
<li><strong>Coverage:</strong> 2006–present (monthly updates)</li>
|
||||
<li><strong>Geography:</strong> Country-level + world aggregate</li>
|
||||
<li><strong>Source URL:</strong> <code class="bg-parchment px-1 rounded">apps.fas.usda.gov/psdonlineapi</code></li>
|
||||
</ul>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Metrics</h3>
|
||||
<div class="overflow-x-auto mb-4">
|
||||
<table class="table text-sm">
|
||||
<thead>
|
||||
<tr><th>Metric</th><th>Definition</th><th>Unit</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Production</td><td>Harvested green coffee output</td><td>1,000 × 60-kg bags</td></tr>
|
||||
<tr><td>Imports</td><td>Physical coffee imported into country</td><td>1,000 × 60-kg bags</td></tr>
|
||||
<tr><td>Exports</td><td>Physical coffee exported from country</td><td>1,000 × 60-kg bags</td></tr>
|
||||
<tr><td>Domestic Consumption</td><td>Coffee consumed within country</td><td>1,000 × 60-kg bags</td></tr>
|
||||
<tr><td>Ending Stocks</td><td>Carry-over stocks at marketing year end</td><td>1,000 × 60-kg bags</td></tr>
|
||||
<tr><td>Stock-to-Use Ratio</td><td>Ending stocks ÷ consumption × 100</td><td>%</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Release schedule</h3>
|
||||
<p class="text-stone mb-4">USDA publishes PSD updates monthly, typically in the second week of the month as part of the <em>World Agricultural Supply and Demand Estimates (WASDE)</em> report. Our pipeline checks for updates daily and downloads new data when the file hash changes.</p>
|
||||
|
||||
<div class="bg-parchment rounded p-4 text-sm text-stone">
|
||||
<strong>Note on marketing years:</strong> Coffee marketing years vary by origin country. Brazil's marketing year runs April–March; Colombia's runs October–September. USDA normalizes all data to a common market year basis for the global aggregate.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- CFTC COT -->
|
||||
<section id="cftc-cot" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">CFTC Commitments of Traders</h2>
|
||||
<p class="text-stone mb-4">The <strong>Commitments of Traders (COT)</strong> report is published weekly by the U.S. Commodity Futures Trading Commission (CFTC). It shows the net positions of large traders in regulated futures markets. It is the primary public indicator of speculative positioning in agricultural commodities.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">What we use</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>Report type:</strong> Disaggregated Futures-Only</li>
|
||||
<li><strong>Commodity:</strong> Coffee C — CFTC code <code class="bg-parchment px-1 rounded">083</code></li>
|
||||
<li><strong>Snapshot date:</strong> Every Tuesday close-of-business</li>
|
||||
<li><strong>Release date:</strong> The following Friday at 3:30 PM ET</li>
|
||||
<li><strong>Coverage:</strong> June 2006–present</li>
|
||||
<li><strong>Source:</strong> <code class="bg-parchment px-1 rounded">cftc.gov/files/dea/history/fut_disagg_txt_{year}.zip</code></li>
|
||||
</ul>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Trader categories</h3>
|
||||
<div class="overflow-x-auto mb-4">
|
||||
<table class="table text-sm">
|
||||
<thead>
|
||||
<tr><th>Category</th><th>Who they are</th><th>What to watch</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>Managed Money</td><td>Hedge funds, CTAs, algorithmic traders</td><td>Primary speculative signal — net long = bullish</td></tr>
|
||||
<tr><td>Producer / Merchant</td><td>Coffee exporters, processors, roasters</td><td>Commercial hedgers — usually net short</td></tr>
|
||||
<tr><td>Swap Dealers</td><td>Banks providing OTC commodity exposure</td><td>Index fund replication — less directional signal</td></tr>
|
||||
<tr><td>Other Reportables</td><td>Large traders not fitting other categories</td><td>Mixed motivations</td></tr>
|
||||
<tr><td>Non-Reportable</td><td>Small speculators below CFTC threshold</td><td>Retail sentiment proxy</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">COT Index</h3>
|
||||
<p class="text-stone mb-4">The <strong>COT Index</strong> normalizes the managed money net position to a 0–100 scale over a trailing window (we publish both 26-week and 52-week). It is calculated as:</p>
|
||||
<div class="bg-parchment rounded p-4 text-sm font-mono mb-4">
|
||||
COT Index = (current net − min over window) ÷ (max over window − min over window) × 100
|
||||
</div>
|
||||
<p class="text-stone mb-4">A reading near 0 indicates managed money is at its most bearish extreme over the window. A reading near 100 indicates maximum bullish positioning. Think of it as an RSI for speculative positioning.</p>
|
||||
</section>
|
||||
|
||||
<!-- KC=F Price -->
|
||||
<section id="kc-price" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">Coffee Futures Price (KC=F)</h2>
|
||||
<p class="text-stone mb-4">The <strong>Coffee C contract</strong> (ticker: KC=F) is the global benchmark price for Arabica coffee, traded on ICE Futures U.S. (formerly New York Board of Trade). Each contract covers 37,500 lbs of green coffee. Price is quoted in US cents per pound (¢/lb).</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">What we use</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>Ticker:</strong> KC=F (front-month continuous contract)</li>
|
||||
<li><strong>Data:</strong> Daily OHLCV (Open, High, Low, Close, Adjusted Close, Volume)</li>
|
||||
<li><strong>Source:</strong> Yahoo Finance via yfinance</li>
|
||||
<li><strong>Coverage:</strong> 1971–present</li>
|
||||
<li><strong>Delay:</strong> ~15-minute delayed (Yahoo Finance standard)</li>
|
||||
</ul>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Derived metrics</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>Daily Return %:</strong> (close − prev close) ÷ prev close × 100</li>
|
||||
<li><strong>20-day SMA:</strong> Simple moving average of the last 20 trading days</li>
|
||||
<li><strong>50-day SMA:</strong> Simple moving average of the last 50 trading days</li>
|
||||
<li><strong>52-week High/Low:</strong> Rolling high/low over the trailing ~252 trading days</li>
|
||||
</ul>
|
||||
|
||||
<div class="bg-parchment rounded p-4 text-sm text-stone">
|
||||
<strong>Front-month continuity:</strong> KC=F is the continuous front-month contract. At roll dates, there is a price gap between expiring and next-month contracts. Adjusted Close accounts for roll adjustments. We use raw Close for current price display and Adjusted Close for historical return calculations.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- ICE Warehouse Stocks -->
|
||||
<section id="ice-stocks" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">ICE Certified Warehouse Stocks</h2>
|
||||
<p class="text-stone mb-4">ICE Futures U.S. publishes daily reports of <strong>certified warehouse stocks</strong> for Coffee C. These are physical bags of Arabica coffee that have been graded and stamped as meeting ICE delivery specifications — making them eligible for delivery against a futures contract at expiration.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Why certified stocks matter</h3>
|
||||
<p class="text-stone mb-4">Certified stocks are the physical backing of the futures market. When certified stocks fall sharply while open interest is high, shorts cannot easily deliver physical coffee — this creates a <strong>squeeze dynamic</strong> that can drive explosive price rallies. Tracking certified stocks alongside positioning data is essential for understanding delivery risk.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">What we track</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>Total Certified Bags:</strong> All ICE-approved warehouse receipts (60-kg bags)</li>
|
||||
<li><strong>Pending Grading:</strong> Coffee being evaluated for certification (may join or exit certified stock)</li>
|
||||
<li><strong>Source:</strong> ICE Report Center (daily publication)</li>
|
||||
<li><strong>Update frequency:</strong> Daily, after market close</li>
|
||||
</ul>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Derived metrics</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li><strong>WoW Change:</strong> Day-over-day change in certified bags</li>
|
||||
<li><strong>30-Day Average:</strong> Smoothed trend removing daily noise</li>
|
||||
<li><strong>52-Week High:</strong> Rolling maximum over trailing 365 days</li>
|
||||
<li><strong>Drawdown from 52w High:</strong> % decline from peak — measures how far stocks have been drawn down</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<!-- Data Quality -->
|
||||
<section id="data-quality" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">Data Quality</h2>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Immutable raw layer</h3>
|
||||
<p class="text-stone mb-4">All source files are stored as immutable gzip-compressed CSVs in a content-addressed landing directory. Files are never modified in place — a new download creates a new file only if the content hash differs from what is already stored. This means the full history of source corrections is preserved.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Incremental models with deduplication</h3>
|
||||
<p class="text-stone mb-4">Foundation models are incremental and deduplicate via a hash key computed from business grain columns and key metrics. If a source issues a correction (CFTC re-states a COT figure, USDA revises a production estimate), the corrected row produces a different hash and is ingested on the next pipeline run. Serving models select the most recent revision per grain.</p>
|
||||
|
||||
<h3 class="text-lg font-semibold mb-2 mt-6">Known limitations</h3>
|
||||
<ul class="list-disc list-inside text-stone space-y-1.5 mb-4">
|
||||
<li>USDA PSD revisions can extend back multiple years — always treat historical figures as estimates subject to revision.</li>
|
||||
<li>Yahoo Finance prices carry a ~15-minute delay and may have minor adjustments at roll dates.</li>
|
||||
<li>COT data reflects Tuesday close positions; the market may move significantly before Friday's release.</li>
|
||||
<li>ICE warehouse stocks do not distinguish between origins — certified stock drawdowns at specific ports are not visible here.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<!-- Update Schedule -->
|
||||
<section id="update-schedule" class="mb-12">
|
||||
<h2 class="text-2xl mb-4">Update Schedule</h2>
|
||||
<div class="overflow-x-auto">
|
||||
<table class="table text-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Source</th>
|
||||
<th>Frequency</th>
|
||||
<th>Typical freshness</th>
|
||||
<th>Notes</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>USDA PSD</td>
|
||||
<td>Monthly</td>
|
||||
<td>~2nd week of month</td>
|
||||
<td>WASDE release day; daily pipeline detects hash change</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>CFTC COT</td>
|
||||
<td>Weekly (Friday)</td>
|
||||
<td>Friday 3:30 PM ET</td>
|
||||
<td>Reflects prior Tuesday positions</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>KC=F Price</td>
|
||||
<td>Daily</td>
|
||||
<td>Next morning</td>
|
||||
<td>Yahoo Finance ~15 min delayed; previous day close available next morning</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ICE Warehouse Stocks</td>
|
||||
<td>Daily</td>
|
||||
<td>After market close</td>
|
||||
<td>ICE publishes report center data daily after the close</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<p class="text-stone mt-4 text-sm">Our pipeline runs continuously. Data is re-checked daily and new data is loaded within hours of publication. The dashboard shows the freshness date on each data section.</p>
|
||||
</section>
|
||||
|
||||
<!-- Questions -->
|
||||
<section class="bg-latte rounded-lg p-6">
|
||||
<h2 class="text-xl mb-2">Questions about the data?</h2>
|
||||
<p class="text-stone text-sm mb-4">If you spot an inconsistency or want to understand how a specific metric is calculated, use the feedback button on any page or reach out directly.</p>
|
||||
<a href="{{ url_for('auth.signup') }}" class="btn">Try BeanFlows free</a>
|
||||
</section>
|
||||
</section>
|
||||
</main>
|
||||
{% endblock %}
|
||||
@@ -77,6 +77,7 @@
|
||||
<li><a href="{{ url_for('public.features') }}">Features</a></li>
|
||||
<li><a href="{{ url_for('billing.pricing') }}">Pricing</a></li>
|
||||
<li><a href="{{ url_for('public.about') }}">About</a></li>
|
||||
<li><a href="{{ url_for('public.methodology') }}">Methodology</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<div>
|
||||
|
||||
Reference in New Issue
Block a user