67c048485b
Phase 1A — KC=F Coffee Futures Prices: - New extract/coffee_prices/ package (yfinance): downloads KC=F daily OHLCV, stores as gzip CSV with SHA256-based idempotency - SQLMesh models: raw/coffee_prices → foundation/fct_coffee_prices → serving/coffee_prices (with 20d/50d SMA, 52-week high/low, daily return %) - Dashboard: 4 metric cards + dual-line chart (close, 20d MA, 50d MA) - API: GET /commodities/<ticker>/prices Phase 1B — Data Methodology Page: - New /methodology route with full-page template (base.html) - 6 anchored sections: USDA PSD, CFTC COT, KC=F price, ICE warehouse stocks, data quality model, update schedule table - "Methodology" link added to marketing footer Phase 1C — Automated Pipeline: - supervisor.sh updated: runs extract_cot, extract_prices, extract_ice in sequence before transform - Webhook failure alerting via ALERT_WEBHOOK_URL env var (ntfy/Slack/Telegram) ICE Warehouse Stocks: - New extract/ice_stocks/ package (niquests): normalizes ICE Report Center CSV to canonical schema, hash-based idempotency, soft-fail on 404 with guidance - SQLMesh models: raw/ice_warehouse_stocks → foundation/fct_ice_warehouse_stocks → serving/ice_warehouse_stocks (30d avg, WoW change, 52w drawdown) - Dashboard: 4 metric cards + line chart (certified bags + 30d avg) - API: GET /commodities/<code>/stocks Foundation: - dim_commodity: added ticker (KC=F) and ice_stock_report_code (COFFEE-C) columns - macros/__init__.py: added prices_glob() and ice_stocks_glob() - pipelines.py: added extract_prices and extract_ice entries Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
136 lines
2.6 KiB
TOML
136 lines
2.6 KiB
TOML
[project]
|
|
name = "materia"
|
|
version = "0.1.0"
|
|
description = "Add your description here"
|
|
readme = "readme.md"
|
|
authors = [
|
|
{ name = "Hendrik Deeman", email = "hendrik@beanflows.coffee" },
|
|
{ name = "Simon Deeman", email = "simon@beanflows.coffee" }
|
|
]
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"python-dotenv>=1.1.0",
|
|
"typer>=0.15.0",
|
|
"pyyaml>=6.0.2",
|
|
"niquests>=3.15.2",
|
|
"hcloud>=2.8.0",
|
|
"prefect>=3.6.15",
|
|
]
|
|
|
|
[project.scripts]
|
|
materia = "materia.cli:app"
|
|
|
|
|
|
[dependency-groups]
|
|
exploration = [
|
|
"ipykernel>=6.29.5",
|
|
]
|
|
|
|
dev = [
|
|
"pre-commit>=4.1.0",
|
|
"pulumi>=3.202.0",
|
|
"pulumi-cloudflare>=6.10.0",
|
|
"pulumi-hcloud>=1.25.0",
|
|
"pytest>=8.4.2",
|
|
"pytest-cov>=7.0.0",
|
|
"pyyaml>=6.0.2",
|
|
"ruff>=0.9.9",
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
psdonline = {workspace = true }
|
|
sqlmesh_materia = {workspace = true }
|
|
cftc_cot = {workspace = true }
|
|
coffee_prices = {workspace = true }
|
|
ice_stocks = {workspace = true }
|
|
|
|
[tool.uv.workspace]
|
|
members = [
|
|
"extract/*",
|
|
"transform/*",
|
|
"web",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["uv_build>=0.9.2,<0.10.0"]
|
|
build-backend = "uv_build"
|
|
|
|
[tool.ruff]
|
|
exclude = [
|
|
".bzr",
|
|
".direnv",
|
|
".eggs",
|
|
".git",
|
|
".git-rewrite",
|
|
".hg",
|
|
".ipynb_checkpoints",
|
|
".mypy_cache",
|
|
".nox",
|
|
".pants.d",
|
|
".pyenv",
|
|
".pytest_cache",
|
|
".pytype",
|
|
".ruff_cache",
|
|
".svn",
|
|
".tox",
|
|
".venv",
|
|
".vscode",
|
|
"__pypackages__",
|
|
"_build",
|
|
"buck-out",
|
|
"build",
|
|
"dist",
|
|
"node_modules",
|
|
"site-packages",
|
|
"venv",
|
|
"notebooks",
|
|
]
|
|
|
|
indent-width = 4
|
|
|
|
target-version = "py313"
|
|
|
|
[tool.ruff.lint]
|
|
|
|
ignore = [
|
|
"E501", # line too long (handled by formatter)
|
|
"PLR0913", # too many arguments to function call
|
|
"PLR2004", # magic value used in comparison
|
|
]
|
|
|
|
# Allow fix for all enabled rules (when `--fix` is provided)
|
|
fixable = ["ALL"]
|
|
unfixable = []
|
|
|
|
# Allow unused variables when underscore-prefixed
|
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
|
[tool.ruff.format]
|
|
# Use double quotes for strings
|
|
quote-style = "double"
|
|
|
|
# Indent with spaces, not tabs
|
|
indent-style = "space"
|
|
|
|
# Respect magic trailing comma
|
|
skip-magic-trailing-comma = false
|
|
|
|
# Automatically detect line ending
|
|
line-ending = "auto"
|
|
|
|
[tool.ruff.lint.isort]
|
|
# Combine as imports on the same line
|
|
combine-as-imports = true
|
|
|
|
# Use a single line for direct imports
|
|
force-single-line = false
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
# Allow unused imports in __init__.py files
|
|
"__init__.py" = ["F401"]
|
|
# Allow print statements and other rules in scripts
|
|
"scripts/*" = ["T201"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|