ff7301d6a8
- Replace brittle ICE_STOCKS_URL env var with API-based URL discovery via the private ICE Report Center JSON API (no auth required) - Add rolling CSV → XLS fallback in extract_ice_stocks() using find_latest_report() from ice_api.py - Add ice_api.py: fetch_report_listings(), find_latest_report() with pagination up to MAX_API_PAGES - Add xls_parse.py: detect_file_format() (magic bytes), xls_to_rows() using xlrd for OLE2/BIFF XLS files - Add extract_ice_aging(): monthly certified stock aging report by age bucket × port → ice_aging/ landing dir - Add extract_ice_historical(): 30-year EOM by-port stocks from static ICE URL → ice_stocks_by_port/ landing dir - Add xlrd>=2.0.1 (parse XLS), xlwt>=1.3.0 (dev, test fixtures) - Add SQLMesh raw + foundation models for both new datasets - Add ice_aging_glob(), ice_stocks_by_port_glob() macros - Add extract_ice_aging + extract_ice_historical pipeline entries - Add 12 unit tests (format detection, XLS roundtrip, API mock, CSV output) Seed files (data/landing/ice_aging/seed/ and ice_stocks_by_port/seed/) must be created locally — data/ is gitignored. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
137 lines
2.6 KiB
TOML
137 lines
2.6 KiB
TOML
[project]
|
|
name = "materia"
|
|
version = "0.1.0"
|
|
description = "Add your description here"
|
|
readme = "readme.md"
|
|
authors = [
|
|
{ name = "Hendrik Deeman", email = "hendrik@beanflows.coffee" },
|
|
{ name = "Simon Deeman", email = "simon@beanflows.coffee" }
|
|
]
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"python-dotenv>=1.1.0",
|
|
"typer>=0.15.0",
|
|
"pyyaml>=6.0.2",
|
|
"niquests>=3.15.2",
|
|
"hcloud>=2.8.0",
|
|
"prefect>=3.6.15",
|
|
"msgspec>=0.19",
|
|
]
|
|
|
|
[project.scripts]
|
|
materia = "materia.cli:app"
|
|
|
|
|
|
[dependency-groups]
|
|
exploration = [
|
|
"ipykernel>=6.29.5",
|
|
]
|
|
|
|
dev = [
|
|
"pre-commit>=4.1.0",
|
|
"pulumi>=3.202.0",
|
|
"pulumi-cloudflare>=6.10.0",
|
|
"pulumi-hcloud>=1.25.0",
|
|
"pytest>=8.4.2",
|
|
"pytest-cov>=7.0.0",
|
|
"pyyaml>=6.0.2",
|
|
"ruff>=0.9.9",
|
|
"xlwt>=1.3.0",
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
psdonline = {workspace = true }
|
|
sqlmesh_materia = {workspace = true }
|
|
cftc_cot = {workspace = true }
|
|
coffee_prices = {workspace = true }
|
|
ice_stocks = {workspace = true }
|
|
[tool.uv.workspace]
|
|
members = [
|
|
"extract/*",
|
|
"transform/*",
|
|
"web",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["uv_build>=0.9.2,<0.10.0"]
|
|
build-backend = "uv_build"
|
|
|
|
[tool.ruff]
|
|
exclude = [
|
|
".bzr",
|
|
".direnv",
|
|
".eggs",
|
|
".git",
|
|
".git-rewrite",
|
|
".hg",
|
|
".ipynb_checkpoints",
|
|
".mypy_cache",
|
|
".nox",
|
|
".pants.d",
|
|
".pyenv",
|
|
".pytest_cache",
|
|
".pytype",
|
|
".ruff_cache",
|
|
".svn",
|
|
".tox",
|
|
".venv",
|
|
".vscode",
|
|
"__pypackages__",
|
|
"_build",
|
|
"buck-out",
|
|
"build",
|
|
"dist",
|
|
"node_modules",
|
|
"site-packages",
|
|
"venv",
|
|
"notebooks",
|
|
]
|
|
|
|
indent-width = 4
|
|
|
|
target-version = "py313"
|
|
|
|
[tool.ruff.lint]
|
|
|
|
ignore = [
|
|
"E501", # line too long (handled by formatter)
|
|
"PLR0913", # too many arguments to function call
|
|
"PLR2004", # magic value used in comparison
|
|
]
|
|
|
|
# Allow fix for all enabled rules (when `--fix` is provided)
|
|
fixable = ["ALL"]
|
|
unfixable = []
|
|
|
|
# Allow unused variables when underscore-prefixed
|
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
|
[tool.ruff.format]
|
|
# Use double quotes for strings
|
|
quote-style = "double"
|
|
|
|
# Indent with spaces, not tabs
|
|
indent-style = "space"
|
|
|
|
# Respect magic trailing comma
|
|
skip-magic-trailing-comma = false
|
|
|
|
# Automatically detect line ending
|
|
line-ending = "auto"
|
|
|
|
[tool.ruff.lint.isort]
|
|
# Combine as imports on the same line
|
|
combine-as-imports = true
|
|
|
|
# Use a single line for direct imports
|
|
force-single-line = false
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
# Allow unused imports in __init__.py files
|
|
"__init__.py" = ["F401"]
|
|
# Allow print statements and other rules in scripts
|
|
"scripts/*" = ["T201"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|