c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
132 lines
2.5 KiB
TOML
132 lines
2.5 KiB
TOML
[project]
|
|
name = "materia"
|
|
version = "0.1.0"
|
|
description = "Add your description here"
|
|
readme = "readme.md"
|
|
authors = [
|
|
{ name = "Hendrik Deeman", email = "hendrik@beanflows.coffee" },
|
|
{ name = "Simon Deeman", email = "simon@beanflows.coffee" }
|
|
]
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"python-dotenv>=1.1.0",
|
|
"typer>=0.15.0",
|
|
"pyyaml>=6.0.2",
|
|
"niquests>=3.15.2",
|
|
"hcloud>=2.8.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
materia = "materia.cli:app"
|
|
|
|
|
|
[dependency-groups]
|
|
exploration = [
|
|
"ipykernel>=6.29.5",
|
|
]
|
|
|
|
dev = [
|
|
"pre-commit>=4.1.0",
|
|
"pulumi>=3.202.0",
|
|
"pulumi-cloudflare>=6.10.0",
|
|
"pulumi-hcloud>=1.25.0",
|
|
"pytest>=8.4.2",
|
|
"pytest-cov>=7.0.0",
|
|
"pyyaml>=6.0.2",
|
|
"ruff>=0.9.9",
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
psdonline = {workspace = true }
|
|
sqlmesh_materia = {workspace = true }
|
|
|
|
[tool.uv.workspace]
|
|
members = [
|
|
"extract/*",
|
|
"transform/*",
|
|
"web",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["uv_build>=0.9.2,<0.10.0"]
|
|
build-backend = "uv_build"
|
|
|
|
[tool.ruff]
|
|
exclude = [
|
|
".bzr",
|
|
".direnv",
|
|
".eggs",
|
|
".git",
|
|
".git-rewrite",
|
|
".hg",
|
|
".ipynb_checkpoints",
|
|
".mypy_cache",
|
|
".nox",
|
|
".pants.d",
|
|
".pyenv",
|
|
".pytest_cache",
|
|
".pytype",
|
|
".ruff_cache",
|
|
".svn",
|
|
".tox",
|
|
".venv",
|
|
".vscode",
|
|
"__pypackages__",
|
|
"_build",
|
|
"buck-out",
|
|
"build",
|
|
"dist",
|
|
"node_modules",
|
|
"site-packages",
|
|
"venv",
|
|
"notebooks",
|
|
]
|
|
|
|
indent-width = 4
|
|
|
|
target-version = "py313"
|
|
|
|
[tool.ruff.lint]
|
|
|
|
ignore = [
|
|
"E501", # line too long (handled by formatter)
|
|
"PLR0913", # too many arguments to function call
|
|
"PLR2004", # magic value used in comparison
|
|
]
|
|
|
|
# Allow fix for all enabled rules (when `--fix` is provided)
|
|
fixable = ["ALL"]
|
|
unfixable = []
|
|
|
|
# Allow unused variables when underscore-prefixed
|
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
|
[tool.ruff.format]
|
|
# Use double quotes for strings
|
|
quote-style = "double"
|
|
|
|
# Indent with spaces, not tabs
|
|
indent-style = "space"
|
|
|
|
# Respect magic trailing comma
|
|
skip-magic-trailing-comma = false
|
|
|
|
# Automatically detect line ending
|
|
line-ending = "auto"
|
|
|
|
[tool.ruff.lint.isort]
|
|
# Combine as imports on the same line
|
|
combine-as-imports = true
|
|
|
|
# Use a single line for direct imports
|
|
force-single-line = false
|
|
|
|
[tool.ruff.lint.per-file-ignores]
|
|
# Allow unused imports in __init__.py files
|
|
"__init__.py" = ["F401"]
|
|
# Allow print statements and other rules in scripts
|
|
"scripts/*" = ["T201"]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|