c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
22 lines
441 B
TOML
22 lines
441 B
TOML
[project]
|
|
name = "psdonline"
|
|
version = "0.1.0"
|
|
description = "Add your description here"
|
|
authors = [
|
|
{ name = "Deeman", email = "hendriknote@gmail.com" }
|
|
]
|
|
requires-python = ">=3.13"
|
|
|
|
dependencies = [
|
|
"niquests>=3.14.1",
|
|
]
|
|
[project.scripts]
|
|
extract_psd = "psdonline.execute:extract_psd_dataset"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/psdonline"]
|