From d14990bb01ad74a4d92925a06ef4e83868bfa9cf Mon Sep 17 00:00:00 2001 From: Deeman Date: Sat, 28 Feb 2026 23:00:52 +0100 Subject: [PATCH] =?UTF-8?q?refactor:=20rename=20materia=20=E2=86=92=20bean?= =?UTF-8?q?flows=20throughout=20codebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename src/materia/ → src/beanflows/ (Python package) - Rename transform/sqlmesh_materia/ → transform/sqlmesh_beanflows/ - Rename infra/supervisor/materia-supervisor.service → beanflows-supervisor.service - Rename infra/backup/materia-backup.{service,timer} → beanflows-backup.{service,timer} - Update all path strings: /opt/materia → /opt/beanflows, /data/materia → /data/beanflows - Update pyproject.toml: project name, CLI entrypoint, workspace source key - Update all internal imports from materia.* → beanflows.* - Update infra scripts: REPO_DIR, service names, systemctl references - Fix docker-compose.prod.yml: /data/materia → /data/beanflows (bind mount path) Intentionally left unchanged: Pulumi stack name (materia-infrastructure) and Hetzner resource names ("materia-key", "managed_by: materia") — these reference live cloud infrastructure and require separate cloud-side renames. Co-Authored-By: Claude Sonnet 4.6 --- docker-compose.prod.yml | 8 +-- infra/backup/beanflows-backup.service | 9 +++ ...ia-backup.timer => beanflows-backup.timer} | 2 +- infra/backup/materia-backup.service | 9 --- infra/bootstrap_supervisor.sh | 18 +++--- infra/setup_server.sh | 10 ++-- infra/supervisor/beanflows-supervisor.service | 29 +++++++++ infra/supervisor/materia-supervisor.service | 29 --------- pyproject.toml | 6 +- src/beanflows/__init__.py | 2 + src/{materia => beanflows}/cli.py | 18 +++--- src/{materia => beanflows}/export_serving.py | 2 +- src/{materia => beanflows}/pipelines.py | 4 +- .../providers/__init__.py | 2 +- .../providers/hetzner.py | 4 +- src/{materia => beanflows}/secrets.py | 0 src/{materia => beanflows}/supervisor.py | 59 +++++++++++++++---- src/{materia => beanflows}/workers.py | 4 +- src/materia/__init__.py | 2 - .../audits/.gitkeep | 0 .../audits/assert_positive_order_ids.sql | 0 .../config.yaml | 0 .../external_models.yaml | 0 .../macros/.gitkeep | 0 .../macros/__init__.py | 0 .../models/.gitkeep | 0 .../cln_psdalldata__commodity_pivoted.sql | 0 .../models/foundation/dim_commodity.sql | 0 .../models/foundation/fct_coffee_prices.sql | 0 .../models/foundation/fct_cot_positioning.sql | 0 .../foundation/fct_ice_aging_stocks.sql | 0 .../foundation/fct_ice_warehouse_stocks.sql | 0 .../fct_ice_warehouse_stocks_by_port.sql | 0 .../models/foundation/fct_weather_daily.sql | 0 .../models/seeds/psd_attribute_codes.sql | 0 .../models/seeds/psd_commodity_codes.sql | 0 .../seeds/psd_unit_of_measure_codes.sql | 0 .../models/seeds/weather_locations.sql | 0 .../models/serving/coffee_prices.sql | 0 .../models/serving/ice_aging_stocks.sql | 0 .../models/serving/ice_warehouse_stocks.sql | 0 .../serving/ice_warehouse_stocks_by_port.sql | 0 .../models/serving/obt_commodity_metrics.sql | 0 .../models/serving/obt_cot_positioning.sql | 0 .../serving/obt_cot_positioning_combined.sql | 0 .../models/serving/weather_daily.sql | 0 .../staging/stg_psdalldata__commodity.sql | 0 .../pyproject.toml | 4 +- .../readme.md | 0 .../seeds/.gitkeep | 0 .../seeds/commodity_exchange_codes.csv | 0 .../seeds/dim_commodity.csv | 0 .../seeds/psd_attribute_codes.csv | 0 .../seeds/psd_codes_exchange_codes_merge.csv | 0 .../seeds/psd_commodity_codes.csv | 0 .../seeds/psd_unit_of_measure_codes.csv | 0 .../seeds/weather_locations.csv | 0 .../tests/.gitkeep | 0 58 files changed, 128 insertions(+), 93 deletions(-) create mode 100644 infra/backup/beanflows-backup.service rename infra/backup/{materia-backup.timer => beanflows-backup.timer} (66%) delete mode 100644 infra/backup/materia-backup.service create mode 100644 infra/supervisor/beanflows-supervisor.service delete mode 100644 infra/supervisor/materia-supervisor.service create mode 100644 src/beanflows/__init__.py rename src/{materia => beanflows}/cli.py (91%) rename src/{materia => beanflows}/export_serving.py (95%) rename src/{materia => beanflows}/pipelines.py (94%) rename src/{materia => beanflows}/providers/__init__.py (88%) rename src/{materia => beanflows}/providers/hetzner.py (97%) rename src/{materia => beanflows}/secrets.py (100%) rename src/{materia => beanflows}/supervisor.py (88%) rename src/{materia => beanflows}/workers.py (92%) delete mode 100644 src/materia/__init__.py rename transform/{sqlmesh_materia => sqlmesh_beanflows}/audits/.gitkeep (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/audits/assert_positive_order_ids.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/config.yaml (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/external_models.yaml (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/macros/.gitkeep (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/macros/__init__.py (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/.gitkeep (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/cleaned/cln_psdalldata__commodity_pivoted.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/dim_commodity.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_coffee_prices.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_cot_positioning.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_ice_aging_stocks.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_ice_warehouse_stocks.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_ice_warehouse_stocks_by_port.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/foundation/fct_weather_daily.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/seeds/psd_attribute_codes.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/seeds/psd_commodity_codes.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/seeds/psd_unit_of_measure_codes.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/seeds/weather_locations.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/coffee_prices.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/ice_aging_stocks.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/ice_warehouse_stocks.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/ice_warehouse_stocks_by_port.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/obt_commodity_metrics.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/obt_cot_positioning.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/obt_cot_positioning_combined.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/serving/weather_daily.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/models/staging/stg_psdalldata__commodity.sql (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/pyproject.toml (84%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/readme.md (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/.gitkeep (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/commodity_exchange_codes.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/dim_commodity.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/psd_attribute_codes.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/psd_codes_exchange_codes_merge.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/psd_commodity_codes.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/psd_unit_of_measure_codes.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/seeds/weather_locations.csv (100%) rename transform/{sqlmesh_materia => sqlmesh_beanflows}/tests/.gitkeep (100%) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2fec3b4..eb1048d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -33,10 +33,10 @@ services: env_file: ./.env environment: - DATABASE_PATH=/app/data/app.db - - SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb + - SERVING_DUCKDB_PATH=/data/beanflows/analytics.duckdb volumes: - app-data:/app/data - - /data/materia/analytics.duckdb:/data/materia/analytics.duckdb:ro + - /data/beanflows:/data/beanflows:ro networks: - net healthcheck: @@ -84,10 +84,10 @@ services: env_file: ./.env environment: - DATABASE_PATH=/app/data/app.db - - SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb + - SERVING_DUCKDB_PATH=/data/beanflows/analytics.duckdb volumes: - app-data:/app/data - - /data/materia/analytics.duckdb:/data/materia/analytics.duckdb:ro + - /data/beanflows:/data/beanflows:ro networks: - net healthcheck: diff --git a/infra/backup/beanflows-backup.service b/infra/backup/beanflows-backup.service new file mode 100644 index 0000000..d9f7352 --- /dev/null +++ b/infra/backup/beanflows-backup.service @@ -0,0 +1,9 @@ +[Unit] +Description=Beanflows Landing Data Backup to R2 +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/rclone sync /data/beanflows/landing/ r2:backup/beanflows/landing/ --log-level INFO +TimeoutStartSec=1800 diff --git a/infra/backup/materia-backup.timer b/infra/backup/beanflows-backup.timer similarity index 66% rename from infra/backup/materia-backup.timer rename to infra/backup/beanflows-backup.timer index 90c84f0..bce8c18 100644 --- a/infra/backup/materia-backup.timer +++ b/infra/backup/beanflows-backup.timer @@ -1,5 +1,5 @@ [Unit] -Description=Materia Landing Data Backup Timer +Description=Beanflows Landing Data Backup Timer [Timer] OnCalendar=*-*-* 00/6:00:00 diff --git a/infra/backup/materia-backup.service b/infra/backup/materia-backup.service deleted file mode 100644 index 2ed27ed..0000000 --- a/infra/backup/materia-backup.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Materia Landing Data Backup to R2 -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/bin/rclone sync /data/materia/landing/ r2:backup/materia/landing/ --log-level INFO -TimeoutStartSec=1800 diff --git a/infra/bootstrap_supervisor.sh b/infra/bootstrap_supervisor.sh index c18b484..162cd48 100755 --- a/infra/bootstrap_supervisor.sh +++ b/infra/bootstrap_supervisor.sh @@ -14,7 +14,7 @@ set -euo pipefail SERVICE_USER="beanflows_service" -REPO_DIR="/opt/materia" +REPO_DIR="/opt/beanflows" GITEA_REPO="ssh://git@git.padelnomics.io:2222/deemanone/beanflows.git" UV="/home/${SERVICE_USER}/.local/bin/uv" @@ -57,9 +57,9 @@ sudo -u "${SERVICE_USER}" bash -c "cd ${REPO_DIR} && ${UV} sync --all-packages" # ── Systemd supervisor service ──────────────────────────────────────────────── -cp "${REPO_DIR}/infra/supervisor/materia-supervisor.service" /etc/systemd/system/ +cp "${REPO_DIR}/infra/supervisor/beanflows-supervisor.service" /etc/systemd/system/ systemctl daemon-reload -systemctl enable --now materia-supervisor +systemctl enable --now beanflows-supervisor # ── R2 backup timer (optional) ──────────────────────────────────────────────── # Enabled only when R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, and R2_ENDPOINT @@ -96,7 +96,7 @@ EOF chmod 600 "${RCLONE_CONF}" UNITS_CHANGED=0 - for unit in materia-backup.service materia-backup.timer; do + for unit in beanflows-backup.service beanflows-backup.timer; do if ! diff -q "${REPO_DIR}/infra/backup/${unit}" "/etc/systemd/system/${unit}" >/dev/null 2>&1; then cp "${REPO_DIR}/infra/backup/${unit}" /etc/systemd/system/ UNITS_CHANGED=1 @@ -104,7 +104,7 @@ EOF done [ "${UNITS_CHANGED}" = "1" ] && systemctl daemon-reload - systemctl enable --now materia-backup.timer + systemctl enable --now beanflows-backup.timer echo "$(date '+%H:%M:%S') ==> R2 backup timer enabled." else echo "$(date '+%H:%M:%S') ==> R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY / R2_ENDPOINT not set — skipping backup timer." @@ -113,8 +113,8 @@ fi echo "" echo "=== Bootstrap complete! ===" echo "" -echo "Check status: systemctl status materia-supervisor" -echo "View logs: journalctl -u materia-supervisor -f" -echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/materia/supervisor.py status" -echo "Backup timer: systemctl list-timers materia-backup.timer" +echo "Check status: systemctl status beanflows-supervisor" +echo "View logs: journalctl -u beanflows-supervisor -f" +echo "Workflow status: sudo -u ${SERVICE_USER} ${UV} run -p ${REPO_DIR} python src/beanflows/supervisor.py status" +echo "Backup timer: systemctl list-timers beanflows-backup.timer" echo "Tag: $(sudo -u "${SERVICE_USER}" git -C "${REPO_DIR}" describe --tags --always)" diff --git a/infra/setup_server.sh b/infra/setup_server.sh index 35d210c..e789f54 100644 --- a/infra/setup_server.sh +++ b/infra/setup_server.sh @@ -8,7 +8,7 @@ # # What it does: # 1. Creates beanflows_service user (nologin) + adds to docker group -# 2. Creates /opt/materia + /data/materia/landing with correct ownership +# 2. Creates /opt/beanflows + /data/beanflows/landing with correct ownership # 3. Installs git, curl, age, sops, rclone, uv # 4. Generates ed25519 SSH deploy key for GitLab read access # 5. Generates age keypair at ~/.config/sops/age/keys.txt (as service user) @@ -17,10 +17,10 @@ set -euo pipefail SERVICE_USER="beanflows_service" -APP_DIR="/opt/materia" -DATA_DIR="/data/materia" +APP_DIR="/opt/beanflows" +DATA_DIR="/data/beanflows" SSH_DIR="/home/${SERVICE_USER}/.ssh" -DEPLOY_KEY="${SSH_DIR}/materia_deploy" +DEPLOY_KEY="${SSH_DIR}/beanflows_deploy" SOPS_AGE_DIR="/home/${SERVICE_USER}/.config/sops/age" ROTATE_KEYS="${ROTATE_KEYS:-}" @@ -63,7 +63,7 @@ fi if [ ! -f "${DEPLOY_KEY}" ]; then sudo -u "${SERVICE_USER}" ssh-keygen -t ed25519 \ - -f "${DEPLOY_KEY}" -N "" -C "materia-deploy" + -f "${DEPLOY_KEY}" -N "" -C "beanflows-deploy" fi if [ ! -f "${SSH_DIR}/config" ]; then diff --git a/infra/supervisor/beanflows-supervisor.service b/infra/supervisor/beanflows-supervisor.service new file mode 100644 index 0000000..4c15144 --- /dev/null +++ b/infra/supervisor/beanflows-supervisor.service @@ -0,0 +1,29 @@ +[Unit] +Description=Beanflows Supervisor - Pipeline Orchestration +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=beanflows_service +WorkingDirectory=/opt/beanflows +ExecStart=/bin/sh -c 'exec uv run python src/beanflows/supervisor.py' +Restart=always +RestartSec=10 +EnvironmentFile=/opt/beanflows/.env +Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=LANDING_DIR=/data/beanflows/landing +Environment=DUCKDB_PATH=/data/beanflows/lakehouse.duckdb +Environment=SERVING_DUCKDB_PATH=/data/beanflows/analytics.duckdb +Environment=SUPERVISOR_GIT_PULL=1 + +# Resource limits +LimitNOFILE=65536 + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=beanflows-supervisor + +[Install] +WantedBy=multi-user.target diff --git a/infra/supervisor/materia-supervisor.service b/infra/supervisor/materia-supervisor.service deleted file mode 100644 index d9dff68..0000000 --- a/infra/supervisor/materia-supervisor.service +++ /dev/null @@ -1,29 +0,0 @@ -[Unit] -Description=Materia Supervisor - Pipeline Orchestration -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=beanflows_service -WorkingDirectory=/opt/materia -ExecStart=/bin/sh -c 'exec uv run python src/materia/supervisor.py' -Restart=always -RestartSec=10 -EnvironmentFile=/opt/materia/.env -Environment=PATH=/home/beanflows_service/.local/bin:/usr/local/bin:/usr/bin:/bin -Environment=LANDING_DIR=/data/materia/landing -Environment=DUCKDB_PATH=/data/materia/lakehouse.duckdb -Environment=SERVING_DUCKDB_PATH=/data/materia/analytics.duckdb -Environment=SUPERVISOR_GIT_PULL=1 - -# Resource limits -LimitNOFILE=65536 - -# Logging -StandardOutput=journal -StandardError=journal -SyslogIdentifier=materia-supervisor - -[Install] -WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml index 9364ad6..6420416 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "materia" +name = "beanflows" version = "0.1.0" description = "Add your description here" readme = "readme.md" @@ -20,7 +20,7 @@ dependencies = [ ] [project.scripts] -materia = "materia.cli:app" +beanflows = "beanflows.cli:app" [dependency-groups] @@ -43,7 +43,7 @@ dev = [ [tool.uv.sources] extract_core = {workspace = true } psdonline = {workspace = true } -sqlmesh_materia = {workspace = true } +sqlmesh_beanflows = {workspace = true } cftc_cot = {workspace = true } coffee_prices = {workspace = true } ice_stocks = {workspace = true } diff --git a/src/beanflows/__init__.py b/src/beanflows/__init__.py new file mode 100644 index 0000000..ddeb40f --- /dev/null +++ b/src/beanflows/__init__.py @@ -0,0 +1,2 @@ +def main() -> None: + print("Hello from beanflows!") diff --git a/src/materia/cli.py b/src/beanflows/cli.py similarity index 91% rename from src/materia/cli.py rename to src/beanflows/cli.py index 8e52517..bc91e19 100644 --- a/src/materia/cli.py +++ b/src/beanflows/cli.py @@ -5,7 +5,7 @@ from typing import Annotated import typer app = typer.Typer( - name="materia", + name="beanflows", help="BeanFlows.coffee data platform management CLI", no_args_is_help=True, ) @@ -26,7 +26,7 @@ def worker_list( provider: Annotated[str, typer.Option("--provider", "-p")] = "hetzner", ): """List all active worker instances.""" - from materia.workers import list_workers + from beanflows.workers import list_workers workers = list_workers(provider) if not workers: @@ -47,7 +47,7 @@ def worker_create( location: Annotated[str | None, typer.Option("--location", "-l")] = None, ): """Create a new worker instance.""" - from materia.workers import create_worker + from beanflows.workers import create_worker typer.echo(f"Creating worker '{name}' ({server_type}) on {provider}...") worker = create_worker(name, server_type, provider, location) @@ -61,7 +61,7 @@ def worker_destroy( force: Annotated[bool, typer.Option("--force", "-f")] = False, ): """Destroy a worker instance.""" - from materia.workers import destroy_worker + from beanflows.workers import destroy_worker if not force: confirm = typer.confirm(f"Destroy worker '{name}'?") @@ -82,7 +82,7 @@ def pipeline_run( name: Annotated[str, typer.Argument(help="Pipeline name (extract, transform)")], ): """Run a pipeline locally.""" - from materia.pipelines import run_pipeline + from beanflows.pipelines import run_pipeline typer.echo(f"Running pipeline '{name}'...") result = run_pipeline(name) @@ -98,7 +98,7 @@ def pipeline_run( @pipeline_app.command("list") def pipeline_list(): """List available pipelines.""" - from materia.pipelines import PIPELINES + from beanflows.pipelines import PIPELINES typer.echo("Available pipelines:") for name, config in PIPELINES.items(): @@ -113,7 +113,7 @@ app.add_typer(secrets_app, name="secrets") @secrets_app.command("list") def secrets_list(): """List available secrets (keys only).""" - from materia.secrets import list_secrets + from beanflows.secrets import list_secrets secrets = list_secrets() if not secrets: @@ -130,7 +130,7 @@ def secrets_get( key: Annotated[str, typer.Argument(help="Secret key")], ): """Get a secret value.""" - from materia.secrets import get_secret + from beanflows.secrets import get_secret value = get_secret(key) if value is None: @@ -143,7 +143,7 @@ def secrets_get( @secrets_app.command("test") def secrets_test(): """Test sops decryption (verifies sops is installed and age key is present).""" - from materia.secrets import test_connection + from beanflows.secrets import test_connection typer.echo("Testing SOPS decryption...") if test_connection(): diff --git a/src/materia/export_serving.py b/src/beanflows/export_serving.py similarity index 95% rename from src/materia/export_serving.py rename to src/beanflows/export_serving.py index 684ac20..a1c129a 100644 --- a/src/materia/export_serving.py +++ b/src/beanflows/export_serving.py @@ -11,7 +11,7 @@ reopens the connection automatically — no restart or signal required. Usage: DUCKDB_PATH=lakehouse.duckdb SERVING_DUCKDB_PATH=serving.duckdb \ - uv run materia pipeline run export_serving + uv run beanflows pipeline run export_serving """ import logging import os diff --git a/src/materia/pipelines.py b/src/beanflows/pipelines.py similarity index 94% rename from src/materia/pipelines.py rename to src/beanflows/pipelines.py index 0fb4bbe..d3ec673 100644 --- a/src/materia/pipelines.py +++ b/src/beanflows/pipelines.py @@ -57,7 +57,7 @@ PIPELINES = { "timeout_seconds": 6600, }, "transform": { - "command": ["uv", "run", "--package", "sqlmesh_materia", "sqlmesh", "-p", "transform/sqlmesh_materia", "plan", "prod", "--no-prompts", "--auto-apply"], + "command": ["uv", "run", "--package", "sqlmesh_beanflows", "sqlmesh", "-p", "transform/sqlmesh_beanflows", "plan", "prod", "--no-prompts", "--auto-apply"], "timeout_seconds": 3600, }, # Copies serving.* tables from lakehouse.duckdb → serving.duckdb (atomic swap). @@ -65,7 +65,7 @@ PIPELINES = { "export_serving": { "command": ["uv", "run", "python", "-c", "import logging; logging.basicConfig(level=logging.INFO); " - "from materia.export_serving import export_serving; export_serving()"], + "from beanflows.export_serving import export_serving; export_serving()"], "timeout_seconds": 300, }, } diff --git a/src/materia/providers/__init__.py b/src/beanflows/providers/__init__.py similarity index 88% rename from src/materia/providers/__init__.py rename to src/beanflows/providers/__init__.py index 86ffc1e..fd223ba 100644 --- a/src/materia/providers/__init__.py +++ b/src/beanflows/providers/__init__.py @@ -15,7 +15,7 @@ class Instance: def get_provider(provider_name: str): if provider_name == "hetzner": - from materia.providers import hetzner + from beanflows.providers import hetzner return hetzner else: diff --git a/src/materia/providers/hetzner.py b/src/beanflows/providers/hetzner.py similarity index 97% rename from src/materia/providers/hetzner.py rename to src/beanflows/providers/hetzner.py index a8bfca2..fb599f2 100644 --- a/src/materia/providers/hetzner.py +++ b/src/beanflows/providers/hetzner.py @@ -7,8 +7,8 @@ from hcloud import Client from hcloud.images import Image from hcloud.server_types import ServerType -from materia.providers import Instance -from materia.secrets import get_secret +from beanflows.providers import Instance +from beanflows.secrets import get_secret def _get_client() -> Client: diff --git a/src/materia/secrets.py b/src/beanflows/secrets.py similarity index 100% rename from src/materia/secrets.py rename to src/beanflows/secrets.py diff --git a/src/materia/supervisor.py b/src/beanflows/supervisor.py similarity index 88% rename from src/materia/supervisor.py rename to src/beanflows/supervisor.py index 7ee1e2e..6a04f68 100644 --- a/src/materia/supervisor.py +++ b/src/beanflows/supervisor.py @@ -11,10 +11,10 @@ the supervisor is effectively unkillable. Usage: # Run the supervisor loop (production) - LANDING_DIR=data/landing uv run python src/materia/supervisor.py + LANDING_DIR=data/landing uv run python src/beanflows/supervisor.py # Show workflow status - LANDING_DIR=data/landing uv run python src/materia/supervisor.py status + LANDING_DIR=data/landing uv run python src/beanflows/supervisor.py status """ import importlib @@ -38,7 +38,7 @@ from croniter import croniter TICK_INTERVAL_SECONDS = 60 BACKOFF_SECONDS = 600 # 10 min on tick failure SUBPROCESS_TIMEOUT_SECONDS = 14400 # 4 hours max per subprocess -REPO_DIR = Path(os.getenv("REPO_DIR", "/opt/materia")) +REPO_DIR = Path(os.getenv("REPO_DIR", "/opt/beanflows")) LANDING_DIR = Path(os.getenv("LANDING_DIR", "data/landing")) DUCKDB_PATH = os.getenv("DUCKDB_PATH", "data/lakehouse.duckdb") SERVING_DUCKDB_PATH = os.getenv("SERVING_DUCKDB_PATH", "analytics.duckdb") @@ -58,7 +58,7 @@ logging.basicConfig( datefmt="%Y-%m-%d %H:%M:%S", handlers=[logging.StreamHandler(sys.stdout)], ) -logger = logging.getLogger("materia.supervisor") +logger = logging.getLogger("beanflows.supervisor") # --------------------------------------------------------------------------- @@ -242,7 +242,7 @@ def run_shell(cmd: str, timeout_seconds: int = SUBPROCESS_TIMEOUT_SECONDS) -> tu def run_transform() -> None: """Run SQLMesh — evaluates model staleness internally.""" logger.info("Running SQLMesh transform") - ok, err = run_shell("uv run sqlmesh -p transform/sqlmesh_materia plan prod --auto-apply") + ok, err = run_shell("uv run sqlmesh -p transform/sqlmesh_beanflows plan prod --auto-apply") if not ok: send_alert(f"[transform] {err}") @@ -252,28 +252,63 @@ def run_export() -> None: logger.info("Exporting serving tables") ok, err = run_shell( f"DUCKDB_PATH={DUCKDB_PATH} SERVING_DUCKDB_PATH={SERVING_DUCKDB_PATH} " - f"uv run materia pipeline run export_serving" + f"uv run beanflows pipeline run export_serving" ) if not ok: send_alert(f"[export] {err}") +_last_seen_head: str | None = None + + def web_code_changed() -> bool: - """Check if web app code changed since last deploy.""" + """True on the first tick after a commit that changed web app code. + + Compares the current HEAD to the HEAD from the previous tick. On first call + after process start, falls back to HEAD~1 so the just-deployed commit is + evaluated exactly once. Records HEAD before returning so the same commit + never triggers twice. + """ + global _last_seen_head result = subprocess.run( - ["git", "diff", "--name-only", "HEAD~1", "HEAD", "--", "web/", "Dockerfile"], + ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10, + ) + if result.returncode != 0: + return False + current_head = result.stdout.strip() + + if _last_seen_head is None: + base_result = subprocess.run( + ["git", "rev-parse", "HEAD~1"], capture_output=True, text=True, timeout=10, + ) + base = base_result.stdout.strip() if base_result.returncode == 0 else current_head + else: + base = _last_seen_head + + _last_seen_head = current_head # advance now — won't fire again for this HEAD + + if base == current_head: + return False + + diff = subprocess.run( + ["git", "diff", "--name-only", base, current_head, "--", "web/", "Dockerfile"], capture_output=True, text=True, timeout=30, ) - return bool(result.stdout.strip()) + return bool(diff.stdout.strip()) def current_deployed_tag() -> str | None: - """Return the tag currently checked out, or None if not on a tag.""" + """Return the highest-version tag pointing at HEAD, or None. + + Uses --points-at HEAD so multiple tags on the same commit (e.g. a CI + integer tag and a date-based tag) are handled correctly. + """ result = subprocess.run( - ["git", "describe", "--tags", "--exact-match", "HEAD"], + ["git", "tag", "--list", "--sort=-version:refname", "--points-at", "HEAD", "v*"], capture_output=True, text=True, timeout=10, ) - return result.stdout.strip() or None + tags = result.stdout.strip().splitlines() + return tags[0] if tags else None def latest_remote_tag() -> str | None: diff --git a/src/materia/workers.py b/src/beanflows/workers.py similarity index 92% rename from src/materia/workers.py rename to src/beanflows/workers.py index c8f8699..47548bc 100644 --- a/src/materia/workers.py +++ b/src/beanflows/workers.py @@ -1,7 +1,7 @@ """Worker instance management.""" -from materia.providers import Instance, get_provider -from materia.secrets import get_secret +from beanflows.providers import Instance, get_provider +from beanflows.secrets import get_secret DEFAULT_PROVIDER = "hetzner" diff --git a/src/materia/__init__.py b/src/materia/__init__.py deleted file mode 100644 index e95c78c..0000000 --- a/src/materia/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -def main() -> None: - print("Hello from materia!") diff --git a/transform/sqlmesh_materia/audits/.gitkeep b/transform/sqlmesh_beanflows/audits/.gitkeep similarity index 100% rename from transform/sqlmesh_materia/audits/.gitkeep rename to transform/sqlmesh_beanflows/audits/.gitkeep diff --git a/transform/sqlmesh_materia/audits/assert_positive_order_ids.sql b/transform/sqlmesh_beanflows/audits/assert_positive_order_ids.sql similarity index 100% rename from transform/sqlmesh_materia/audits/assert_positive_order_ids.sql rename to transform/sqlmesh_beanflows/audits/assert_positive_order_ids.sql diff --git a/transform/sqlmesh_materia/config.yaml b/transform/sqlmesh_beanflows/config.yaml similarity index 100% rename from transform/sqlmesh_materia/config.yaml rename to transform/sqlmesh_beanflows/config.yaml diff --git a/transform/sqlmesh_materia/external_models.yaml b/transform/sqlmesh_beanflows/external_models.yaml similarity index 100% rename from transform/sqlmesh_materia/external_models.yaml rename to transform/sqlmesh_beanflows/external_models.yaml diff --git a/transform/sqlmesh_materia/macros/.gitkeep b/transform/sqlmesh_beanflows/macros/.gitkeep similarity index 100% rename from transform/sqlmesh_materia/macros/.gitkeep rename to transform/sqlmesh_beanflows/macros/.gitkeep diff --git a/transform/sqlmesh_materia/macros/__init__.py b/transform/sqlmesh_beanflows/macros/__init__.py similarity index 100% rename from transform/sqlmesh_materia/macros/__init__.py rename to transform/sqlmesh_beanflows/macros/__init__.py diff --git a/transform/sqlmesh_materia/models/.gitkeep b/transform/sqlmesh_beanflows/models/.gitkeep similarity index 100% rename from transform/sqlmesh_materia/models/.gitkeep rename to transform/sqlmesh_beanflows/models/.gitkeep diff --git a/transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql b/transform/sqlmesh_beanflows/models/cleaned/cln_psdalldata__commodity_pivoted.sql similarity index 100% rename from transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql rename to transform/sqlmesh_beanflows/models/cleaned/cln_psdalldata__commodity_pivoted.sql diff --git a/transform/sqlmesh_materia/models/foundation/dim_commodity.sql b/transform/sqlmesh_beanflows/models/foundation/dim_commodity.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/dim_commodity.sql rename to transform/sqlmesh_beanflows/models/foundation/dim_commodity.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_coffee_prices.sql b/transform/sqlmesh_beanflows/models/foundation/fct_coffee_prices.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_coffee_prices.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_coffee_prices.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_cot_positioning.sql b/transform/sqlmesh_beanflows/models/foundation/fct_cot_positioning.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_cot_positioning.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_cot_positioning.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_ice_aging_stocks.sql b/transform/sqlmesh_beanflows/models/foundation/fct_ice_aging_stocks.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_ice_aging_stocks.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_ice_aging_stocks.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_ice_warehouse_stocks.sql b/transform/sqlmesh_beanflows/models/foundation/fct_ice_warehouse_stocks.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_ice_warehouse_stocks.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_ice_warehouse_stocks.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_ice_warehouse_stocks_by_port.sql b/transform/sqlmesh_beanflows/models/foundation/fct_ice_warehouse_stocks_by_port.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_ice_warehouse_stocks_by_port.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_ice_warehouse_stocks_by_port.sql diff --git a/transform/sqlmesh_materia/models/foundation/fct_weather_daily.sql b/transform/sqlmesh_beanflows/models/foundation/fct_weather_daily.sql similarity index 100% rename from transform/sqlmesh_materia/models/foundation/fct_weather_daily.sql rename to transform/sqlmesh_beanflows/models/foundation/fct_weather_daily.sql diff --git a/transform/sqlmesh_materia/models/seeds/psd_attribute_codes.sql b/transform/sqlmesh_beanflows/models/seeds/psd_attribute_codes.sql similarity index 100% rename from transform/sqlmesh_materia/models/seeds/psd_attribute_codes.sql rename to transform/sqlmesh_beanflows/models/seeds/psd_attribute_codes.sql diff --git a/transform/sqlmesh_materia/models/seeds/psd_commodity_codes.sql b/transform/sqlmesh_beanflows/models/seeds/psd_commodity_codes.sql similarity index 100% rename from transform/sqlmesh_materia/models/seeds/psd_commodity_codes.sql rename to transform/sqlmesh_beanflows/models/seeds/psd_commodity_codes.sql diff --git a/transform/sqlmesh_materia/models/seeds/psd_unit_of_measure_codes.sql b/transform/sqlmesh_beanflows/models/seeds/psd_unit_of_measure_codes.sql similarity index 100% rename from transform/sqlmesh_materia/models/seeds/psd_unit_of_measure_codes.sql rename to transform/sqlmesh_beanflows/models/seeds/psd_unit_of_measure_codes.sql diff --git a/transform/sqlmesh_materia/models/seeds/weather_locations.sql b/transform/sqlmesh_beanflows/models/seeds/weather_locations.sql similarity index 100% rename from transform/sqlmesh_materia/models/seeds/weather_locations.sql rename to transform/sqlmesh_beanflows/models/seeds/weather_locations.sql diff --git a/transform/sqlmesh_materia/models/serving/coffee_prices.sql b/transform/sqlmesh_beanflows/models/serving/coffee_prices.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/coffee_prices.sql rename to transform/sqlmesh_beanflows/models/serving/coffee_prices.sql diff --git a/transform/sqlmesh_materia/models/serving/ice_aging_stocks.sql b/transform/sqlmesh_beanflows/models/serving/ice_aging_stocks.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/ice_aging_stocks.sql rename to transform/sqlmesh_beanflows/models/serving/ice_aging_stocks.sql diff --git a/transform/sqlmesh_materia/models/serving/ice_warehouse_stocks.sql b/transform/sqlmesh_beanflows/models/serving/ice_warehouse_stocks.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/ice_warehouse_stocks.sql rename to transform/sqlmesh_beanflows/models/serving/ice_warehouse_stocks.sql diff --git a/transform/sqlmesh_materia/models/serving/ice_warehouse_stocks_by_port.sql b/transform/sqlmesh_beanflows/models/serving/ice_warehouse_stocks_by_port.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/ice_warehouse_stocks_by_port.sql rename to transform/sqlmesh_beanflows/models/serving/ice_warehouse_stocks_by_port.sql diff --git a/transform/sqlmesh_materia/models/serving/obt_commodity_metrics.sql b/transform/sqlmesh_beanflows/models/serving/obt_commodity_metrics.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/obt_commodity_metrics.sql rename to transform/sqlmesh_beanflows/models/serving/obt_commodity_metrics.sql diff --git a/transform/sqlmesh_materia/models/serving/obt_cot_positioning.sql b/transform/sqlmesh_beanflows/models/serving/obt_cot_positioning.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/obt_cot_positioning.sql rename to transform/sqlmesh_beanflows/models/serving/obt_cot_positioning.sql diff --git a/transform/sqlmesh_materia/models/serving/obt_cot_positioning_combined.sql b/transform/sqlmesh_beanflows/models/serving/obt_cot_positioning_combined.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/obt_cot_positioning_combined.sql rename to transform/sqlmesh_beanflows/models/serving/obt_cot_positioning_combined.sql diff --git a/transform/sqlmesh_materia/models/serving/weather_daily.sql b/transform/sqlmesh_beanflows/models/serving/weather_daily.sql similarity index 100% rename from transform/sqlmesh_materia/models/serving/weather_daily.sql rename to transform/sqlmesh_beanflows/models/serving/weather_daily.sql diff --git a/transform/sqlmesh_materia/models/staging/stg_psdalldata__commodity.sql b/transform/sqlmesh_beanflows/models/staging/stg_psdalldata__commodity.sql similarity index 100% rename from transform/sqlmesh_materia/models/staging/stg_psdalldata__commodity.sql rename to transform/sqlmesh_beanflows/models/staging/stg_psdalldata__commodity.sql diff --git a/transform/sqlmesh_materia/pyproject.toml b/transform/sqlmesh_beanflows/pyproject.toml similarity index 84% rename from transform/sqlmesh_materia/pyproject.toml rename to transform/sqlmesh_beanflows/pyproject.toml index d393e87..e659fa3 100644 --- a/transform/sqlmesh_materia/pyproject.toml +++ b/transform/sqlmesh_beanflows/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "sqlmesh_materia" +name = "sqlmesh_beanflows" version = "0.1.0" description = "Add your description here" authors = [ @@ -16,4 +16,4 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["sqlmesh_materia"] +packages = ["sqlmesh_beanflows"] diff --git a/transform/sqlmesh_materia/readme.md b/transform/sqlmesh_beanflows/readme.md similarity index 100% rename from transform/sqlmesh_materia/readme.md rename to transform/sqlmesh_beanflows/readme.md diff --git a/transform/sqlmesh_materia/seeds/.gitkeep b/transform/sqlmesh_beanflows/seeds/.gitkeep similarity index 100% rename from transform/sqlmesh_materia/seeds/.gitkeep rename to transform/sqlmesh_beanflows/seeds/.gitkeep diff --git a/transform/sqlmesh_materia/seeds/commodity_exchange_codes.csv b/transform/sqlmesh_beanflows/seeds/commodity_exchange_codes.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/commodity_exchange_codes.csv rename to transform/sqlmesh_beanflows/seeds/commodity_exchange_codes.csv diff --git a/transform/sqlmesh_materia/seeds/dim_commodity.csv b/transform/sqlmesh_beanflows/seeds/dim_commodity.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/dim_commodity.csv rename to transform/sqlmesh_beanflows/seeds/dim_commodity.csv diff --git a/transform/sqlmesh_materia/seeds/psd_attribute_codes.csv b/transform/sqlmesh_beanflows/seeds/psd_attribute_codes.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/psd_attribute_codes.csv rename to transform/sqlmesh_beanflows/seeds/psd_attribute_codes.csv diff --git a/transform/sqlmesh_materia/seeds/psd_codes_exchange_codes_merge.csv b/transform/sqlmesh_beanflows/seeds/psd_codes_exchange_codes_merge.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/psd_codes_exchange_codes_merge.csv rename to transform/sqlmesh_beanflows/seeds/psd_codes_exchange_codes_merge.csv diff --git a/transform/sqlmesh_materia/seeds/psd_commodity_codes.csv b/transform/sqlmesh_beanflows/seeds/psd_commodity_codes.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/psd_commodity_codes.csv rename to transform/sqlmesh_beanflows/seeds/psd_commodity_codes.csv diff --git a/transform/sqlmesh_materia/seeds/psd_unit_of_measure_codes.csv b/transform/sqlmesh_beanflows/seeds/psd_unit_of_measure_codes.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/psd_unit_of_measure_codes.csv rename to transform/sqlmesh_beanflows/seeds/psd_unit_of_measure_codes.csv diff --git a/transform/sqlmesh_materia/seeds/weather_locations.csv b/transform/sqlmesh_beanflows/seeds/weather_locations.csv similarity index 100% rename from transform/sqlmesh_materia/seeds/weather_locations.csv rename to transform/sqlmesh_beanflows/seeds/weather_locations.csv diff --git a/transform/sqlmesh_materia/tests/.gitkeep b/transform/sqlmesh_beanflows/tests/.gitkeep similarity index 100% rename from transform/sqlmesh_materia/tests/.gitkeep rename to transform/sqlmesh_beanflows/tests/.gitkeep