c1d00dcdc4
Remove distributed R2/Iceberg/SSH pipeline architecture in favor of
local subprocess execution with NVMe storage. Landing data backed up
to R2 via rclone timer.
- Strip Iceberg catalog, httpfs, boto3, paramiko, prefect, pyarrow
- Pipelines run via subprocess.run() with bounded timeouts
- Extract writes to {LANDING_DIR}/psd/{year}/{month}/{etag}.csv.gzip
- SQLMesh reads LANDING_DIR variable, writes to DUCKDB_PATH
- Delete unused provider stubs (ovh, scaleway, oracle)
- Add rclone systemd timer for R2 backup every 6h
- Update supervisor to run pipelines with env vars
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
"""Pytest configuration and fixtures."""
|
|
|
|
from unittest.mock import Mock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_esc_env(tmp_path):
|
|
"""Mock Pulumi ESC environment variables."""
|
|
ssh_key_path = tmp_path / "test_key"
|
|
ssh_key_path.write_text("-----BEGIN OPENSSH PRIVATE KEY-----\ntest\n-----END OPENSSH PRIVATE KEY-----")
|
|
|
|
return {
|
|
"HETZNER_API_TOKEN": "test-hetzner-token",
|
|
"SSH_PUBLIC_KEY": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITest",
|
|
"SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\ntest\n-----END OPENSSH PRIVATE KEY-----",
|
|
"SSH_PRIVATE_KEY_PATH": str(ssh_key_path),
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_secrets(mock_esc_env):
|
|
"""Mock the secrets module to return test secrets."""
|
|
with patch("materia.secrets._load_environment", return_value=mock_esc_env):
|
|
yield
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_hcloud_client():
|
|
"""Mock Hetzner Cloud client."""
|
|
with patch("materia.providers.hetzner.Client") as mock_client:
|
|
client_instance = Mock()
|
|
mock_client.return_value = client_instance
|
|
|
|
client_instance.ssh_keys.get_all.return_value = []
|
|
client_instance.ssh_keys.create.return_value = Mock(id=1, name="materia-key")
|
|
|
|
mock_server = Mock()
|
|
mock_server.id = 12345
|
|
mock_server.name = "test-worker"
|
|
mock_server.status = "running"
|
|
mock_server.public_net.ipv4.ip = "192.0.2.1"
|
|
mock_server.server_type.name = "ccx12"
|
|
mock_server.wait_until_status_is = Mock()
|
|
mock_server.delete = Mock()
|
|
|
|
mock_response = Mock()
|
|
mock_response.server = mock_server
|
|
|
|
client_instance.servers.create.return_value = mock_response
|
|
client_instance.servers.get_all.return_value = []
|
|
client_instance.servers.get_by_id.return_value = mock_server
|
|
|
|
yield client_instance
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_ssh_wait():
|
|
"""Mock SSH wait function to return immediately."""
|
|
with patch("materia.providers.hetzner.wait_for_ssh", return_value=True):
|
|
yield
|