0a83b2cb74
- New extraction package (cftc_cot): downloads yearly Disaggregated Futures ZIPs from CFTC, etag-based dedup, dynamic inner filename discovery, gzip normalization - SQLMesh 3-layer architecture: raw (technical) → foundation (business model) → serving (mart) - dim_commodity seed: conformed dimension mapping USDA ↔ CFTC codes — the commodity ontology - fct_cot_positioning: typed, deduplicated weekly positioning facts for all commodities - obt_cot_positioning: Coffee C mart with COT Index (26w/52w), WoW delta, OI ratios - Analytics functions + REST API endpoints: /commodities/<code>/positioning[/latest] - Dashboard widget: Managed Money net, COT Index card, dual-axis Chart.js chart - 23 passing tests (10 unit + 2 SQLMesh model + existing regression suite) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
61 lines
1.7 KiB
Python
61 lines
1.7 KiB
Python
"""Pipeline execution via local subprocess."""
|
|
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class PipelineResult:
|
|
success: bool
|
|
output: str
|
|
error: str | None = None
|
|
|
|
|
|
PIPELINES = {
|
|
"extract": {
|
|
"command": ["uv", "run", "--package", "psdonline", "extract_psd"],
|
|
"timeout_seconds": 1800,
|
|
},
|
|
"extract_cot": {
|
|
"command": ["uv", "run", "--package", "cftc_cot", "extract_cot"],
|
|
"timeout_seconds": 1800,
|
|
},
|
|
"transform": {
|
|
"command": ["uv", "run", "--package", "sqlmesh_materia", "sqlmesh", "-p", "transform/sqlmesh_materia", "plan", "prod", "--no-prompts", "--auto-apply"],
|
|
"timeout_seconds": 3600,
|
|
},
|
|
}
|
|
|
|
|
|
def run_pipeline(pipeline_name: str) -> PipelineResult:
|
|
assert pipeline_name, "pipeline_name must not be empty"
|
|
|
|
if pipeline_name not in PIPELINES:
|
|
return PipelineResult(
|
|
success=False,
|
|
output="",
|
|
error=f"Unknown pipeline: {pipeline_name}. Available: {', '.join(PIPELINES.keys())}",
|
|
)
|
|
|
|
pipeline = PIPELINES[pipeline_name]
|
|
timeout_seconds = pipeline["timeout_seconds"]
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
pipeline["command"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout_seconds,
|
|
)
|
|
return PipelineResult(
|
|
success=result.returncode == 0,
|
|
output=result.stdout,
|
|
error=result.stderr if result.returncode != 0 else None,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
return PipelineResult(
|
|
success=False,
|
|
output="",
|
|
error=f"Pipeline '{pipeline_name}' timed out after {timeout_seconds} seconds",
|
|
)
|