deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

uncomment code

Browse Source
This commit is contained in:
Deeman
2025-05-08 18:02:45 +02:00
parent d84e39fd9f
commit 10d9424ff1
1 changed files with 60 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
extract/dlt_script.py
View File

@@ -112,75 +112,75 @@ def run_pipeline():
{"name": "psd_commodity_attributes", "endpoint": {"path": "/api/psd/commodityAttributes"}}, {"name": "psd_commodity_attributes", "endpoint": {"path": "/api/psd/commodityAttributes"}},
# Dynamic ESR exports resources for each commodity and year # Dynamic ESR exports resources for each commodity and year
#*[ *[
# { {
# "name": f"esr_exports_commodity_{comm_code}_year_{year}", "name": f"esr_exports_commodity_{comm_code}_year_{year}",
# "endpoint": { "endpoint": {
# "path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}", "path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}",
# "response_actions": [ "response_actions": [
# {"status_code": 404, "action": "ignore"} {"status_code": 404, "action": "ignore"}
# ] ]
# }, },
# "primary_key": ["commodityCode", "countryCode", "weekEnding"], "primary_key": ["commodityCode", "countryCode", "weekEnding"],
# "write_disposition": "append" "write_disposition": "append"
# } }
# for comm_code in esr_commodity_codes for comm_code in esr_commodity_codes
# for year in years for year in years
#], ],
# Sample GATS census imports for major trading partners (for all recent years) # Sample GATS census imports for major trading partners (for all recent years)
# You can expand to more countries as needed # You can expand to more countries as needed
#*[ *[
# { {
# "name": f"gats_census_imports_{country_code}_{year}_{month:02d}", "name": f"gats_census_imports_{country_code}_{year}_{month:02d}",
# "endpoint": { "endpoint": {
# "path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}", "path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}",
# "response_actions": [ "response_actions": [
# {"status_code": 404, "action": "ignore"} {"status_code": 404, "action": "ignore"}
# ] ]
# }, },
# "write_disposition": "append" "write_disposition": "append"
# } }
# for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners
# for year in years for year in years
# for month in range(1, 13) for month in range(1, 13)
# if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month) if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month)
#], ],
# Dynamic PSD commodity data for all commodities and years # Dynamic PSD commodity data for all commodities and years
#*[ *[
# { {
# "name": f"psd_commodity_{comm_code}_year_{year}", "name": f"psd_commodity_{comm_code}_year_{year}",
# "endpoint": { "endpoint": {
# "path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}", "path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}",
# "response_actions": [ "response_actions": [
# {"status_code": 404, "action": "ignore"} {"status_code": 404, "action": "ignore"}
# ] ]
# }, },
# "write_disposition": "append" "write_disposition": "append"
# } }
# for comm_code in psd_commodity_codes for comm_code in psd_commodity_codes
# for year in years for year in years
#], ],
# World-level data for all commodities # World-level data for all commodities
#*[ *[
# { {
# "name": f"psd_world_commodity_{comm_code}_year_{year}", "name": f"psd_world_commodity_{comm_code}_year_{year}",
# "endpoint": { "endpoint": {
# "path": f"/api/psd/commodity/{comm_code}/world/year/{year}", "path": f"/api/psd/commodity/{comm_code}/world/year/{year}",
# "response_actions": [ "response_actions": [
# {"status_code": 404, "action": "ignore"} {"status_code": 404, "action": "ignore"}
# ] ]
# }, },
# "write_disposition": "append" "write_disposition": "append"
# } }
# for comm_code in psd_commodity_codes for comm_code in psd_commodity_codes
# for year in years for year in years
#], ],
] ]
}) })
# Initialize and run the pipeline # Initialize and run the pipeline
pipeline = dlt.pipeline( pipeline = dlt.pipeline(
pipeline_name="usda_fas_open_data", pipeline_name="usda_fas_open_data",