deemanone/beanflows
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

uncomment code

Browse Source
This commit is contained in:
Deeman
2025-05-08 18:02:45 +02:00
parent d84e39fd9f
commit 10d9424ff1
1 changed files with 60 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
extract/dlt_script.py
View File

@@ -112,72 +112,72 @@ def run_pipeline():
{"name": "psd_commodity_attributes", "endpoint": {"path": "/api/psd/commodityAttributes"}},
# Dynamic ESR exports resources for each commodity and year
#*[
# {
# "name": f"esr_exports_commodity_{comm_code}_year_{year}",
# "endpoint": {
# "path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}",
# "response_actions": [
# {"status_code": 404, "action": "ignore"}
# ]
# },
# "primary_key": ["commodityCode", "countryCode", "weekEnding"],
# "write_disposition": "append"
# }
# for comm_code in esr_commodity_codes
# for year in years
#],
*[
{
"name": f"esr_exports_commodity_{comm_code}_year_{year}",
"endpoint": {
"path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}",
"response_actions": [
{"status_code": 404, "action": "ignore"}
]
},
"primary_key": ["commodityCode", "countryCode", "weekEnding"],
"write_disposition": "append"
}
for comm_code in esr_commodity_codes
for year in years
],
# Sample GATS census imports for major trading partners (for all recent years)
# You can expand to more countries as needed
#*[
# {
# "name": f"gats_census_imports_{country_code}_{year}_{month:02d}",
# "endpoint": {
# "path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}",
# "response_actions": [
# {"status_code": 404, "action": "ignore"}
# ]
# },
# "write_disposition": "append"
# }
# for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners
# for year in years
# for month in range(1, 13)
# if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month)
#],
*[
{
"name": f"gats_census_imports_{country_code}_{year}_{month:02d}",
"endpoint": {
"path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}",
"response_actions": [
{"status_code": 404, "action": "ignore"}
]
},
"write_disposition": "append"
}
for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners
for year in years
for month in range(1, 13)
if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month)
],
# Dynamic PSD commodity data for all commodities and years
#*[
# {
# "name": f"psd_commodity_{comm_code}_year_{year}",
# "endpoint": {
# "path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}",
# "response_actions": [
# {"status_code": 404, "action": "ignore"}
# ]
# },
# "write_disposition": "append"
# }
# for comm_code in psd_commodity_codes
# for year in years
#],
*[
{
"name": f"psd_commodity_{comm_code}_year_{year}",
"endpoint": {
"path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}",
"response_actions": [
{"status_code": 404, "action": "ignore"}
]
},
"write_disposition": "append"
}
for comm_code in psd_commodity_codes
for year in years
],
# World-level data for all commodities
#*[
# {
# "name": f"psd_world_commodity_{comm_code}_year_{year}",
# "endpoint": {
# "path": f"/api/psd/commodity/{comm_code}/world/year/{year}",
# "response_actions": [
# {"status_code": 404, "action": "ignore"}
# ]
# },
# "write_disposition": "append"
# }
# for comm_code in psd_commodity_codes
# for year in years
#],
*[
{
"name": f"psd_world_commodity_{comm_code}_year_{year}",
"endpoint": {
"path": f"/api/psd/commodity/{comm_code}/world/year/{year}",
"response_actions": [
{"status_code": 404, "action": "ignore"}
]
},
"write_disposition": "append"
}
for comm_code in psd_commodity_codes
for year in years
],
]
})