From 10d9424ff143ecc91ce637c8676e3523916fe51f Mon Sep 17 00:00:00 2001
From: Deeman <hendriknote@gmail.com>
Date: Thu, 8 May 2025 18:02:45 +0200
Subject: [PATCH] uncomment code

---
 extract/dlt_script.py | 120 +++++++++++++++++++++---------------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/extract/dlt_script.py b/extract/dlt_script.py
index 3cb5d4c..8d8cbf1 100644
--- a/extract/dlt_script.py
+++ b/extract/dlt_script.py
@@ -112,75 +112,75 @@ def run_pipeline():
             {"name": "psd_commodity_attributes", "endpoint": {"path": "/api/psd/commodityAttributes"}},
             
             # Dynamic ESR exports resources for each commodity and year
-            #*[
-            #    {
-            #        "name": f"esr_exports_commodity_{comm_code}_year_{year}",
-            #        "endpoint": {
-            #            "path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}",
-            #            "response_actions": [
-            #                {"status_code": 404, "action": "ignore"}
-            #            ]
-            #        },
-            #        "primary_key": ["commodityCode", "countryCode", "weekEnding"],
-            #        "write_disposition": "append"
-            #    }
-            #    for comm_code in esr_commodity_codes
-            #    for year in years
-            #],
+            *[
+                {
+                    "name": f"esr_exports_commodity_{comm_code}_year_{year}",
+                    "endpoint": {
+                        "path": f"/api/esr/exports/commodityCode/{comm_code}/allCountries/marketYear/{year}",
+                        "response_actions": [
+                            {"status_code": 404, "action": "ignore"}
+                        ]
+                    },
+                    "primary_key": ["commodityCode", "countryCode", "weekEnding"],
+                    "write_disposition": "append"
+                }
+                for comm_code in esr_commodity_codes
+                for year in years
+            ],
             
             # Sample GATS census imports for major trading partners (for all recent years)
             # You can expand to more countries as needed
-            #*[
-            #    {
-            #        "name": f"gats_census_imports_{country_code}_{year}_{month:02d}",
-            #        "endpoint": {
-            #            "path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}",
-            #            "response_actions": [
-            #                {"status_code": 404, "action": "ignore"}
-            #            ]
-            #        },
-            #        "write_disposition": "append"
-            #    }
-            #    for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners
-            #    for year in years
-            #    for month in range(1, 13)
-            #    if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month)
-            #],
+            *[
+                {
+                    "name": f"gats_census_imports_{country_code}_{year}_{month:02d}",
+                    "endpoint": {
+                        "path": f"/api/gats/censusImports/partnerCode/{country_code}/year/{year}/month/{month:02d}",
+                        "response_actions": [
+                            {"status_code": 404, "action": "ignore"}
+                        ]
+                    },
+                    "write_disposition": "append"
+                }
+                for country_code in ['CH', 'CA', 'MX', 'BR', 'EU'] # Major trading partners
+                for year in years
+                for month in range(1, 13)
+                if not (year == datetime.datetime.now().year and month > datetime.datetime.now().month)
+            ],
             
             # Dynamic PSD commodity data for all commodities and years
-            #*[
-            #    {
-            #        "name": f"psd_commodity_{comm_code}_year_{year}",
-            #        "endpoint": {
-            #            "path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}",
-            #            "response_actions": [
-            #                {"status_code": 404, "action": "ignore"}
-            #            ]
-            #        },
-            #        "write_disposition": "append"
-            #    }
-            #    for comm_code in psd_commodity_codes
-            #    for year in years
-            #],
+            *[
+                {
+                    "name": f"psd_commodity_{comm_code}_year_{year}",
+                    "endpoint": {
+                        "path": f"/api/psd/commodity/{comm_code}/country/all/year/{year}",
+                        "response_actions": [
+                            {"status_code": 404, "action": "ignore"}
+                        ]
+                    },
+                    "write_disposition": "append"
+                }
+                for comm_code in psd_commodity_codes
+                for year in years
+            ],
             
             # World-level data for all commodities
-            #*[
-            #    {
-            #        "name": f"psd_world_commodity_{comm_code}_year_{year}",
-            #        "endpoint": {
-            #            "path": f"/api/psd/commodity/{comm_code}/world/year/{year}",
-            #            "response_actions": [
-            #                {"status_code": 404, "action": "ignore"}
-            #            ]
-            #        },
-            #        "write_disposition": "append"
-            #    }
-            #    for comm_code in psd_commodity_codes
-            #    for year in years
-            #],
+            *[
+                {
+                    "name": f"psd_world_commodity_{comm_code}_year_{year}",
+                    "endpoint": {
+                        "path": f"/api/psd/commodity/{comm_code}/world/year/{year}",
+                        "response_actions": [
+                            {"status_code": 404, "action": "ignore"}
+                        ]
+                    },
+                    "write_disposition": "append"
+                }
+                for comm_code in psd_commodity_codes
+                for year in years
+            ],
         ]
     })
-    
+     
     # Initialize and run the pipeline
     pipeline = dlt.pipeline(
         pipeline_name="usda_fas_open_data",