From 55f179ba54f79c3cbcb786334dd00b1b50e4e9bc Mon Sep 17 00:00:00 2001 From: Deeman Date: Wed, 25 Feb 2026 09:56:05 +0100 Subject: [PATCH] fix(transform): increase geonames object size limit and remove stale column ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - stg_population_geonames: add maximum_object_size=40MB to read_json() call; geonames cities_global.json.gz is ~30MB, exceeding DuckDB's 16MB default - dim_locations: remove stale 'population_year AS population_year' column ref; stg_population_geonames has ref_year, not population_year — caused BinderException Co-Authored-By: Claude Sonnet 4.6 --- .../sqlmesh_padelnomics/models/foundation/dim_locations.sql | 1 - .../models/staging/stg_population_geonames.sql | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/sqlmesh_padelnomics/models/foundation/dim_locations.sql b/transform/sqlmesh_padelnomics/models/foundation/dim_locations.sql index c6177d3..13b4a0c 100644 --- a/transform/sqlmesh_padelnomics/models/foundation/dim_locations.sql +++ b/transform/sqlmesh_padelnomics/models/foundation/dim_locations.sql @@ -35,7 +35,6 @@ locations AS ( admin1_code, admin2_code, population, - population_year AS population_year, ref_year FROM staging.stg_population_geonames WHERE lat IS NOT NULL AND lon IS NOT NULL diff --git a/transform/sqlmesh_padelnomics/models/staging/stg_population_geonames.sql b/transform/sqlmesh_padelnomics/models/staging/stg_population_geonames.sql index 6f40d10..699c90e 100644 --- a/transform/sqlmesh_padelnomics/models/staging/stg_population_geonames.sql +++ b/transform/sqlmesh_padelnomics/models/staging/stg_population_geonames.sql @@ -28,7 +28,8 @@ WITH parsed AS ( SELECT UNNEST(rows) AS row FROM read_json( @LANDING_DIR || '/geonames/*/*/cities_global.json.gz', - auto_detect = true + auto_detect = true, + maximum_object_size = 40000000 ) ) WHERE (row ->> 'geoname_id') IS NOT NULL