From 025dda16c64dc22547d3d002af88af64b78c7bd4 Mon Sep 17 00:00:00 2001 From: Deeman Date: Tue, 7 Oct 2025 22:32:45 +0200 Subject: [PATCH] update dedupe logic -> much faster now --- .../cleaned/cln_psdalldata__commodity_pivoted.sql | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql b/transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql index 1c16d0b..cbe6bdd 100644 --- a/transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql +++ b/transform/sqlmesh_materia/models/cleaned/cln_psdalldata__commodity_pivoted.sql @@ -10,9 +10,9 @@ MODEL ( SELECT max(hkey) as hkey, commodity_code, - commodity_name, + max(commodity_name) as commodity_name, country_code, - country_name, + max(country_name) as country_name, ingest_date, COALESCE(SUM(CASE WHEN attribute_name = 'Production' THEN value END), 0) AS Production, COALESCE(SUM(CASE WHEN attribute_name = 'Imports' THEN value END), 0) AS Imports, @@ -47,11 +47,9 @@ WHERE attribute_name IN ( ) GROUP BY commodity_code, - commodity_name, country_code, - country_name, ingest_date ORDER BY - commodity_name, - country_name, - ingest_date; + commodity_code, + country_code, + ingest_date