fix(pipeline): stg_playtomic_availability glob reads all files, filters by date range
The @start_ds in the glob pattern only matched files for the first day of the batch, so incremental restates only loaded 1 day of data. Changed to wildcard glob with explicit BETWEEN @start_ds AND @end_ds filter on the date column. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -34,7 +34,7 @@ all_jsonl AS (
|
|||||||
tenant_id,
|
tenant_id,
|
||||||
slots AS slots_json
|
slots AS slots_json
|
||||||
FROM read_json(
|
FROM read_json(
|
||||||
@LANDING_DIR || '/playtomic/*/*/availability_' || @start_ds || '*.jsonl.gz',
|
@LANDING_DIR || '/playtomic/*/*/availability_*.jsonl.gz',
|
||||||
format = 'newline_delimited',
|
format = 'newline_delimited',
|
||||||
columns = {
|
columns = {
|
||||||
date: 'VARCHAR',
|
date: 'VARCHAR',
|
||||||
@@ -46,6 +46,7 @@ all_jsonl AS (
|
|||||||
filename = true
|
filename = true
|
||||||
)
|
)
|
||||||
WHERE tenant_id IS NOT NULL
|
WHERE tenant_id IS NOT NULL
|
||||||
|
AND CAST(date AS DATE) BETWEEN @start_ds AND @end_ds
|
||||||
),
|
),
|
||||||
raw_resources AS (
|
raw_resources AS (
|
||||||
SELECT
|
SELECT
|
||||||
|
|||||||
Reference in New Issue
Block a user