diff --git a/extract/psdonline/pyproject.toml b/extract/psdonline/pyproject.toml index e6e8174..aa85bba 100644 --- a/extract/psdonline/pyproject.toml +++ b/extract/psdonline/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pendulum>=3.1.0", ] [project.scripts] -extract_psd = "psdonline.execute:main" +extract_psd = "psdonline.execute:extract_psd_dataset" [build-system] requires = ["hatchling"] diff --git a/extract/psdonline/src/psdonline/__init__.py b/extract/psdonline/src/psdonline/__init__.py index 6b49590..e69de29 100644 --- a/extract/psdonline/src/psdonline/__init__.py +++ b/extract/psdonline/src/psdonline/__init__.py @@ -1,2 +0,0 @@ -def hello() -> str: - return "Hello from psdonline!" diff --git a/pyproject.toml b/pyproject.toml index 8dc2722..2c3724f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,11 @@ dev = [ [tool.uv.sources] psdonline = {workspace = true } +sqlmesh_materia = {workspace = true } [tool.uv.workspace] members = [ "extract/*", - "load/*", "transform/*", ] diff --git a/transform/sqlmesh_materia/config.yaml b/transform/sqlmesh_materia/config.yaml index feb572b..a65abf0 100644 --- a/transform/sqlmesh_materia/config.yaml +++ b/transform/sqlmesh_materia/config.yaml @@ -1,24 +1,27 @@ # --- Gateway Connection --- gateways: - duckdb: + + dev: connection: # For more information on configuring the connection to your execution engine, visit: # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection # https://sqlmesh.readthedocs.io/en/stable/integrations/engines/duckdb/#connection-options type: duckdb - database: db.db - # concurrent_tasks: 1 - # register_comments: True - # pre_ping: False - # pretty_sql: False - # catalogs: - # extensions: - # connector_config: - # secrets: - # filesystems: - # token: + database: materia_dev_db.db + extensions: + - name: zipfs -default_gateway: duckdb + prod: + connection: + # For more information on configuring the connection to your execution engine, visit: + # https://sqlmesh.readthedocs.io/en/stable/reference/configuration/#connection + # https://sqlmesh.readthedocs.io/en/stable/integrations/engines/duckdb/#connection-options + type: duckdb + database: materia_prod_db.db + extensions: + - name: zipfs + +default_gateway: dev # --- Model Defaults --- # https://sqlmesh.readthedocs.io/en/stable/reference/model_configuration/#model-defaults diff --git a/transform/sqlmesh_materia/db.db b/transform/sqlmesh_materia/db.db index e7635a5..3d18492 100644 Binary files a/transform/sqlmesh_materia/db.db and b/transform/sqlmesh_materia/db.db differ diff --git a/transform/sqlmesh_materia/models/example/full_model.sql b/transform/sqlmesh_materia/models/example/full_model.sql deleted file mode 100644 index 362b049..0000000 --- a/transform/sqlmesh_materia/models/example/full_model.sql +++ /dev/null @@ -1,15 +0,0 @@ -MODEL ( - name sqlmesh_example.full_model, - kind FULL, - cron '@daily', - grain item_id, - audits (assert_positive_order_ids), -); - -SELECT - item_id, - COUNT(DISTINCT id) AS num_orders, -FROM - sqlmesh_example.incremental_model -GROUP BY item_id - \ No newline at end of file diff --git a/transform/sqlmesh_materia/models/example/incremental_model.sql b/transform/sqlmesh_materia/models/example/incremental_model.sql deleted file mode 100644 index d2db527..0000000 --- a/transform/sqlmesh_materia/models/example/incremental_model.sql +++ /dev/null @@ -1,19 +0,0 @@ -MODEL ( - name sqlmesh_example.incremental_model, - kind INCREMENTAL_BY_TIME_RANGE ( - time_column event_date - ), - start '2020-01-01', - cron '@daily', - grain (id, event_date) -); - -SELECT - id, - item_id, - event_date, -FROM - sqlmesh_example.seed_model -WHERE - event_date BETWEEN @start_date AND @end_date - \ No newline at end of file diff --git a/transform/sqlmesh_materia/models/example/seed_model.sql b/transform/sqlmesh_materia/models/example/seed_model.sql deleted file mode 100644 index 192d2df..0000000 --- a/transform/sqlmesh_materia/models/example/seed_model.sql +++ /dev/null @@ -1,13 +0,0 @@ -MODEL ( - name sqlmesh_example.seed_model, - kind SEED ( - path '../seeds/seed_data.csv' - ), - columns ( - id INTEGER, - item_id INTEGER, - event_date DATE - ), - grain (id, event_date) -); - \ No newline at end of file diff --git a/transform/sqlmesh_materia/pyproject.toml b/transform/sqlmesh_materia/pyproject.toml index 0f82dbe..700c81f 100644 --- a/transform/sqlmesh_materia/pyproject.toml +++ b/transform/sqlmesh_materia/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "sqlmesh-materia" +name = "sqlmesh_materia" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -11,8 +11,6 @@ dependencies = [ "sqlmesh>=0.200.0", ] -[project.scripts] -plan = "sqlmesh plan" [build-system] requires = ["hatchling"] diff --git a/transform/sqlmesh_materia/seeds/seed_data.csv b/transform/sqlmesh_materia/seeds/seed_data.csv deleted file mode 100644 index 2e3902e..0000000 --- a/transform/sqlmesh_materia/seeds/seed_data.csv +++ /dev/null @@ -1,8 +0,0 @@ -id,item_id,event_date -1,2,2020-01-01 -2,1,2020-01-01 -3,3,2020-01-03 -4,1,2020-01-04 -5,1,2020-01-05 -6,1,2020-01-06 -7,1,2020-01-07 diff --git a/uv.lock b/uv.lock index 457ee04..6ad8a65 100644 --- a/uv.lock +++ b/uv.lock @@ -1203,7 +1203,7 @@ wheels = [ [[package]] name = "sqlmesh-materia" version = "0.1.0" -source = { editable = "transform/sqlmesh-materia" } +source = { editable = "transform/sqlmesh_materia" } dependencies = [ { name = "sqlmesh" }, ]