feat: allow sourcing most options from config and dbt project yml

demo_duckdb/models/jaffle_shop/main/customers.yml

@@ -10,16 +10,22 @@ models:
           - unique
           - not_null
 
+        data_type: INTEGER
       - name: first_name
         description: Customer's first name. PII.
+        data_type: VARCHAR
       - name: last_name
         description: Customer's last name. PII.
+        data_type: VARCHAR
       - name: first_order
         description: Date (UTC) of a customer's first order
+        data_type: DATE
       - name: most_recent_order
         description: Date (UTC) of a customer's most recent order
+        data_type: DATE
       - name: number_of_orders
         description: Count of the number of orders a customer has placed
+        data_type: BIGINT
       - name: customer_lifetime_value
         data_type: DOUBLE
       - name: customer_average_value

demo_duckdb/models/jaffle_shop/main/orders.yml

@@ -9,6 +9,7 @@ models:
           - unique
           - not_null
         description: This is a unique identifier for an order
+        data_type: INTEGER
       - name: customer_id
         description: Foreign key to the customers table
         tests:
@@ -17,34 +18,42 @@ models:
               to: ref('customers')
               field: customer_id
 
+        data_type: INTEGER
       - name: order_date
         description: Date (UTC) that the order was placed
+        data_type: DATE
       - name: status
         tests:
           - accepted_values:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
 
         description: '{{ doc("orders_status") }}'
+        data_type: VARCHAR
       - name: credit_card_amount
         description: Amount of the order (AUD) paid for by credit card
         tests:
           - not_null
 
+        data_type: DOUBLE
       - name: coupon_amount
         description: Amount of the order (AUD) paid for by coupon
         tests:
           - not_null
 
+        data_type: DOUBLE
       - name: bank_transfer_amount
         description: Amount of the order (AUD) paid for by bank transfer
         tests:
           - not_null
 
+        data_type: DOUBLE
       - name: gift_card_amount
         description: Amount of the order (AUD) paid for by gift card
         tests:
           - not_null
+        data_type: DOUBLE
       - name: amount
         description: Total amount (AUD) of the order
         tests:
           - not_null
+        data_type: DOUBLE

demo_duckdb/models/staging/jaffle_shop/main/stg_payments.yml

@@ -6,13 +6,13 @@ models:
         tests:
           - unique
           - not_null
-        data_type: INTEGER
+        data_type: integer
       - name: order_id
-        data_type: INTEGER
+        data_type: integer
       - name: payment_method
         tests:
           - accepted_values:
-              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
-        data_type: VARCHAR
+              values: ["credit_card", "coupon", "bank_transfer", "gift_card"]
+        data_type: varchar
       - name: amount
-        data_type: DOUBLE
+        data_type: double

demo_duckdb/models/staging/stg_payments.sql

@@ -1,3 +1,5 @@
+{{ config(dbt_osmosis_output_to_lower = true) }}
+
 with source as (
 
     {#-

demo_duckdb/seeds/jaffle_shop/main/raw_customers.yml

@@ -1,6 +1,8 @@
 version: 2
 seeds:
   - name: raw_customers
+    meta:
+      foo: bar
     columns:
       - name: id
         data_type: INTEGER

demo_duckdb/seeds/jaffle_shop/main/raw_orders.yml

@@ -10,3 +10,4 @@ seeds:
         data_type: DATE
       - name: status
         description: '{{ doc("orders_status") }}'
+        data_type: VARCHAR

demo_sqlite/models/orders.yml

@@ -1,54 +1,58 @@
 version: 2
 models:
-- name: orders
-  description: This table has basic information about orders, as well as some derived
-    facts based on payments
-
-  columns:
-  - name: order_id
-    tests:
-    - unique
-    - not_null
-    description: This is a unique identifier for an order
-
-  - name: customer_id
-    description: Foreign key to the customers table
-    tests:
-    - not_null
-    - relationships:
-        to: ref('customers')
-        field: customer_id
-
-  - name: order_date
-    description: Date (UTC) that the order was placed
-
-  - name: status
-    description: '{{ doc("orders_status") }}'
-    tests:
-    - accepted_values:
-        values: [placed, shipped, completed, return_pending, returned]
-
-  - name: amount
-    description: Total amount (AUD) of the order
-    tests:
-    - not_null
-
-  - name: credit_card_amount
-    description: Amount of the order (AUD) paid for by credit card
-    tests:
-    - not_null
-
-  - name: coupon_amount
-    description: Amount of the order (AUD) paid for by coupon
-    tests:
-    - not_null
-
-  - name: bank_transfer_amount
-    description: Amount of the order (AUD) paid for by bank transfer
-    tests:
-    - not_null
-
-  - name: gift_card_amount
-    description: Amount of the order (AUD) paid for by gift card
-    tests:
-    - not_null
+  - name: orders
+    description:
+      This table has basic information about orders, as well as some derived
+      facts based on payments
+
+    meta:
+      output-to-lower: true
+
+    columns:
+      - name: order_id
+        tests:
+          - unique
+          - not_null
+        description: This is a unique identifier for an order
+
+      - name: customer_id
+        description: Foreign key to the customers table
+        tests:
+          - not_null
+          - relationships:
+              to: ref('customers')
+              field: customer_id
+
+      - name: order_date
+        description: Date (UTC) that the order was placed
+
+      - name: status
+        description: '{{ doc("orders_status") }}'
+        tests:
+          - accepted_values:
+              values: [placed, shipped, completed, return_pending, returned]
+
+      - name: amount
+        description: Total amount (AUD) of the order
+        tests:
+          - not_null
+
+      - name: credit_card_amount
+        description: Amount of the order (AUD) paid for by credit card
+        tests:
+          - not_null
+
+      - name: coupon_amount
+        description: Amount of the order (AUD) paid for by coupon
+        tests:
+          - not_null
+
+      - name: bank_transfer_amount
+        description: Amount of the order (AUD) paid for by bank transfer
+        tests:
+          - not_null
+
+      - name: gift_card_amount
+        description: Amount of the order (AUD) paid for by gift card
+        tests:
+          - not_null

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dbt-osmosis"
-version = "1.0.2"
+version = "1.1.0"
 description = "A dbt utility for managing YAML to make developing with dbt more delightful."
 readme = "README.md"
 license = { text = "Apache-2.0" }

src/dbt_osmosis/cli/main.py

@@ -27,6 +27,7 @@
     remove_columns_not_in_database,
     sort_columns_as_in_database,
     sync_node_to_yaml,
+    synchronize_data_types,
 )
 
 T = t.TypeVar("T")
@@ -184,12 +185,12 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
     help="Add inheritance for the specified keys. IE policy_tags",
 )
 @click.option(
-    "--numeric-precision",
+    "--numeric-precision-and-scale",
     is_flag=True,
     help="Numeric types will have precision and scale, e.g. Number(38, 8).",
 )
 @click.option(
-    "--char-length",
+    "--string-length",
     is_flag=True,
     help="Character types will have length, e.g. Varchar(128).",
 )
@@ -229,7 +230,9 @@ def refactor(
     )
     context = YamlRefactorContext(
         project=create_dbt_project_context(settings),
-        settings=YamlRefactorSettings(**kwargs, create_catalog_if_not_exists=False),
+        settings=YamlRefactorSettings(
+            **{k: v for k, v in kwargs.items() if v is not None}, create_catalog_if_not_exists=False
+        ),
     )
     if vars:
         settings.vars = context.yaml_handler.load(io.StringIO(vars))  # pyright: ignore[reportUnknownMemberType]
@@ -242,6 +245,7 @@ def refactor(
     remove_columns_not_in_database(context=context)
     inherit_upstream_column_knowledge(context=context)
     sort_columns_as_in_database(context=context)
+    synchronize_data_types(context=context)
     sync_node_to_yaml(context=context)
     commit_yamls(context=context)
 
@@ -282,7 +286,9 @@ def organize(
     )
     context = YamlRefactorContext(
         project=create_dbt_project_context(settings),
-        settings=YamlRefactorSettings(**kwargs, create_catalog_if_not_exists=False),
+        settings=YamlRefactorSettings(
+            **{k: v for k, v in kwargs.items() if v is not None}, create_catalog_if_not_exists=False
+        ),
     )
     if vars:
         settings.vars = context.yaml_handler.load(io.StringIO(vars))  # pyright: ignore[reportUnknownMemberType]
@@ -347,12 +353,12 @@ def organize(
     help="Add inheritance for the specified keys. IE policy_tags",
 )
 @click.option(
-    "--numeric-precision",
+    "--numeric-precision-and-scale",
     is_flag=True,
     help="Numeric types will have precision and scale, e.g. Number(38, 8).",
 )
 @click.option(
-    "--char-length",
+    "--string-length",
     is_flag=True,
     help="Character types will have length, e.g. Varchar(128).",
 )
@@ -390,7 +396,9 @@ def document(
     )
     context = YamlRefactorContext(
         project=create_dbt_project_context(settings),
-        settings=YamlRefactorSettings(**kwargs, create_catalog_if_not_exists=False),
+        settings=YamlRefactorSettings(
+            **{k: v for k, v in kwargs.items() if v is not None}, create_catalog_if_not_exists=False
+        ),
     )
     if vars:
         settings.vars = context.yaml_handler.load(io.StringIO(vars))  # pyright: ignore[reportUnknownMemberType]