Merge pull request #57 from fivetran/feature/add-employees-model-time…

…stamp-fixes-common-fields

[Feature] Add employees model, timestamp fixes, add common fields

.buildkite/scripts/run_models.sh

@@ -22,15 +22,15 @@ dbt seed --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --t
 dbt compile --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
 dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw, netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false}' --target "$db" --full-refresh
 dbt test --vars '{netsuite_schema: netsuite_source_integrations_tests_sqlw}' --target "$db"
 
 else
 
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false}' --target "$db" --full-refresh
+dbt run --vars '{netsuite2__multibook_accounting_enabled: false, netsuite2__using_exchange_rate: false, netsuite2__using_vendor_categories: false, netsuite2__using_jobs: false, netsuite2__using_employees: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 fi
 

CHANGELOG.md

@@ -1,3 +1,46 @@
+# dbt_netsuite_source v0.11.0
+[PR #57](https://github.com/fivetran/dbt_netsuite_source/pull/57) includes the following update: 
+
+## Breaking Changes 
+- Casted specific timestamp fields across all staging models as dates where the Netsuite UI does not perform timezone conversion. Keeping these fields as type timestamp causes issues in reporting tools that perform automatic timezone conversion. 
+  - As this will change the datatype of the underlying fields, this will require a  `--full-refresh` for downstream incremental models.
+- Existing fields that were converted from timestamp to date in the following `stg_netsuite2__*` models:
+
+| **Table**   | **Source Field Name** | **Field Alias** | 
+| -------- | ------- | ------- | 
+| accounting_periods | startdate | starting_at | 
+| accounting_periods | enddate | ending_at |
+| customers | firstorderdate | date_first_order_at |
+| transactions | closedate | closed_at |
+| transactions | duedate | due_date_at |
+| transactions | trandate | transaction_date |
+
+- Adds additional commonly used fields within the `stg_netsuite2__*` models.
+
+| **Table**  | **Source Field Name** | **Field Alias** | 
+| -------- | ------- | ------- | 
+| accounts | accountsearchdisplaynamecopy | display_name | 
+| customers | altname | alt_name |
+| subsidiaries | iselimination | is_elimination |
+| transaction_accounting_lines | exchangerate | exchange_rate |
+| transaction_lines | eliminate | is_eliminate |
+| transaction_lines | netamount | netamount |
+| transactions | reversal | reversal_transaction_id |
+| transactions | reversaldate | reversal_date |
+| transactions | reversaldefer | is_reversal_defer | 
+
+> **IMPORTANT**: Nearly all of the affected models have pass-through functionality. If you have already been using passthrough column variables to include the above newly added fields (without aliases), you **MUST** remove the fields from your passthrough variable configuration in order to avoid duplicate column errors.
+
+## Feature Updates
+- Introduced the `stg_netsuite2__employees` model to bring in data from the `employee` source table. 
+  - Since this model is only used by a subset of customers, we've introduced the variable `netsuite2__using_employees` to allow users who don't utilize the `employee` table in Netsuite2 the ability to disable that functionality within your `dbt_project.yml`. This value is set to true by default. [Instructions are available in the README for how to disable this variable](https://github.com/fivetran/dbt_netsuite_source/?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only). 
+
+## Under the Hood
+- Created new seed data in `integration_tests` to support the new `stg_netsuite2__employees` model, as well as the new fields introduced into the new Netsuite2 staging models.
+
+## Contributors
+- [@jmongerlyra](https://github.com/jmongerlyra) ([PR #54](https://github.com/fivetran/dbt_netsuite_source/pull/54))
+
 # dbt_netsuite_source v0.10.1
 [PR #51](https://github.com/fivetran/dbt_netsuite_source/pull/51) includes the following update: 
 

README.md

@@ -59,6 +59,7 @@ To use this dbt package, you must have At least either one Fivetran **Netsuite**
 - customer
 - classification
 - department
+- employee
 - entity
 - entityaddress
 - item
@@ -89,7 +90,7 @@ If you  are **not** using the [Netsuite transformation package](https://github.c
 ```yaml
 packages:
   - package: fivetran/netsuite_source
-    version: [">=0.10.0", "<0.11.0"]
+    version: [">=0.11.0", "<0.12.0"]
 ```
 
 ### Step 3: Define Netsuite.com or Netsuite2 Source
@@ -116,8 +117,10 @@ vars:
     netsuite2__using_exchange_rate: false #True by default. Disable `exchange_rate` if you don't utilize exchange rates. If you set this variable to false and are using the `netsuite` transformation package, ensure it is scoped globally so that the `netsuite` package can access it as well.  
     netsuite2__using_vendor_categories: false # True by default. Disable `vendorcategory` if you don't categorize your vendors
     netsuite2__using_jobs: false # True by default. Disable `job` if you don't use jobs
+    netsuite2__using_employees: false # True by default. Disable `employee` if you don't use employees.
 
 ```
+
 > **Note**: The Netsuite dbt package currently only supports disabling transforms of [Multi-Book Accounting](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/book_3831565332.html) related tables (`accountingbooksubsidiary` and `accountingbook`), the `vendorcategory` source table, and the `job` source table. Please create an issue to request additional tables and/or [features](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/bridgehead_N233872.html) to exclude.
 >
 > To determine if a table or field is activated by a feature, access the [Records Catalog](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/article_159367781370.html).

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.10.1'
+version: '0.11.0'
 
 models:
   netsuite_source:
@@ -41,6 +41,7 @@ vars:
     netsuite2_currencies: "{{ source('netsuite2','currency') }}"
     netsuite2_customers: "{{ source('netsuite2','customer') }}"
     netsuite2_departments: "{{ source('netsuite2','department') }}"
+    netsuite2_employees: "{{ source('netsuite2','employee') }}"
     netsuite2_entities: "{{ source('netsuite2','entity') }}"
     netsuite2_items: "{{ source('netsuite2','item') }}"
     netsuite2_jobs: "{{ source('netsuite2','job') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_6
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_6
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_source_integration_tests_3
+      schema: netsuite_source_integration_tests_6
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,13 +1,13 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.10.1'
+version: '0.11.0'
 
 profile: 'integration_tests'
 config-version: 2
 
 vars:
   netsuite_source:
-    netsuite_schema: netsuite_source_integration_tests_3
+    netsuite_schema: netsuite_source_integration_tests_6
     netsuite_data_model_override: netsuite
     netsuite_accounting_books_identifier: "netsuite_accounting_books_data"
     netsuite_accounting_periods_identifier: "netsuite_accounting_periods_data"
@@ -34,6 +34,7 @@ vars:
     netsuite2_currency_identifier: "netsuite2_currency_data"
     netsuite2_customer_identifier: "netsuite2_customer_data"
     netsuite2_department_identifier: "netsuite2_department_data"
+    netsuite2_employee_identifier: "netsuite2_employee_data"
     netsuite2_entity_identifier: "netsuite2_entities_data"
     netsuite2_item_identifier: "netsuite2_item_data"
     netsuite2_job_identifier: "netsuite2_job_data"

integration_tests/seeds/netsuite2_customer_data.csv

@@ -1,5 +1,5 @@
 id,_fivetran_deleted,_fivetran_synced,accountnumber,alcoholrecipienttype,altemail,altname,altphone,assignedwebsite,balancesearch,billingratecard,billingschedule,billingtransactiontype,billpay,buyingreason,buyingtimeframe,campaignevent,category,clickstream,comments,companyname,consolbalancesearch,consoldaysoverduesearch,consoloverduebalancesearch,consolunbilledorderssearch,contact,contactlist,creditholdoverride,creditlimit,currency,custentity1,custentity10,custentity12,custentity13,custentity14,custentity15,custentity16,custentity18,custentity21,custentity4,custentity5,custentity6,custentity7,custentity8,custentity9,custentity_2663_customer_refund,custentity_2663_direct_debit,custentity_cust_priority,custentity_fmt_cur_sales_order,custentity_fmt_cust_credit_on_hold,custentity_fmt_customer_credit_on_hold,custentity_lead_category,custentity_ppob,custentity_supervisor,date_deleted,datecreated,defaultbankaccount,defaultbillingaddress,defaultorderpriority,defaultshippingaddress,duplicate,email,emailpreference,emailtransactions,enddate,entityid,entitynumber,entitystatus,entitytitle,estimatedbudget,externalid,fax,faxtransactions,firstname,firstorderdate,firstsaledate,firstvisit,giveaccess,globalsubscriptionstatus,homephone,isbudgetapproved,isinactive,isperson,keywords,language,lastmodifieddate,lastname,lastorderdate,lastpagevisited,lastsaledate,lastvisit,leadsource,middlename,mobilephone,negativenumberformat,numberformat,oncredithold,overduebalancesearch,parent,partner,phone,prefccprocessor,pricelevel,printoncheckas,printtransactions,probability,receivablesaccount,referrer,reminderdays,resalenumber,salesreadiness,salesrep,salutation,searchstage,shipcomplete,shippingcarrier,shippingitem,sourcewebsite,startdate,terms,territory,thirdpartyacct,thirdpartycarrier,thirdpartycountry,thirdpartyzipcode,title,unbilledorderssearch,url,visits,weblead,dateclosed
-1409,FALSE,2022-05-27 13:17:39,,,,Joe Shmoe,,,0,,,,F,,,,,,,,0,0,0,0,,,AUTO,,1,,,,,,,,F,,,,,,,,,,50,,F,,2,,,,2017-05-27 8:56:10,,,,,T,[email protected],DEFAULT,F,,CCS,,6,Joe Shmoe,,,,F,Joe,,,,F,1,,F,F,T,,en_US,2017-05-09 6:07:35,Li,,,,,,,,,,F,0,,,,,,,F,0,-10,,,,,1008,,Lead,F,nonups,,,,2,,,,,,,0,,,F,2017-05-27 0:00:00
+1409,FALSE,2022-05-27 13:17:39,,,,Joe Shmoe,,,0,,,,F,,,,,,,,0,0,0,0,,,AUTO,,1,,,,,,,,F,,,,,,,,,,50,,F,,2,,,,2017-05-27 8:56:10,,,,,T,[email protected],DEFAULT,F,,CCS,,6,Joe Shmoe,,,,F,Joe,2017-05-07 0:00:00,,,F,1,,F,F,T,,en_US,2017-05-09 6:07:35,Li,,,,,,,,,,F,0,,,,,,,F,0,-10,,,,,1008,,Lead,F,nonups,,,,2,,,,,,,0,,,F,2017-05-27 0:00:00
 1408,FALSE,2022-05-27 13:17:39,,,,Fake Name,,,0,,,,F,,,,,,,,0,0,0,0,,,AUTO,,1,,,,,,,,F,,,,,,,,,,50,,F,,2,,,,2017-05-27 8:52:49,,,,,T,[email protected],DEFAULT,F,,CCS,,6,Fake Name,,,,F,Fake,,,,F,1,,F,F,T,,en_US,2017-05-09 6:07:33,Li,,,,,,,,,,F,0,,,,,,,F,0,-10,,,,,1008,,Lead,F,nonups,,,,2,,,,,,,0,,,F,2017-05-27 0:00:00
 1388,FALSE,2022-05-27 13:17:39,,,,Jess is Best,,,0,,,,F,,,,,Cat > Couch,,,0,0,0,0,,,AUTO,,1,,,,,,,,F,,,3,,,,,,,50,,,,,,,,2017-05-10 19:14:00,,,,,T,[email protected],DEFAULT,F,,CCS,,6,Jess is Best,,,,F,Jess,,,2017-05-10 10:07:29,T,2,,F,F,T,,en_US,2017-05-09 6:07:26,Best,,,,2017-05-23 8:15:26,,,,,,F,0,,,,,,,F,0,-10,,,,,,,Lead,F,nonups,,,,2,8,,,,,,0,,11,T,2017-05-10 0:00:00
 1440,FALSE,2022-05-27 13:17:39,,,,Cedar Cat,,,0,,,,F,,,,,Cat,,CCS,0,0,0,0,,,AUTO,,1,,,,,,,,F,,,,,,,,,,50,,F,F,,,,,2017-05-11 21:46:11,,,,,T,[email protected],DEFAULT,F,,CCS,7,6,Cedar Cat,,,,F,,,,2017-05-11 14:45:51,F,1,,F,F,F,,en_US,2017-05-09 6:07:47,,,,,2017-05-11 15:17:17,100237,,,,,F,0,,,,,,,F,0,-10,,,,,,,Lead,F,nonups,,,,2,,,,,,,0,,3,F,2017-05-11 0:00:00

integration_tests/seeds/netsuite2_employee_data.csv

@@ -0,0 +1,4 @@
+_fivetran_synced,id,entityid,firstname,lastname,department,subsidiary,email,supervisor,approvallimit,expenselimit,purchaseorderapprovallimit,purchaseorderlimit,currency,isinactive
+2022-05-27 10:45:22,523532,[email protected],Sponge,Bob,113,1,[email protected],431513,,,,0,1,F
+2022-05-27 10:45:22,513513,[email protected],Scooby,Doo,341,1,[email protected],,,,,0,1,F
+2022-05-27 10:45:22,409409,[email protected],Darth,Vader,351,1,[email protected],,,,,0,1,T

integration_tests/seeds/netsuite2_transaction_data.csv

@@ -1,4 +1,4 @@
-_fivetran_synced,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal
-2022-05-27 13:41:13,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F
-2022-05-27 13:41:13,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F
-2022-05-27 13:41:13,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F
+_fivetran_synced,id,transactionnumber,type,memo,trandate,status,createddate,duedate,closedate,currency,entity,postingperiod,posting,isreversal,reversal,reversaldate,reversaldefer
+2022-05-27 13:41:13,5914,,VendBill,,2017-05-07 0:00:00,A,2017-05-07 10:34:43,2017-05-05 0:00:00,,1,1570,,T,F,325251,2023-08-01 0:00:00,F
+2022-05-27 13:41:13,3,,VendBill,,2015-09-10 0:00:00,B,2017-05-12 3:41:25,2015-10-09 0:00:00,2015-10-05 0:00:00,1,36,251,T,F,325252,2023-08-02 0:00:00,F
+2022-05-27 13:41:13,302,,VendBill,ZZZZZ,2016-03-20 0:00:00,B,2017-04-25 3:41:25,2016-04-19 0:00:00,2017-05-07 0:00:00,1,5,261,T,F,325323,2023-08-03 0:00:00,F