Merge pull request #4 from fivetran/MagicBot/dbt-utils-cross-db-migra…

…tion

Updates for dbt-utils to dbt-core cross-db macro migration

.buildkite/hooks/pre-command

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Export secrets for Docker containers.
+# Restrict exposing secrets only to the steps that need them
+export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -0,0 +1,73 @@
+steps:
+  - label: ":postgres: Run Tests - Postgres"
+    key: "run-dbt-postgres"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_POSTGRES_DBT_DBNAME"
+            - "CI_POSTGRES_DBT_HOST"
+            - "CI_POSTGRES_DBT_PASS"
+            - "CI_POSTGRES_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh postgres 
+
+  - label: ":snowflake-db: Run Tests - Snowflake"
+    key: "run_dbt_snowflake"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_SNOWFLAKE_DBT_ACCOUNT"
+            - "CI_SNOWFLAKE_DBT_DATABASE"
+            - "CI_SNOWFLAKE_DBT_PASS"
+            - "CI_SNOWFLAKE_DBT_ROLE"
+            - "CI_SNOWFLAKE_DBT_USER"
+            - "CI_SNOWFLAKE_DBT_WAREHOUSE"
+    commands: |
+      bash .buildkite/scripts/run_models.sh snowflake
+
+  - label: ":gcloud: Run Tests - BigQuery"
+    key: "run_dbt_bigquery"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "GCLOUD_SERVICE_KEY"
+    commands: |
+      bash .buildkite/scripts/run_models.sh bigquery
+
+  - label: ":amazon-redshift: Run Tests - Redshift"
+    key: "run_dbt_redshift"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_REDSHIFT_DBT_DBNAME"
+            - "CI_REDSHIFT_DBT_HOST"
+            - "CI_REDSHIFT_DBT_PASS"
+            - "CI_REDSHIFT_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":bricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

.buildkite/scripts/run_models.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+apt-get update
+apt-get install libsasl2-dev
+
+python3 -m venv venv
+. venv/bin/activate
+pip install --upgrade pip setuptools
+pip install -r integration_tests/requirements.txt
+mkdir -p ~/.dbt
+cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+db=$1
+echo `pwd`
+cd integration_tests
+dbt deps
+dbt seed --target "$db" --full-refresh
+dbt run --target "$db" --full-refresh
+dbt test --target "$db"

.github/pull_request_template.md

@@ -1,4 +1,3 @@
-Pull Request
 **Are you a current Fivetran customer?** 
 <!--- Please tell us your name, title and company -->
 
@@ -29,9 +28,9 @@ Pull Request
 
 **How did you test the PR changes?** 
 <!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] CircleCi). --> 
+<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] CircleCi <!--- CircleCi testing is only applicable to Fivetran employees. --> 
+- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
 - [ ] Local (please provide additional testing details below)
 
 **Select which warehouse(s) were used to test the PR**

CHANGELOG.md

@@ -1,3 +1,42 @@
+# dbt_social_media_reporting v0.2.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #4](https://github.com/fivetran/dbt_social_media_reporting/pull/4) includes the following breaking changes:
+- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
+    - `any_value`
+    - `bool_or`
+    - `cast_bool_to_text`
+    - `concat`
+    - `date_trunc`
+    - `dateadd`
+    - `datediff`
+    - `escape_single_quotes`
+    - `except`
+    - `hash`
+    - `intersect`
+    - `last_day`
+    - `length`
+    - `listagg`
+    - `position`
+    - `replace`
+    - `right`
+    - `safe_cast`
+    - `split_part`
+    - `string_literal`
+    - `type_bigint`
+    - `type_float`
+    - `type_int`
+    - `type_numeric`
+    - `type_string`
+    - `type_timestamp`
+    - `array_append`
+    - `array_concat`
+    - `array_construct`
+- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
+    - `dbt.current_timestamp_backcompat`
+    - `dbt.current_timestamp_in_utc_backcompat`
+- Dependencies on `fivetran/fivetran_utils` have been upgraded, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+
 # dbt_social_media_reporting v0.1.0
 
 The original release! This dbt package aggregates and models data from multiple Fivetran social media connectors. The package standardizes the schemas from the various social media connectors and creates a single reporting model for all activity. It enables you to analyze your post performance via clicks, impressions, shares, likes and comments.

README.md

@@ -1,4 +1,15 @@
-[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 
+<p align="center">
+    <a alt="License"
+        href="https://github.com/fivetran/dbt_social_media_reporting/blob/main/LICENSE">
+        <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
+    <a alt="dbt-core">
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_<2.0.0-orange.svg" /></a>
+    <a alt="Maintained?">
+        <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
+    <a alt="PRs">
+        <img src="https://img.shields.io/badge/Contributions-welcome-blueviolet" /></a>
+</p>
+
 # Social Media Reporting
 
 This dbt package aggregates and models data from multiple Fivetran social media connectors. The package standardizes the schemas from the various social media connectors and creates a single reporting model for all activity. It enables you to analyze your post performance by clicks, impressions, shares, likes, and comments.
@@ -26,7 +37,7 @@ Include in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/social_media_reporting
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 ```
 
 ## Package Maintenance
@@ -149,6 +160,45 @@ dispatch:
     search_order: ['spark_utils', 'dbt_utils']
 ```
 
+# 🔍 Does this package have dependencies?
+This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site.
+> IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts.
+```yml
+packages:
+    - package: fivetran/facebook_pages
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/facebook_pages_source
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/instagram_business
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/instagram_business_source
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/twitter_organic
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/twitter_organic_source
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/linkedin_pages
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/linkedin_pages_source
+      version: [">=0.2.0", "<0.3.0"]
+
+    - package: fivetran/fivetran_utils
+      version: [">=0.4.0", "<0.5.0"]
+
+    - package: dbt-labs/dbt_utils
+      version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
+```
+
 ## Contributions
 
 Additional contributions to this package are very welcome! Please create issues

dbt_project.yml

@@ -1,21 +1,17 @@
 name: 'social_media_reporting'
-version: '0.1.0'
+version: '0.2.0'
 config-version: 2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
+require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
   twitter_posts_report: "{{ ref('twitter_organic__tweets') }}"
   facebook_posts_report: "{{ ref('facebook_pages__posts_report') }}"
   linkedin_posts_report: "{{ ref('linkedin_pages__posts') }}"
   instagram_posts_report: "{{ ref('instagram_business__posts') }}"
-
   social_media_rollup__twitter_enabled: true
   social_media_rollup__facebook_enabled: true
   social_media_rollup__linkedin_enabled: true
   social_media_rollup__instagram_enabled: true
-
 models:
   social_media_reporting:
     +schema: social_media_reporting
     +materialized: table
-