From 11f9a3e3842db6b747352f746bb0ebac42c1f2bf Mon Sep 17 00:00:00 2001 From: David Date: Thu, 12 Oct 2023 17:31:18 +0200 Subject: [PATCH] added example --- GETTING_STARTED.md | 138 ++++++++++++++++++ README.md | 74 +++++++--- {docker => examples/jaffle_shop}/Dockerfile | 0 .../jaffle_shop}/docker-compose.yml | 17 ++- .../jaffle_shop/jaffle_shop/dbt_project.yml | 26 ++++ .../jaffle_shop/models/customers.sql | 69 +++++++++ .../jaffle_shop/jaffle_shop/models/docs.md | 14 ++ .../jaffle_shop/jaffle_shop/models/orders.sql | 56 +++++++ .../jaffle_shop/models/overview.md | 11 ++ .../jaffle_shop/jaffle_shop/models/schema.yml | 82 +++++++++++ .../jaffle_shop/models/staging/schema.yml | 31 ++++ .../models/staging/stg_customers.sql | 22 +++ .../jaffle_shop/models/staging/stg_orders.sql | 23 +++ .../models/staging/stg_payments.sql | 25 ++++ .../jaffle_shop/my_prefect_dbt_flow.py | 26 ++++ examples/jaffle_shop/jaffle_shop/profiles.yml | 12 ++ .../jaffle_shop/seeds/raw_customers.csv | 101 +++++++++++++ .../jaffle_shop/seeds/raw_orders.csv | 100 +++++++++++++ .../jaffle_shop/seeds/raw_payments.csv | 114 +++++++++++++++ .../jaffle_shop}/requirements.txt | 0 20 files changed, 920 insertions(+), 21 deletions(-) create mode 100644 GETTING_STARTED.md rename {docker => examples/jaffle_shop}/Dockerfile (100%) rename {docker => examples/jaffle_shop}/docker-compose.yml (68%) create mode 100644 examples/jaffle_shop/jaffle_shop/dbt_project.yml create mode 100644 examples/jaffle_shop/jaffle_shop/models/customers.sql create mode 100644 examples/jaffle_shop/jaffle_shop/models/docs.md create mode 100644 examples/jaffle_shop/jaffle_shop/models/orders.sql create mode 100644 examples/jaffle_shop/jaffle_shop/models/overview.md create mode 100644 examples/jaffle_shop/jaffle_shop/models/schema.yml create mode 100644 examples/jaffle_shop/jaffle_shop/models/staging/schema.yml create mode 100644 examples/jaffle_shop/jaffle_shop/models/staging/stg_customers.sql create mode 100644 examples/jaffle_shop/jaffle_shop/models/staging/stg_orders.sql create mode 100644 examples/jaffle_shop/jaffle_shop/models/staging/stg_payments.sql create mode 100644 examples/jaffle_shop/jaffle_shop/my_prefect_dbt_flow.py create mode 100644 examples/jaffle_shop/jaffle_shop/profiles.yml create mode 100644 examples/jaffle_shop/jaffle_shop/seeds/raw_customers.csv create mode 100644 examples/jaffle_shop/jaffle_shop/seeds/raw_orders.csv create mode 100644 examples/jaffle_shop/jaffle_shop/seeds/raw_payments.csv rename {docker => examples/jaffle_shop}/requirements.txt (100%) diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md new file mode 100644 index 0000000..015264a --- /dev/null +++ b/GETTING_STARTED.md @@ -0,0 +1,138 @@ +# Getting started guide + +The Prefect-dbt-flow library allows you to seamlessly integrate dbt workflows into Prefect. This usage guide will walk you through the steps required to create and manage a Prefect flow for your dbt project. + +## Example guide +This guide will walk you through setting up and running a sample Prefect-dbt-flow using Docker Compose. Follow these steps to get started: + +### 1. Clone this repository +Clone the Prefect-dbt-flow repository and navigate to the example directory. +```bash +git clone git@github.com:datarootsio/prefect-dbt-flow.git +cd prefect-dbt-flow/example/jaffle_shop +``` + +### 2. Install Docker Compose +Ensure that you have Docker Compose installed on your system. If you haven't already installed it, refer to the [Docker Compose Installation Guide](https://docs.docker.com/compose/install/) for instructions. + +### 3. Start the Docker Container +Start the Docker container by running the following command. This command will launch three services defined in the docker-compose file: +- A PostgreSQL database, +- A Prefect server accessible at: `http://0.0.0.0:4200/`, +- A CLI environment with all the required components installed. +```bash +docker compose up -d +``` + +### 4. Access the cli service +To access the CLI service, use the following command: +```bash +docker compose run cli +``` + +### 5. Run the Prefect flow +Inside the CLI environment, run the Prefect-dbt-flow using the following command: +```bash +python my_prefect_dbt_flow +``` +This command will execute the Prefect flow and print its status to the terminal. + +### 6. View the reseults +To view the results and monitor the flow, follow these steps: + +- Open a web browser and go to `http://0.0.0.0:4200/`. +- In the Prefect Server interface, click on the flow run. It should have a name similar to `adjective-animal`. +- From there, you can explore the dbt job DAG and its associated logs. + +With these steps, you can set up and run a Prefect-dbt-flow and monitor its progress through the Prefect Server interface. + +## Installation +Before using Prefect-dbt-flow, you need to install the library. You can do this using pip: +```shell +pip install prefect-dbt-flow +``` +You can install an specific version of **Prefect** if you need to: +```shell +pip install prefect==2.13.5 +``` + +## Creating a Prefect Flow +To get started, you'll need to create a Prefect flow that incorporates your dbt project. Here's a step-by-step guide: +1. **Import the Required Modules:** + Start by importing the necessary modules from prefect_dbt_flow: + ```python + from prefect_dbt_flow import dbt_flow + ``` +2. **Define the Prefect Flow:** + Create a Prefect flow by initializing a `dbtFlow.dbt_flow` object. You can configure it with your dbt project, profile, and additional options: + * **project**: A DbtProject object representing the dbt project configuration. + * **profile**: A DbtProfile object representing the dbt profile configuration. + * **dag_options**: A DbtDagOptions object to specify dbt DAG configurations. + * **flow_kwargs**: A dictionary of Prefect flow arguments. + Here's a basic example of how to use dbt_flow(): + ```python + my_flow = dbtFlow.dbt_flow( + project=dbtFlow.DbtProject( + name="my_flow", + project_dir="path_to/dbt_project", + profiles_dir="path_to/dbt_profiles", + ), + profile=dbtFlow.DbtProfile( + target="dev", + ), + dag_options=dbtFlow.DbtDagOptions( + run_test_after_model=True, + ), + ) + ``` + With this basic setup, you have created a Prefect flow that manages your dbt project. When you run the script, Prefect will execute the dbt tasks defined in your project. +3. **Run the Flow:** + To execute the Prefect flow, add the following code block: + ```python + if __name__ == "__main__": + my_flow() + ``` +4. **Start the prefect server** + You will need to start prefect before the run + ```shell + prefect server start + ``` + You can check up the dashoard at `http://0.0.0.0:4200` +5. **Running the Prefect Flow:** + To run the Prefect flow, simply execute your Python script: + ```shell + python my_prefect_dbt_flow.py + ``` + Make sure you are in the correct directory or provide the full path to your script. Prefect will execute the dbt tasks defined in your flow, providing orchestration and monitoring capabilities. +6. **See the run** + You will be able to see the results of the run on the prefect dashboard at `http://0.0.0.0:4200` + +## Advanced Configuration +In the previous section, you configured your dbt project within the Prefect flow. Here's how you can customize the configuration further: + +### Dbt Project Configuration: +You specified the name, project directory, and profiles directory when creating the DbtProject object. Adjust these values to match your dbt project's setup. +- `DbtProject`: Represents your dbt project configuration. + - `name`: Name of the dbt project. + - `project_dir`: Path to the directory containing the project.yml configuration file. + - `profiles_dir`: Path to the directory containing the profiles.yml file. + +### Dbt Profile Configuration: +The DbtProfile object allows you to set the target profile for your dbt project. This profile should match the configuration in your dbt profiles.yml file. +- `DbtProfile`: Represents the dbt profile configuration. + - `target`: Specify the dbt target (e.g., "dev" or "prod"). + +### Dag Options: +The DbtDagOptions object lets you define various options for your dbt workflow. In the provided example, we set run_test_after_model to True, indicating that dbt tests should run after each dbt model. +- `DbtDagOptions`: Allows you to specify dbt DAG configurations. + - `select`: Specify a dbt module to include in the run. + - `exclude`: Specify a dbt module to exclude in the run. + - `run_test_after_model`: Set this to True to run tests after running models. + +### Prefect flow configuration +Prefect-dbt-flow integrates with Prefect's monitoring and error handling capabilities. You can use Prefect features like scheduling, notifications, and task retries to monitor and manage your dbt flows effectively. You can pass this additional Prefect flow configuration options using a dictionary into: `flow_kwargs`. + +For more information on these features, consult the [Prefect documentation.](https://docs.prefect.io/2.10.12/api-ref/prefect/flows/#prefect.flows.flow) + +## Conclusion +Prefect-dbt-flow simplifies the orchestration and management of dbt workflows within a Prefect flow. By following the steps in this guide, you can easily create and execute data pipelines that incorporate dbt projects. Be aware of breaking changes as this library is actively developed, and consult the changelog for updates. Happy data engineering! :rocket: \ No newline at end of file diff --git a/README.md b/README.md index 220fbac..95820dc 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,58 @@ -![dataroots.png](https://dataroots.io/assets/logo/logo-rainbow.png) -[![maintained by dataroots](https://img.shields.io/badge/maintained%20by-dataroots-%2300b189)](https://dataroots.io) +

+ logo +

+

+ Maintained by dataroots + Python versions + PiPy + Downloads + Code style: black + Mypy checked + +

# prefect-dbt-flow -Welcome to the prefect-dbt-flow integration repository! This project aims to provide a seamless integration for simplifying the execution of dbt workflows using Prefect. +Prefect-dbt-flow is a Python library that enables Prefect to convert dbt workflows into independent tasks within a Prefect flow. This integration simplifies the orchestration and execution of dbt models and tests using Prefect, allowing you to build robust data pipelines and monitor your dbt projects efficiently. -## Requirements -Before you get started, make sure you have the following prerequisites installed on your system: +**Active Development Notice:** Prefect-dbt-flow is actively under development and may not be ready for production use. We advise users to be aware of potential breaking changes as the library evolves. Please check the changelog for updates. -- python -- prefect -- dbt +## Table of Contents +- [Introduction](#introduction) +- [Why Use Prefect-dbt-flow?](#why-use-prefect-dbt-flow) +- [How to Install](#how-to-install) +- [Basic Usage](#basic-usage) +- [Inspiration](#inspiration) +- [License](#license) -## Installation -``` bash -pip install prefect-dbt-flow -``` +## Introduction +Prefect-dbt-flow is a tool designed to streamline the integration of dbt workflows into Prefect. dbt is an immensely popular tool for building and testing data transformation models, and Prefect is a versatile workflow management system. This integration brings together the best of both worlds, empowering data engineers and analysts to create robust data pipelines. -## Usage +## Why Use Prefect-dbt-flow? +### Simplified Orchestration +With Prefect-dbt-flow, you can orchestrate your dbt workflows with ease. Define and manage your dbt projects and models as Prefect tasks, creating a seamless pipeline for data transformation. -### Create a flow +[Simplified Orchestration]() -``` python -import prefect_dbt_flow as dbtFlow +### Monitoring and Error Handling +Prefect provides extensive monitoring capabilities and error handling. Now, you can gain deep insights into the execution of your dbt workflows and take immediate action in case of issues. + +[Monitoring and Error Handling]() + +### Workflow Consistency +Ensure your dbt workflows run consistently by managing them through Prefect. This consistency is crucial for maintaining data quality and reliability. + +[Workflow Consistency]() +## How to Install +You can install Prefect-dbt-flow via pip: +```shell +pip install prefect-dbt-flow +``` +## Basic Usage +Here's an example of how to use Prefect-dbt-flow to create a Prefect flow for your dbt project: +```python +import prefect_dbt_flow as dbtFlow my_flow = dbtFlow.dbt_flow( project=dbtFlow.DbtProject( name="my_flow", @@ -36,10 +66,16 @@ my_flow = dbtFlow.dbt_flow( run_test_after_model=True, ), ) - if __name__ == "__main__": my_flow() ``` +For more information consult the [Getting started guide](GETTING_STARTED.md) + +## Inspiration +Prefect-dbt-flow draws inspiration from various projects in the data engineering and workflow orchestration space, including: +- cosmos by astronomer +- anna-geller => prefect-dataplatform +- dbt + Dagster -## License -This project is licensed under the MIT License. \ No newline at end of file +# License +This project is licensed under the MIT License. You are free to use, modify, and distribute this software as per the terms of the license. If you find this project helpful, please consider giving it a star on GitHub. \ No newline at end of file diff --git a/docker/Dockerfile b/examples/jaffle_shop/Dockerfile similarity index 100% rename from docker/Dockerfile rename to examples/jaffle_shop/Dockerfile diff --git a/docker/docker-compose.yml b/examples/jaffle_shop/docker-compose.yml similarity index 68% rename from docker/docker-compose.yml rename to examples/jaffle_shop/docker-compose.yml index ed6fe25..670fec2 100644 --- a/docker/docker-compose.yml +++ b/examples/jaffle_shop/docker-compose.yml @@ -5,7 +5,7 @@ services: image: postgres:15.2-alpine restart: always environment: - - POSTGRES_USER=postgres + - POSTGRES_USER=admin - POSTGRES_PASSWORD=pass123 - POSTGRES_DB=data expose: @@ -23,4 +23,17 @@ services: - PREFECT_SERVER_API_HOST=0.0.0.0 - PREFECT_API_DATABASE_CONNECTION_URL=sqlite+aiosqlite:////opt/prefect/prefect.db ports: - - 4200:4200 \ No newline at end of file + - 4200:4200 + + cli: + build: . + entrypoint: "bash" + working_dir: "/root/flows" + volumes: + - "./jaffle_shop:/root/flows" + environment: + - PREFECT_API_URL=http://server:4200/api + +networks: + default: + name: prefect-dbt-network \ No newline at end of file diff --git a/examples/jaffle_shop/jaffle_shop/dbt_project.yml b/examples/jaffle_shop/jaffle_shop/dbt_project.yml new file mode 100644 index 0000000..1f62197 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/dbt_project.yml @@ -0,0 +1,26 @@ +name: 'example_jaffle_shop' + +config-version: 2 +version: '0.1' + +profile: 'example_jaffle_shop' + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +analysis-paths: ["analysis"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_modules" + - "logs" + +require-dbt-version: [">=1.0.0", "<2.0.0"] + +models: + example_jaffle_shop: + materialized: table + staging: + materialized: view \ No newline at end of file diff --git a/examples/jaffle_shop/jaffle_shop/models/customers.sql b/examples/jaffle_shop/jaffle_shop/models/customers.sql new file mode 100644 index 0000000..016a004 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/customers.sql @@ -0,0 +1,69 @@ +with customers as ( + + select * from {{ ref('stg_customers') }} + +), + +orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +customer_orders as ( + + select + customer_id, + + min(order_date) as first_order, + max(order_date) as most_recent_order, + count(order_id) as number_of_orders + from orders + + group by customer_id + +), + +customer_payments as ( + + select + orders.customer_id, + sum(amount) as total_amount + + from payments + + left join orders on + payments.order_id = orders.order_id + + group by orders.customer_id + +), + +final as ( + + select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order, + customer_orders.most_recent_order, + customer_orders.number_of_orders, + customer_payments.total_amount as customer_lifetime_value + + from customers + + left join customer_orders + on customers.customer_id = customer_orders.customer_id + + left join customer_payments + on customers.customer_id = customer_payments.customer_id + +) + +select * from final diff --git a/examples/jaffle_shop/jaffle_shop/models/docs.md b/examples/jaffle_shop/jaffle_shop/models/docs.md new file mode 100644 index 0000000..c6ae93b --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/docs.md @@ -0,0 +1,14 @@ +{% docs orders_status %} + +Orders can be one of the following statuses: + +| status | description | +|----------------|------------------------------------------------------------------------------------------------------------------------| +| placed | The order has been placed but has not yet left the warehouse | +| shipped | The order has ben shipped to the customer and is currently in transit | +| completed | The order has been received by the customer | +| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse | +| returned | The order has been returned by the customer and received at the warehouse | + + +{% enddocs %} diff --git a/examples/jaffle_shop/jaffle_shop/models/orders.sql b/examples/jaffle_shop/jaffle_shop/models/orders.sql new file mode 100644 index 0000000..cbb2934 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/orders.sql @@ -0,0 +1,56 @@ +{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} + +with orders as ( + + select * from {{ ref('stg_orders') }} + +), + +payments as ( + + select * from {{ ref('stg_payments') }} + +), + +order_payments as ( + + select + order_id, + + {% for payment_method in payment_methods -%} + sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, + {% endfor -%} + + sum(amount) as total_amount + + from payments + + group by order_id + +), + +final as ( + + select + orders.order_id, + orders.customer_id, + orders.order_date, + orders.status, + + {% for payment_method in payment_methods -%} + + order_payments.{{ payment_method }}_amount, + + {% endfor -%} + + order_payments.total_amount as amount + + from orders + + + left join order_payments + on orders.order_id = order_payments.order_id + +) + +select * from final diff --git a/examples/jaffle_shop/jaffle_shop/models/overview.md b/examples/jaffle_shop/jaffle_shop/models/overview.md new file mode 100644 index 0000000..0544c42 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/overview.md @@ -0,0 +1,11 @@ +{% docs __overview__ %} + +## Data Documentation for Jaffle Shop + +`jaffle_shop` is a fictional ecommerce store. + +This [dbt](https://www.getdbt.com/) project is for testing out code. + +The source code can be found [here](https://github.com/clrcrl/jaffle_shop). + +{% enddocs %} diff --git a/examples/jaffle_shop/jaffle_shop/models/schema.yml b/examples/jaffle_shop/jaffle_shop/models/schema.yml new file mode 100644 index 0000000..381349c --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/schema.yml @@ -0,0 +1,82 @@ +version: 2 + +models: + - name: customers + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders + + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + + - name: first_name + description: Customer's first name. PII. + + - name: last_name + description: Customer's last name. PII. + + - name: first_order + description: Date (UTC) of a customer's first order + + - name: most_recent_order + description: Date (UTC) of a customer's most recent order + + - name: number_of_orders + description: Count of the number of orders a customer has placed + + - name: total_order_amount + description: Total value (AUD) of a customer's orders + + - name: orders + description: This table has basic information about orders, as well as some derived facts based on payments + + columns: + - name: order_id + tests: + - unique + - not_null + description: This is a unique identifier for an order + + - name: customer_id + description: Foreign key to the customers table + tests: + - not_null + - relationships: + to: ref('customers') + field: customer_id + + - name: order_date + description: Date (UTC) that the order was placed + + - name: status + description: '{{ doc("orders_status") }}' + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: amount + description: Total amount (AUD) of the order + tests: + - not_null + + - name: credit_card_amount + description: Amount of the order (AUD) paid for by credit card + tests: + - not_null + + - name: coupon_amount + description: Amount of the order (AUD) paid for by coupon + tests: + - not_null + + - name: bank_transfer_amount + description: Amount of the order (AUD) paid for by bank transfer + tests: + - not_null + + - name: gift_card_amount + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null diff --git a/examples/jaffle_shop/jaffle_shop/models/staging/schema.yml b/examples/jaffle_shop/jaffle_shop/models/staging/schema.yml new file mode 100644 index 0000000..c207e4c --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/staging/schema.yml @@ -0,0 +1,31 @@ +version: 2 + +models: + - name: stg_customers + columns: + - name: customer_id + tests: + - unique + - not_null + + - name: stg_orders + columns: + - name: order_id + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: stg_payments + columns: + - name: payment_id + tests: + - unique + - not_null + - name: payment_method + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/examples/jaffle_shop/jaffle_shop/models/staging/stg_customers.sql b/examples/jaffle_shop/jaffle_shop/models/staging/stg_customers.sql new file mode 100644 index 0000000..cad0472 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/staging/stg_customers.sql @@ -0,0 +1,22 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_customers') }} + +), + +renamed as ( + + select + id as customer_id, + first_name, + last_name + + from source + +) + +select * from renamed diff --git a/examples/jaffle_shop/jaffle_shop/models/staging/stg_orders.sql b/examples/jaffle_shop/jaffle_shop/models/staging/stg_orders.sql new file mode 100644 index 0000000..a654dcb --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/staging/stg_orders.sql @@ -0,0 +1,23 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} + +), + +renamed as ( + + select + id as order_id, + user_id as customer_id, + order_date, + status + + from source + +) + +select * from renamed diff --git a/examples/jaffle_shop/jaffle_shop/models/staging/stg_payments.sql b/examples/jaffle_shop/jaffle_shop/models/staging/stg_payments.sql new file mode 100644 index 0000000..700cf7f --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/models/staging/stg_payments.sql @@ -0,0 +1,25 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_payments') }} + +), + +renamed as ( + + select + id as payment_id, + order_id, + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars + amount / 100 as amount + + from source + +) + +select * from renamed diff --git a/examples/jaffle_shop/jaffle_shop/my_prefect_dbt_flow.py b/examples/jaffle_shop/jaffle_shop/my_prefect_dbt_flow.py new file mode 100644 index 0000000..3e57607 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/my_prefect_dbt_flow.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from prefect_dbt_flow import dbt_flow +from prefect_dbt_flow.dbt import DbtProfile, DbtProject + +# from prefect.task_runners import SequentialTaskRunner + + +my_dbt_flow = dbt_flow( + project=DbtProject( + name="example_jaffle_shop", + project_dir=Path(__file__).parent, + profiles_dir=Path(__file__).parent, + ), + profile=DbtProfile( + target="dev", + ), + # flow_kwargs={ + # # Ensure only one process has access to the duckdb db + # # file at the same time + # "task_runner": SequentialTaskRunner(), + # }, +) + +if __name__ == "__main__": + my_dbt_flow() diff --git a/examples/jaffle_shop/jaffle_shop/profiles.yml b/examples/jaffle_shop/jaffle_shop/profiles.yml new file mode 100644 index 0000000..184101e --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/profiles.yml @@ -0,0 +1,12 @@ +example_jaffle_shop: + target: dev + outputs: + dev: + type: postgres + host: data-db + user: admin + password: pass123 + port: 5432 + dbname: data + schema: example + connect_timeout: 10 \ No newline at end of file diff --git a/examples/jaffle_shop/jaffle_shop/seeds/raw_customers.csv b/examples/jaffle_shop/jaffle_shop/seeds/raw_customers.csv new file mode 100644 index 0000000..e386bb3 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/seeds/raw_customers.csv @@ -0,0 +1,101 @@ +id,first_name,last_name +1,Michael,P. +2,Shawn,M. +3,Kathleen,P. +4,Jimmy,C. +5,Katherine,R. +6,Sarah,R. +7,Martin,M. +8,Frank,R. +9,Jennifer,F. +10,Henry,W. +11,Fred,S. +12,Amy,D. +13,Kathleen,M. +14,Steve,F. +15,Teresa,H. +16,Amanda,H. +17,Kimberly,R. +18,Johnny,K. +19,Virginia,F. +20,Anna,A. +21,Willie,H. +22,Sean,H. +23,Mildred,A. +24,David,G. +25,Victor,H. +26,Aaron,R. +27,Benjamin,B. +28,Lisa,W. +29,Benjamin,K. +30,Christina,W. +31,Jane,G. +32,Thomas,O. +33,Katherine,M. +34,Jennifer,S. +35,Sara,T. +36,Harold,O. +37,Shirley,J. +38,Dennis,J. +39,Louise,W. +40,Maria,A. +41,Gloria,C. +42,Diana,S. +43,Kelly,N. +44,Jane,R. +45,Scott,B. +46,Norma,C. +47,Marie,P. +48,Lillian,C. +49,Judy,N. +50,Billy,L. +51,Howard,R. +52,Laura,F. +53,Anne,B. +54,Rose,M. +55,Nicholas,R. +56,Joshua,K. +57,Paul,W. +58,Kathryn,K. +59,Adam,A. +60,Norma,W. +61,Timothy,R. +62,Elizabeth,P. +63,Edward,G. +64,David,C. +65,Brenda,W. +66,Adam,W. +67,Michael,H. +68,Jesse,E. +69,Janet,P. +70,Helen,F. +71,Gerald,C. +72,Kathryn,O. +73,Alan,B. +74,Harry,A. +75,Andrea,H. +76,Barbara,W. +77,Anne,W. +78,Harry,H. +79,Jack,R. +80,Phillip,H. +81,Shirley,H. +82,Arthur,D. +83,Virginia,R. +84,Christina,R. +85,Theresa,M. +86,Jason,C. +87,Phillip,B. +88,Adam,T. +89,Margaret,J. +90,Paul,P. +91,Todd,W. +92,Willie,O. +93,Frances,R. +94,Gregory,H. +95,Lisa,P. +96,Jacqueline,A. +97,Shirley,D. +98,Nicole,M. +99,Mary,G. +100,Jean,M. \ No newline at end of file diff --git a/examples/jaffle_shop/jaffle_shop/seeds/raw_orders.csv b/examples/jaffle_shop/jaffle_shop/seeds/raw_orders.csv new file mode 100644 index 0000000..45160e1 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/seeds/raw_orders.csv @@ -0,0 +1,100 @@ +id,user_id,order_date,status +1,1,2018-01-01,returned +2,3,2018-01-02,completed +3,94,2018-01-04,completed +4,50,2018-01-05,completed +5,64,2018-01-05,completed +6,54,2018-01-07,completed +7,88,2018-01-09,completed +8,2,2018-01-11,returned +9,53,2018-01-12,completed +10,7,2018-01-14,completed +11,99,2018-01-14,completed +12,59,2018-01-15,completed +13,84,2018-01-17,completed +14,40,2018-01-17,returned +15,25,2018-01-17,completed +16,39,2018-01-18,completed +17,71,2018-01-18,completed +18,64,2018-01-20,returned +19,54,2018-01-22,completed +20,20,2018-01-23,completed +21,71,2018-01-23,completed +22,86,2018-01-24,completed +23,22,2018-01-26,return_pending +24,3,2018-01-27,completed +25,51,2018-01-28,completed +26,32,2018-01-28,completed +27,94,2018-01-29,completed +28,8,2018-01-29,completed +29,57,2018-01-31,completed +30,69,2018-02-02,completed +31,16,2018-02-02,completed +32,28,2018-02-04,completed +33,42,2018-02-04,completed +34,38,2018-02-06,completed +35,80,2018-02-08,completed +36,85,2018-02-10,completed +37,1,2018-02-10,completed +38,51,2018-02-10,completed +39,26,2018-02-11,completed +40,33,2018-02-13,completed +41,99,2018-02-14,completed +42,92,2018-02-16,completed +43,31,2018-02-17,completed +44,66,2018-02-17,completed +45,22,2018-02-17,completed +46,6,2018-02-19,completed +47,50,2018-02-20,completed +48,27,2018-02-21,completed +49,35,2018-02-21,completed +50,51,2018-02-23,completed +51,71,2018-02-24,completed +52,54,2018-02-25,return_pending +53,34,2018-02-26,completed +54,54,2018-02-26,completed +55,18,2018-02-27,completed +56,79,2018-02-28,completed +57,93,2018-03-01,completed +58,22,2018-03-01,completed +59,30,2018-03-02,completed +60,12,2018-03-03,completed +61,63,2018-03-03,completed +62,57,2018-03-05,completed +63,70,2018-03-06,completed +64,13,2018-03-07,completed +65,26,2018-03-08,completed +66,36,2018-03-10,completed +67,79,2018-03-11,completed +68,53,2018-03-11,completed +69,3,2018-03-11,completed +70,8,2018-03-12,completed +71,42,2018-03-12,shipped +72,30,2018-03-14,shipped +73,19,2018-03-16,completed +74,9,2018-03-17,shipped +75,69,2018-03-18,completed +76,25,2018-03-20,completed +77,35,2018-03-21,shipped +78,90,2018-03-23,shipped +79,52,2018-03-23,shipped +80,11,2018-03-23,shipped +81,76,2018-03-23,shipped +82,46,2018-03-24,shipped +83,54,2018-03-24,shipped +84,70,2018-03-26,placed +85,47,2018-03-26,shipped +86,68,2018-03-26,placed +87,46,2018-03-27,placed +88,91,2018-03-27,shipped +89,21,2018-03-28,placed +90,66,2018-03-30,shipped +91,47,2018-03-31,placed +92,84,2018-04-02,placed +93,66,2018-04-03,placed +94,63,2018-04-03,placed +95,27,2018-04-04,placed +96,90,2018-04-06,placed +97,89,2018-04-07,placed +98,41,2018-04-07,placed +99,85,2018-04-09,placed \ No newline at end of file diff --git a/examples/jaffle_shop/jaffle_shop/seeds/raw_payments.csv b/examples/jaffle_shop/jaffle_shop/seeds/raw_payments.csv new file mode 100644 index 0000000..3989cb2 --- /dev/null +++ b/examples/jaffle_shop/jaffle_shop/seeds/raw_payments.csv @@ -0,0 +1,114 @@ +id,order_id,payment_method,amount +1,1,credit_card,1000 +2,2,credit_card,2000 +3,3,coupon,100 +4,4,coupon,2500 +5,5,bank_transfer,1700 +6,6,credit_card,600 +7,7,credit_card,1600 +8,8,credit_card,2300 +9,9,gift_card,2300 +10,9,bank_transfer,0 +11,10,bank_transfer,2600 +12,11,credit_card,2700 +13,12,credit_card,100 +14,13,credit_card,500 +15,13,bank_transfer,1400 +16,14,bank_transfer,300 +17,15,coupon,2200 +18,16,credit_card,1000 +19,17,bank_transfer,200 +20,18,credit_card,500 +21,18,credit_card,800 +22,19,gift_card,600 +23,20,bank_transfer,1500 +24,21,credit_card,1200 +25,22,bank_transfer,800 +26,23,gift_card,2300 +27,24,coupon,2600 +28,25,bank_transfer,2000 +29,25,credit_card,2200 +30,25,coupon,1600 +31,26,credit_card,3000 +32,27,credit_card,2300 +33,28,bank_transfer,1900 +34,29,bank_transfer,1200 +35,30,credit_card,1300 +36,31,credit_card,1200 +37,32,credit_card,300 +38,33,credit_card,2200 +39,34,bank_transfer,1500 +40,35,credit_card,2900 +41,36,bank_transfer,900 +42,37,credit_card,2300 +43,38,credit_card,1500 +44,39,bank_transfer,800 +45,40,credit_card,1400 +46,41,credit_card,1700 +47,42,coupon,1700 +48,43,gift_card,1800 +49,44,gift_card,1100 +50,45,bank_transfer,500 +51,46,bank_transfer,800 +52,47,credit_card,2200 +53,48,bank_transfer,300 +54,49,credit_card,600 +55,49,credit_card,900 +56,50,credit_card,2600 +57,51,credit_card,2900 +58,51,credit_card,100 +59,52,bank_transfer,1500 +60,53,credit_card,300 +61,54,credit_card,1800 +62,54,bank_transfer,1100 +63,55,credit_card,2900 +64,56,credit_card,400 +65,57,bank_transfer,200 +66,58,coupon,1800 +67,58,gift_card,600 +68,59,gift_card,2800 +69,60,credit_card,400 +70,61,bank_transfer,1600 +71,62,gift_card,1400 +72,63,credit_card,2900 +73,64,bank_transfer,2600 +74,65,credit_card,0 +75,66,credit_card,2800 +76,67,bank_transfer,400 +77,67,credit_card,1900 +78,68,credit_card,1600 +79,69,credit_card,1900 +80,70,credit_card,2600 +81,71,credit_card,500 +82,72,credit_card,2900 +83,73,bank_transfer,300 +84,74,credit_card,3000 +85,75,credit_card,1900 +86,76,coupon,200 +87,77,credit_card,0 +88,77,bank_transfer,1900 +89,78,bank_transfer,2600 +90,79,credit_card,1800 +91,79,credit_card,900 +92,80,gift_card,300 +93,81,coupon,200 +94,82,credit_card,800 +95,83,credit_card,100 +96,84,bank_transfer,2500 +97,85,bank_transfer,1700 +98,86,coupon,2300 +99,87,gift_card,3000 +100,87,credit_card,2600 +101,88,credit_card,2900 +102,89,bank_transfer,2200 +103,90,bank_transfer,200 +104,91,credit_card,1900 +105,92,bank_transfer,1500 +106,92,coupon,200 +107,93,gift_card,2600 +108,94,coupon,700 +109,95,coupon,2400 +110,96,gift_card,1700 +111,97,bank_transfer,1400 +112,98,bank_transfer,1000 +113,99,credit_card,2400 \ No newline at end of file diff --git a/docker/requirements.txt b/examples/jaffle_shop/requirements.txt similarity index 100% rename from docker/requirements.txt rename to examples/jaffle_shop/requirements.txt