From 2a1cff392c59fbed26f7275d4d6d672dd17ed707 Mon Sep 17 00:00:00 2001 From: Patricia Goldberg Date: Thu, 25 Nov 2021 15:49:24 +0100 Subject: [PATCH 1/4] Accept output format as a config --- README.md | 16 ++++++++++++++++ tap_pardot/client.py | 2 +- tap_pardot/streams.py | 8 ++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8260853..a3ffc72 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,22 @@ # tap-pardot Singer tap for replicating Pardot data. +## Create a Config file +``` +{ + "client_id": "secret_client_id", + "client_secret": "secret_client_secret", + "refresh_token": "abc123", + "start_date": "2017-11-02T00:00:00Z", + "api_output_type": "bulk" +} +``` +The `client_id` and `client_secret` keys are your OAuth Salesforce App secrets. The refresh_token is a secret created during the OAuth flow. For more info on the Pardot OAuth flow, visit the Pardot [API documentation](https://developer.salesforce.com/docs/marketing/pardot/guide/authentication.html). + +The `start_data` is used by the tap as a bound on the query request, for more information about the format check [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/docs/BEST_PRACTICES.md#dates). + +The `api_output_type` is used to define the output on the API call. The default is "bulk" (more information on the "bulk" output call on [Query the Pardot API](https://developer.salesforce.com/docs/marketing/pardot/guide/bulk-data-pull.html#query-the-pardot-api)). With the bulk API call, the call is optimized, on the other hand, it doesn't return additional data in the response (such as nested objects and custom fields). + --- Copyright © 2019 Stitch diff --git a/tap_pardot/client.py b/tap_pardot/client.py index 44acdaf..642cd99 100644 --- a/tap_pardot/client.py +++ b/tap_pardot/client.py @@ -236,7 +236,7 @@ def _fetch(self, method, endpoint, format_params, **kwargs): base_formatting.extend(format_params) url = (ENDPOINT_BASE + self.get_url).format(*base_formatting) - params = {"format": "json", "output": "bulk", **kwargs} + params = {"format": "json", **kwargs} content = self._make_request(method, url, params) diff --git a/tap_pardot/streams.py b/tap_pardot/streams.py index 013dd5f..e58e9d0 100644 --- a/tap_pardot/streams.py +++ b/tap_pardot/streams.py @@ -30,6 +30,9 @@ def get_default_start(self): def get_params(self): return {} + def get_api_output_type(self): + return self.config.get("api_output_type", "bulk") + def get_bookmark(self): return ( singer.bookmarks.get_bookmark( @@ -116,6 +119,7 @@ def get_default_start(self): def get_params(self): return { "created_after": self.config["start_date"], + "output": self.get_api_output_type(), "id_greater_than": self.get_bookmark(), "sort_by": "id", "sort_order": "ascending", @@ -139,6 +143,7 @@ class UpdatedAtReplicationStream(Stream): def get_params(self): return { "updated_after": self.get_bookmark(), + "output": self.get_api_output_type(), "sort_by": "updated_at", "sort_order": "ascending", } @@ -209,6 +214,7 @@ def post_sync(self): def get_params(self): return { "created_after": self.config["start_date"], + "output": self.get_api_output_type(), "id_greater_than": self.get_bookmark("id"), "sort_by": "id", "sort_order": "ascending", @@ -265,6 +271,7 @@ def post_sync(self): def get_params(self): return { "id_greater_than": self.get_bookmark("id"), + "output": self.get_api_output_type(), "updated_after": self.get_bookmark("last_updated"), "sort_by": "id", "sort_order": "ascending", @@ -459,6 +466,7 @@ def get_params(self): # filter by updated_after "updated_after": self.get_bookmark("updated_at") or self.config["start_date"], + "output": self.get_api_output_type(), "id_greater_than": self.get_bookmark("id") or 0, "sort_by": "id", "sort_order": "ascending", From a1382362d0325d461a57caca0548822e64edd9ea Mon Sep 17 00:00:00 2001 From: Patricia Goldberg Date: Fri, 26 Nov 2021 09:29:53 +0100 Subject: [PATCH 2/4] Add output format to describe method --- tap_pardot/client.py | 2 +- tap_pardot/streams.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tap_pardot/client.py b/tap_pardot/client.py index 642cd99..9f3a3ea 100644 --- a/tap_pardot/client.py +++ b/tap_pardot/client.py @@ -216,7 +216,7 @@ def _make_request(self, method, url, params=None): def describe(self, endpoint, **kwargs): url = (ENDPOINT_BASE + self.describe_url).format(endpoint, '{}') - params = {"format": "json", "output": "bulk", **kwargs} + params = {"format": "json", **kwargs} content = self._make_request("get", url, params) diff --git a/tap_pardot/streams.py b/tap_pardot/streams.py index e58e9d0..d5b48bb 100644 --- a/tap_pardot/streams.py +++ b/tap_pardot/streams.py @@ -302,7 +302,10 @@ def post_sync(self): super(ChildStream, self).post_sync() def get_params(self): - return {"offset": self.get_bookmark("offset")} + return { + "offset": self.get_bookmark("offset"), + "output": self.get_api_output_type(), + } def get_records(self, parent_ids): params = {self.parent_id_param: parent_ids, **self.get_params()} From ac04e0c4c541069dbdcafbdc5b8f10b30618324e Mon Sep 17 00:00:00 2001 From: Patricia Goldberg Date: Fri, 26 Nov 2021 10:56:33 +0100 Subject: [PATCH 3/4] Add more information to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3ffc72..e8c9056 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The `client_id` and `client_secret` keys are your OAuth Salesforce App secrets. The `start_data` is used by the tap as a bound on the query request, for more information about the format check [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/docs/BEST_PRACTICES.md#dates). -The `api_output_type` is used to define the output on the API call. The default is "bulk" (more information on the "bulk" output call on [Query the Pardot API](https://developer.salesforce.com/docs/marketing/pardot/guide/bulk-data-pull.html#query-the-pardot-api)). With the bulk API call, the call is optimized, on the other hand, it doesn't return additional data in the response (such as nested objects and custom fields). +The `api_output_type` is used to define the output on the API call. The default is "bulk" (more information on the "bulk" output call on [Query the Pardot API](https://developer.salesforce.com/docs/marketing/pardot/guide/bulk-data-pull.html#query-the-pardot-api)). With the bulk API call, the call is optimized, on the other hand, it doesn't return additional data in the response (such as nested objects and custom fields). If additional data is needed, change this variable (to either "simple" or "full"), and add the additional data explicitly to the select statement. --- From 90fddf717d62b41cbfdb2cc9af5faaced9e7e66e Mon Sep 17 00:00:00 2001 From: Patricia Goldberg Date: Mon, 29 Nov 2021 10:08:04 +0100 Subject: [PATCH 4/4] Adjust readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e8c9056..41a9d85 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,11 @@ Singer tap for replicating Pardot data. "api_output_type": "bulk" } ``` -The `client_id` and `client_secret` keys are your OAuth Salesforce App secrets. The refresh_token is a secret created during the OAuth flow. For more info on the Pardot OAuth flow, visit the Pardot [API documentation](https://developer.salesforce.com/docs/marketing/pardot/guide/authentication.html). +The `client_id` and `client_secret` keys are your OAuth Salesforce App secrets. The `refresh_token` is a secret created during the OAuth flow. For more info on the Pardot OAuth flow, visit the Pardot [API documentation](https://developer.salesforce.com/docs/marketing/pardot/guide/authentication.html). -The `start_data` is used by the tap as a bound on the query request, for more information about the format check [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/docs/BEST_PRACTICES.md#dates). +The `start_date` is used by the tap as a bound on the query request, for more information about the format check [Singer best practices for dates](https://github.com/singer-io/getting-started/blob/master/docs/BEST_PRACTICES.md#dates). -The `api_output_type` is used to define the output on the API call. The default is "bulk" (more information on the "bulk" output call on [Query the Pardot API](https://developer.salesforce.com/docs/marketing/pardot/guide/bulk-data-pull.html#query-the-pardot-api)). With the bulk API call, the call is optimized, on the other hand, it doesn't return additional data in the response (such as nested objects and custom fields). If additional data is needed, change this variable (to either "simple" or "full"), and add the additional data explicitly to the select statement. +The `api_output_type` is used to define the output on the API call. The default is "bulk" (more information on the "bulk" output call on [Query the Pardot API](https://developer.salesforce.com/docs/marketing/pardot/guide/bulk-data-pull.html#query-the-pardot-api) and [Changing the api response format](https://developer.salesforce.com/docs/marketing/pardot/guide/version-3-4-overview.html#changing-the-api-response-format)). With the bulk API call, the call is optimized, on the other hand, it doesn't return additional data in the response (such as nested objects and custom fields). If additional data is needed, change this variable (to either "simple" or "full"), and add the additional data explicitly to the select statement. ---