Add tap-tester suite (#2)

* initial commit

* first pass at client.py and auth

* added dynamic entity discovery & stream generation

* changes/fixes for dicsovery mode

* fixes/additions to streams.py

* getting sync.py working w/ dynamic scema creation

* adding transform.py code

* changes for sync mode

* changes after internal code review

* removed unused files

* pylint related chages

* adding config_path from args

* adding pagination, param building

* removed unused `tenant_id` param

* adding $metadata endpoint

* changes for $metadata endpoint

* adding transform unit tests

* removing schemas

* removing msal req

* adding 429  retry logic

* add refresh_token to REQUIRED_CONFIG_KEYS

* update README

* replacing custom code
using `valid_replication_keys` kwarg from get_standard_metadata instead

* add tap_tester base

* add tap_tester discover

* fix for tap name

* build explicit config for DynamicsClient

* check expected_streams are subset of found_streams

* entities from ms dynamics can contain numbers

* add tap-tester field selection

* fix to check for subset of found_catalog_names

* fix for max_pagesize being None in tap-tester

* add tap-tester automated fields

* fix to use correct variable and formatting changes

* add tap-tester bookmarks

* fix for bookmarks  tap-tester
failing with blank start_date

* add tap-tester start_date

* remove unused imports

* remove more unused imports

* additioinal excluded entities from testing

* removing tap-tester field selection

* removing get_properties as it's defined in base

* removing get_properties as it's defined in base

* change start_date to match test data

* changes to base:
* setting start date in get_properties
* not testing dependency stream as it has too many records in test data

* recommended changes from PR #2 review
* adding expected_replication_methods
* only testing INCREMENTAL streams
* 2nd sync should have more records

* removing unrelated checks for irelevant bugs

* fix to  repliation_key set "inclusion": automatic

* PR #2 recommended changes:
* fix docstring for test_run()
* set original_properties=True for ensure_connection()
* remove lookback references

* remove `user_agent` from get_properties()

* add tests for FULL_TABLE streams

* removing reference to lookback as there is none used

* adjust assertion for full table sync

setup.py

@@ -6,7 +6,7 @@
     readme = f.read()
 
 setup(
-    name="tap-dynamics",
+    name="tap-ms-dynamics",
     version="0.1.0",
     description="Singer.io tap for the Microsoft Dataverse Web API",
     long_description=readme,
@@ -17,7 +17,7 @@
     install_requires=open(requirements_file).readlines(),
     entry_points="""
     [console_scripts]
-    tap-dynamics=tap_dynamics:main
+    tap-ms-dynamics=tap_dynamics:main
     """,
     packages=find_packages(exclude=["tests"]),
     package_data = {

tap_dynamics/client.py

@@ -73,6 +73,7 @@ def __init__(self,
                 start_date=None):
         self.organization_uri = organization_uri
         self.api_version = api_version if api_version else API_VERSION
+        max_pagesize = MAX_PAGESIZE if max_pagesize is None else max_pagesize # tap-tester was failing otherwise
         self.max_pagesize = max_pagesize if max_pagesize <= MAX_PAGESIZE else MAX_PAGESIZE
         self.client_id = client_id
         self.client_secret = client_secret
@@ -230,7 +231,12 @@ def build_entity_metadata(self):
         yield from entity_metadata.values()
 
     @staticmethod
-    def build_params(orderby_key: str = 'modifiedon', replication_key: str = 'modifiedon', filter_value: str = None) -> dict:
-        filter_param = f'{replication_key} ge {filter_value}'
+    def build_params(orderby_key: str = 'modifiedon',
+                    replication_key: str = 'modifiedon',
+                    filter_value: str = None) -> dict:
         orderby_param = f'{orderby_key} asc'
-        return {"$orderby": orderby_param, "$filter": filter_param}
+
+        if filter_value:
+            filter_param = f'{replication_key} ge {filter_value}'
+            return {"$orderby": orderby_param, "$filter": filter_param}
+        return {"$orderby": orderby_param}

tap_dynamics/discover.py

@@ -42,6 +42,15 @@ def get_schemas(config, config_path):
             valid_replication_keys=stream_object.valid_replication_keys
         )
 
+        meta = metadata.to_map(meta)
+
+        if stream_object.replication_key:
+            meta = metadata.write(meta,
+                                 ('properties', stream_object.replication_key),
+                                 'inclusion', 'automatic')
+
+        meta = metadata.to_list(meta)
+
         schemas[stream_name] = schema
         schemas_metadata[stream_name] = meta
 

tap_dynamics/streams.py

@@ -2,7 +2,7 @@
 from singer import Transformer, metrics
 
 from tap_dynamics.client import DynamicsClient
-from tap_dynamics.transform import flatten_entity_attributes
+from tap_dynamics.transform import (flatten_entity_attributes, get_client_args)
 
 LOGGER = singer.get_logger()
 
@@ -31,6 +31,9 @@
     'systemusersyncmappingprofiles',
     'teamsyncattributemappingprofiles',
     'timestampdatemapping',
+    'lookupmapping',
+    'msdyn_casesuggestion',
+    'msdyn_knowledgearticlesuggestion'
 ])
 
 STRING_TYPES = set([
@@ -110,6 +113,7 @@ class IncrementalStream(BaseStream):
     def get_records(self, max_pagesize: int = 100, bookmark_datetime: str = None):
         endpoint = self.stream_endpoint
 
+        max_pagesize = MAX_PAGESIZE if max_pagesize is None else max_pagesize # tap-tester was failing otherwise
         pagesize = max_pagesize if max_pagesize <= MAX_PAGESIZE else MAX_PAGESIZE
         header = {'Prefer': f'odata.maxpagesize={pagesize}'}
 
@@ -176,6 +180,7 @@ class FullTableStream(BaseStream):
     def get_records(self, max_pagesize: int = 100):
         endpoint = self.stream_endpoint
 
+        max_pagesize = MAX_PAGESIZE if max_pagesize is None else max_pagesize # tap-tester was failing otherwise
         pagesize = max_pagesize if max_pagesize <= MAX_PAGESIZE else MAX_PAGESIZE
         header = {'Prefer': f'odata.maxpagesize={pagesize}'}
 
@@ -228,7 +233,9 @@ def get_streams(config: dict, config_path: str) -> dict:
     STREAMS = {} # pylint: disable=invalid-name
 
     config["config_path"] = config_path
-    client = DynamicsClient(**config)
+    client_config = get_client_args(config)
+
+    client = DynamicsClient(**client_config)
 
     # dynamically build streams by iterating over entities and calling build_schema()
     for stream in client.build_entity_metadata():

tap_dynamics/transform.py

@@ -38,3 +38,17 @@ def transform_metadata_xml(xml:str) -> dict:
             entity_def.update({entity_name: {"Key": entity_key, "Properties": props}})
 
     return entity_def
+
+def get_client_args(config):
+    return {
+        "organization_uri": config.get('organization_uri'),
+        "config_path":      config.get('config_path'),
+        "max_pagesize":     config.get('max_pagesize'),
+        "api_version":      config.get('api_version'),
+        "client_id":        config.get('client_id'),
+        "client_secret":    config.get('client_secret'),
+        "user_agent":       config.get('user_agent'),
+        "redirect_uri":     config.get('redirect_uri'),
+        "refresh_token":    config.get('refresh_token'),
+        "start_date":       config.get('start_date'),
+    }