diff --git a/connectors/reference_connector/app.py b/connectors/reference_connector/app.py index e9d6caf0..aec2fa15 100644 --- a/connectors/reference_connector/app.py +++ b/connectors/reference_connector/app.py @@ -1,4 +1,4 @@ -import sys, argparse, traceback +import sys, argparse, traceback, os from car_framework.context import context from car_framework.app import BaseApp @@ -16,9 +16,10 @@ class App(BaseApp): def __init__(self): super().__init__('This script is used for pushing asset data to CP4S CAR ingestion microservice') # Add parameters need to connect data source - self.parser.add_argument('-server', dest='server', type=str, required=True, help='The url of the Asset data server') - self.parser.add_argument('-username', dest='username', type=str, required=True, help='The user name for the Asset data server') - self.parser.add_argument('-password', dest='password', type=str, required=True, help='The password for the Asset data server') + # Make sure the "dest" variable is same as the env variable. For example below dest='CONFIGURATION_AUTH_URL and os.getenv('CONFIGURATION_AUTH_URL',None); CONFIGURATION_AUTH_URL is same. + self.parser.add_argument('-server', dest='CONFIGURATION_AUTH_URL', default=os.getenv('CONFIGURATION_AUTH_URL',None), type=str, required=False, help='The url of the Asset data server') + self.parser.add_argument('-username', dest='CONFIGURATION_AUTH_USERNAME', default=os.getenv('CONFIGURATION_AUTH_USERNAME',None), type=str, required=False, help='The user name for the Asset data server') + self.parser.add_argument('-password', dest='CONFIGURATION_AUTH_PASSWORD', default=os.getenv('CONFIGURATION_AUTH_PASSWORD',None), type=str, required=False, help='The password for the Asset data server') def setup(self): super().setup() diff --git a/connectors/reference_connector/connector/data_handler.py b/connectors/reference_connector/connector/data_handler.py index 7b72d107..73f19c96 100644 --- a/connectors/reference_connector/connector/data_handler.py +++ b/connectors/reference_connector/connector/data_handler.py @@ -43,7 +43,7 @@ def __init__(self): def create_source_report_object(self): if not (self.source and self.report): # create source and report entry and it is compuslory for each imports API call - self.source = {'_key': context().args.CONNECTION_NAME, 'name': context().args.server, 'description': 'Reference Asset server'} + self.source = {'_key': context().args.CONNECTION_NAME, 'name': context().args.CONFIGURATION_AUTH_URL, 'description': 'Reference Asset server'} self.report = {'_key': str(self.timestamp), 'timestamp' : self.timestamp, 'type': 'Reference Asset server', 'description': 'Reference Asset server'} return {'source': self.source, 'report': self.report} diff --git a/connectors/reference_connector/connector/server_access.py b/connectors/reference_connector/connector/server_access.py index 0922ca54..d570e009 100644 --- a/connectors/reference_connector/connector/server_access.py +++ b/connectors/reference_connector/connector/server_access.py @@ -9,7 +9,7 @@ class AssetServer(BaseAssetServer): def __init__(self): # Api authentication to call data-source API - auth = base64.encodebytes(('%s:%s' % (context().args.username, context().args.password)).encode('utf8')).decode('utf8').strip() + auth = base64.encodebytes(('%s:%s' % (context().args.CONFIGURATION_AUTH_USERNAME, context().args.CONFIGURATION_AUTH_PASSWORD)).encode('utf8')).decode('utf8').strip() self.server_headers = {'Accept' : 'application/json', 'Authorization': 'Basic ' + auth} self.cache = {} @@ -42,7 +42,7 @@ def get_object(self, url): # Get list of entities as per the identifiers passed. def get_objects(self, asset_server_endpoint, ids): try: - resp = requests.get('%s/%s/?pk=%s' % (context().args.server, asset_server_endpoint, ','.join([str(id) for id in ids])), headers=self.server_headers) + resp = requests.get('%s/%s/?pk=%s' % (context().args.CONFIGURATION_AUTH_URL, asset_server_endpoint, ','.join([str(id) for id in ids])), headers=self.server_headers) if resp.status_code != 200: raise DatasourceFailure('Error when getting resources: %s' % (resp.status_code)) json_data = resp.json() @@ -55,7 +55,7 @@ def get_objects(self, asset_server_endpoint, ids): # Pulls asset data for all collection entities def get_collection(self, asset_server_endpoint): try: - resp = requests.get('%s/%s' % (context().args.server, asset_server_endpoint), headers=self.server_headers) + resp = requests.get('%s/%s' % (context().args.CONFIGURATION_AUTH_URL, asset_server_endpoint), headers=self.server_headers) if resp.status_code != 200: raise DatasourceFailure('Error when getting resources: %s' % (resp.status_code)) json_data = resp.json() @@ -69,12 +69,12 @@ def get_collection(self, asset_server_endpoint): def _cache(self, endpoint, obj): id = obj.get('pk') if id: - self.cache['%s/%s/%s/' % (context().args.server, endpoint, id)] = obj + self.cache['%s/%s/%s/' % (context().args.CONFIGURATION_AUTH_URL, endpoint, id)] = obj # To get the save point in data source. If data source doesn't have it then this function can be deleted. def get_model_state_id(self): try: - resp = requests.get('%s/model_state_id' % context().args.server, headers=self.server_headers) + resp = requests.get('%s/model_state_id' % context().args.CONFIGURATION_AUTH_URL, headers=self.server_headers) if resp.status_code != 200: return None json_data = resp.json() @@ -85,7 +85,7 @@ def get_model_state_id(self): # This function has logic to gather all information required to pull data between two save points def get_model_state_delta(self, last_model_state_id, new_model_state_id): try: - resp = requests.get('%s/delta' % context().args.server, params={'from':last_model_state_id, 'to': new_model_state_id}, headers=self.server_headers) + resp = requests.get('%s/delta' % context().args.CONFIGURATION_AUTH_URL, params={'from':last_model_state_id, 'to': new_model_state_id}, headers=self.server_headers) if resp.status_code != 200: raise DatasourceFailure('Error when trying to retrieve asset model delta: %d' % resp.status_code) delta = resp.json() diff --git a/guide-build-connectors.md b/guide-build-connectors.md index bc02c557..b948887a 100644 --- a/guide-build-connectors.md +++ b/guide-build-connectors.md @@ -17,6 +17,12 @@ Delete the server folder. This is only used to provide dummy data to demo the co Update the arguments required to connect to the target data source in the `__init__` method. The example in the reference connector uses `server`, `username` and `password`. +Make sure that env variable and destination variable are same, for e.g.: + +```self.parser.add_argument('-subscriptionID', dest='CONFIGURATION_AUTH_SUBSCRIPTION_ID', default=os.getenv('CONFIGURATION_AUTH_SUBSCRIPTION_ID',None), type=str, required=False, help='Subscription ID for the data source account')``` + +In above case `CONFIGURATION_AUTH_SUBSCRIPTION_ID` is same for `dest` and env variable + `get_schema_extension` method is `app.py` has an example how to extend [core CAR schema](https://github.com/IBM/cp4s-car-schema/blob/master/doc/generated/importSchema_v2.json). If the connector **doesn't need to extend the schema `get_schema_extension` method should be deleted**. If connector needs to extend the schema more information can be found in [schema-extension](./schema-extension.md) doc.