External resolver

cdci_data_analysis/analysis/drupal_helper.py

@@ -21,6 +21,7 @@
 from enum import Enum, auto
 from astropy.coordinates import SkyCoord, Angle
 from astropy import units as u
+import xml.etree.ElementTree as ET
 
 from cdci_data_analysis.analysis import tokenHelper
 from ..analysis.exceptions import RequestNotUnderstood, InternalError, RequestNotAuthorized
@@ -551,11 +552,14 @@ def post_content_to_gallery(decoded_token,
         if update_astro_entity:
             auto_update = kwargs.pop('auto_update', 'False') == 'True'
             if auto_update is True:
-                name_resolver_url = disp_conf.name_resolver_url
+                local_name_resolver_url = disp_conf.local_name_resolver_url
+                external_name_resolver_url = disp_conf.external_name_resolver_url
                 entities_portal_url = disp_conf.entities_portal_url
-                resolved_obj = resolve_name(name_resolver_url=name_resolver_url,
+                resolved_obj = resolve_name(local_name_resolver_url=local_name_resolver_url,
+                                            external_name_resolver_url=external_name_resolver_url,
                                             entities_portal_url=entities_portal_url,
-                                            name=src_name)
+                                            name=src_name,
+                                            sentry_dsn=sentry_dsn)
                 if resolved_obj is not None:
                     msg = ''
                     if 'message' in resolved_obj:
@@ -1488,39 +1492,98 @@ def check_matching_coords(source_1_name, source_1_coord_ra, source_1_coord_dec,
     return False
 
 
-def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name: str = None):
+def resolve_name(local_name_resolver_url: str, external_name_resolver_url: str, entities_portal_url: str = None, name: str = None, sentry_dsn=None):
     resolved_obj = {}
     if name is not None:
         quoted_name = urllib.parse.quote(name.strip())
-        res = requests.get(name_resolver_url.format(quoted_name))
+        local_name_resolver_url_formatted = local_name_resolver_url.format(quoted_name)
+        try:
+            res = requests.get(local_name_resolver_url_formatted)
+            if res.status_code == 200:
+                returned_resolved_obj = res.json()
+                if 'success' in returned_resolved_obj:
+                    resolved_obj['name'] = name.replace('_', ' ')
+                    if returned_resolved_obj['success']:
+                        logger.info(f"object {name} successfully resolved")
+                        if 'ra' in returned_resolved_obj:
+                            resolved_obj['RA'] = float(returned_resolved_obj['ra'])
+                        if 'dec' in returned_resolved_obj:
+                            resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
+                        if 'object_ids' in returned_resolved_obj:
+                            resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
+                        if 'object_type' in returned_resolved_obj:
+                            resolved_obj['object_type'] = returned_resolved_obj['object_type']
+                        resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+                        resolved_obj['message'] = f'{name} successfully resolved'
+                    elif not returned_resolved_obj['success']:
+                        logger.info(f"resolution of the object {name} unsuccessful")
+                        resolved_obj['message'] = f'{name} could not be resolved'
+            else:
+                logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the local resolver.\n"
+                               f"The request lead to the error {res.text}, "
+                               "this might be due to an error in the url or the service "
+                               "requested is currently not available. The external resolver will be used.")
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the local resolver. '
+                                           f'URL: {local_name_resolver_url_formatted} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}')
+        except (ConnectionError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout) as e:
+            logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                           f'using the url: {local_name_resolver_url_formatted}. Exception details: {str(e)}')
+            if sentry_dsn is not None:
+                sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
+                                       f'URL: {local_name_resolver_url_formatted} '
+                                       f"Exception details: {str(e)}")
+        res = requests.get(external_name_resolver_url.format(quoted_name))
         if res.status_code == 200:
-            returned_resolved_obj = res.json()
-            if 'success' in returned_resolved_obj:
-                resolved_obj['name'] = name.replace('_', ' ')
-                if returned_resolved_obj['success']:
-                    logger.info(f"object {name} successfully resolved")
-                    if 'ra' in returned_resolved_obj:
-                        resolved_obj['RA'] = float(returned_resolved_obj['ra'])
-                    if 'dec' in returned_resolved_obj:
-                        resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
-                    if 'object_ids' in returned_resolved_obj:
-                        resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
-                    if 'object_type' in returned_resolved_obj:
-                        resolved_obj['object_type'] = returned_resolved_obj['object_type']
-                    resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
-                    resolved_obj['message'] = f'{name} successfully resolved'
-                elif not returned_resolved_obj['success']:
-                    logger.info(f"resolution of the object {name} unsuccessful")
+            root = ET.fromstring(res.text)
+            resolved_obj['name'] = name.replace('_', ' ')
+            resolver_tag = root.find('.//Resolver')
+            if resolver_tag is not None:
+                ra_tag = resolver_tag.find('.//jradeg')
+                dec_tag = resolver_tag.find('.//jdedeg')
+                if ra_tag is None or dec_tag is None:
+                    info_tag = root.find('.//INFO')
                     resolved_obj['message'] = f'{name} could not be resolved'
+                    if info_tag is not None:
+                        message_info = info_tag.text
+                        resolved_obj['message'] += f': {message_info}'
+                else:
+                    resolved_obj['RA'] = float(ra_tag.text)
+                    resolved_obj['DEC'] = float(dec_tag.text)
+                    resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
+            else:
+                warning_msg = ("There seems to be some problem in completing the request for the resolution of the object"
+                               f" \"{name}\" using the external resolver.")
+                resolved_obj['message'] = f'{name} could not be resolved'
+                info_tag = root.find('.//INFO')
+                if info_tag is not None:
+                    warning_msg += (f"The request lead to the error {info_tag.text}, "
+                                   "this might be due to an error in the name of the object that ha been provided.")
+                    resolved_obj['message'] += f': {info_tag.text}'
+                logger.warning(warning_msg)
+                if sentry_dsn is not None:
+                    sentry.capture_message(f'Failed to resolve object "{name}" using the remote resolver. '
+                                           f'URL: {local_name_resolver_url.format(quoted_name)} '
+                                           f'Status Code: {res.status_code} '
+                                           f'Response: {res.text}'
+                                           f"Info returned from the resolver: {resolved_obj['message']}")
         else:
-            logger.warning(f"there seems to be some problem in completing the request for the resolution of the object: {name}\n"
-                           f"the request lead to the error {res.text}, "
+            logger.warning("There seems to be some problem in completing the request for the resolution of the object"
+                           f" \"{name}\" using the external resolver.\n"
+                           f"The request lead to the error {res.text}, "
                            "this might be due to an error in the url or the service "
-                           "requested is currently not available, "
-                           "please check your request and try to issue it again")
-            raise InternalError('issue when performing a request to the local resolver',
-                                status_code=500,
-                                payload={'drupal_helper_error_message': res.text})
+                           "requested is currently not available. The object could not be resolved.")
+            if sentry_dsn is not None:
+                sentry.capture_message(f'Failed to resolve object "{name}" using the remote resolver. '
+                                       f'URL: {local_name_resolver_url.format(quoted_name)} '
+                                       f'Status Code: {res.status_code} '
+                                       f'Response: {res.text}')
+            resolved_obj['message'] = f'{name} could not be resolved: {res.text}'
     return resolved_obj
 
 

cdci_data_analysis/analysis/ivoa_helper.py

@@ -0,0 +1,93 @@
+import os.path
+
+from queryparser.adql import ADQLQueryTranslator
+from queryparser.exceptions import QuerySyntaxError
+
+from psycopg2 import connect, DatabaseError
+
+from ..flask_app.sentry import sentry
+from ..app_logging import app_logging
+
+logger = app_logging.getLogger('ivoa_helper')
+
+
+def parse_adql_query(query):
+    try:
+        adt = ADQLQueryTranslator(query)
+
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=adt.to_postgresql()
+        )
+
+    except QuerySyntaxError as qe:
+        logger.error(f'Error parsing ADQL query: {qe}')
+        output_obj = dict(
+            mysql_query=None,
+            psql_query=None
+        )
+    return output_obj
+
+
+def run_ivoa_query(query, **kwargs):
+    parsed_query_obj = parse_adql_query(query)
+
+    # TODO use a specific dedicated table and schema to refer to the product_gallery DB ?
+    # tables = parsed_query_obj.get('tables', [])
+    # if len(tables) == 1 and tables[0] == 'product_gallery':
+    logger.info('Performing query on the product_gallery')
+    vo_psql_pg_host = kwargs.get('vo_psql_pg_host', None)
+    vo_psql_pg_user = kwargs.get('vo_psql_pg_user', None)
+    vo_psql_pg_password = kwargs.get('vo_psql_pg_password', None)
+    vo_psql_pg_db = kwargs.get('vo_psql_pg_db', None)
+    product_gallery_url = kwargs.get('product_gallery_url', None)
+    result_list = run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                                      vo_psql_pg_host=vo_psql_pg_host,
+                                                      vo_psql_pg_user=vo_psql_pg_user,
+                                                      vo_psql_pg_password=vo_psql_pg_password,
+                                                      vo_psql_pg_db=vo_psql_pg_db,
+                                                      product_gallery_url=product_gallery_url)
+    return result_list
+
+
+def run_ivoa_query_from_product_gallery(parsed_query_obj,
+                                        vo_psql_pg_host,
+                                        vo_psql_pg_user,
+                                        vo_psql_pg_password,
+                                        vo_psql_pg_db,
+                                        product_gallery_url=None
+                                        ):
+    result_list = []
+
+    try:
+        with connect(
+            host=vo_psql_pg_host,
+            database=vo_psql_pg_db,
+            user=vo_psql_pg_user,
+            password=vo_psql_pg_password
+        ) as connection:
+            db_query = parsed_query_obj.get('psql_query')
+            with connection.cursor() as cursor:
+                cursor.execute(db_query)
+                for row in cursor:
+                    list_row = list(row)
+                    if product_gallery_url is not None:
+                        for index, value in enumerate(list_row):
+                            description = cursor.description[index]
+                            if description.name in {'path', 'path_alias'} and value is not None and isinstance(value, str):
+                                if value.startswith('/'):
+                                    value = value[1:]
+                                list_row[index] = os.path.join(product_gallery_url, value)
+                    result_list.append(list_row)
+
+    except (Exception, DatabaseError) as e:
+        sentry.capture_message(f"Error when querying to the Postgresql server: {str(e)}")
+        logger.error(f"Error when querying to the Postgresql server: {str(e)}")
+
+    finally:
+        if connection is not None:
+            cursor.close()
+            connection.close()
+            logger.info('Database connection closed')
+
+    return result_list

cdci_data_analysis/config_dir/conf_env.yml.example

@@ -115,11 +115,24 @@ dispatcher:
         product_gallery_secret_key: PRODUCT_GALLERY_SECRET_KEY
         # timezone used within the drupal configuration, these two values have to be always aligned
         product_gallery_timezone: PRODUCT_GALLERY_SECRET_KEY
-        # url of the name resolver
-        name_resolver_url: NAME_RESOLVER_URL
+        # url of the local name resolver
+        local_name_resolver_url: NAME_RESOLVER_URL
+        # url of the external name resolver
+        external_name_resolver_url: NAME_RESOLVER_URL
         # url of the online catalog for astrophysical entities
         entities_portal_url: ENTITIES_PORTAL_URL
         # url for the conversion of a given time, in UTC format, to the correspondent REVNUM
         converttime_revnum_service_url: COVERTTIME_REVNUM_SERVICE_URL
 
-
+    # virtual observatory related configurations (eg mysql credentials)
+    vo_options:
+        # mysql credentials
+        vo_mysql_pg_host: MYSQL_PG_HOST
+        vo_mysql_pg_user: MYSQL_PG_USER
+        vo_mysql_pg_password: MYSQL_PG_PASSWORD
+        vo_mysql_pg_db: MYSQL_PG_DB
+        # postgresql credentials
+        vo_psql_pg_host: PSQL_PG_HOST
+        vo_psql_pg_user: PSQL_PG_USER
+        vo_psql_pg_password: PSQL_PG_PASSWORD
+        vo_psql_pg_db: PSQL_PG_DB

cdci_data_analysis/configurer.py

@@ -260,12 +260,23 @@ def __init__(self, cfg_dict, origin=None):
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_secret_key', None),
                                      disp_dict.get('product_gallery_options', {}).get('product_gallery_timezone',
                                                                                       "Europe/Zurich"),
-                                     disp_dict.get('product_gallery_options', {}).get('name_resolver_url', 'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('local_name_resolver_url',
+                                                                                      'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
+                                     disp_dict.get('product_gallery_options', {}).get('external_name_resolver_url',
+                                                                                      'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}'),
                                      disp_dict.get('product_gallery_options', {}).get('entities_portal_url', 'http://cdsportal.u-strasbg.fr/?target={}'),
                                      disp_dict.get('product_gallery_options', {}).get('converttime_revnum_service_url', 'https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM'),
                                      disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
                                      disp_dict.get('renku_options', {}).get('renku_base_project_url', None),
                                      disp_dict.get('renku_options', {}).get('ssh_key_path', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_mysql_pg_db', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_host', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_user', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_password', None),
+                                     disp_dict.get('vo_options', {}).get('vo_psql_pg_db', None)
                                      )
 
         # not used?
@@ -338,12 +349,21 @@ def set_conf_dispatcher(self,
                             product_gallery_url,
                             product_gallery_secret_key,
                             product_gallery_timezone,
-                            name_resolver_url,
+                            local_name_resolver_url,
+                            external_name_resolver_url,
                             entities_portal_url,
                             converttime_revnum_service_url,
                             renku_gitlab_repository_url,
                             renku_base_project_url,
                             renku_gitlab_ssh_key_path,
+                            vo_mysql_pg_host,
+                            vo_mysql_pg_user,
+                            vo_mysql_pg_password,
+                            vo_mysql_pg_db,
+                            vo_psql_pg_host,
+                            vo_psql_pg_user,
+                            vo_psql_pg_password,
+                            vo_psql_pg_db
                             ):
         # Generic to dispatcher
         #print(dispatcher_url, dispatcher_port)
@@ -389,12 +409,21 @@ def set_conf_dispatcher(self,
         self.product_gallery_url = product_gallery_url
         self.product_gallery_secret_key = product_gallery_secret_key
         self.product_gallery_timezone = product_gallery_timezone
-        self.name_resolver_url = name_resolver_url
+        self.local_name_resolver_url = local_name_resolver_url
+        self.external_name_resolver_url = external_name_resolver_url
         self.entities_portal_url = entities_portal_url
         self.converttime_revnum_service_url = converttime_revnum_service_url
         self.renku_gitlab_repository_url = renku_gitlab_repository_url
         self.renku_gitlab_ssh_key_path = renku_gitlab_ssh_key_path
         self.renku_base_project_url = renku_base_project_url
+        self.vo_mysql_pg_host = vo_mysql_pg_host
+        self.vo_mysql_pg_user = vo_mysql_pg_user
+        self.vo_mysql_pg_password = vo_mysql_pg_password
+        self.vo_mysql_pg_db = vo_mysql_pg_db
+        self.vo_psql_pg_host = vo_psql_pg_host
+        self.vo_psql_pg_user = vo_psql_pg_user
+        self.vo_psql_pg_password = vo_psql_pg_password
+        self.vo_psql_pg_db = vo_psql_pg_db
 
     def get_data_serve_conf(self, instr_name):
         if instr_name in self.data_server_conf_dict.keys():