diff --git a/api/app.py b/api/app.py index a199fa37c6..35e2109803 100644 --- a/api/app.py +++ b/api/app.py @@ -46,7 +46,7 @@ def run(self): self.app.config['DEBUG'] = True self.app.run(host='0.0.0.0', port=5050) - elif 'local' in os.environ['ENVIRONMENT']: + elif 'local' in os.environ['ENVIRONMENT'] or os.environ['ENVIRONMENT'] == 'frontend-ci': logger.debug('Starting dev server on port 5050') self.app.config['ENV'] = 'development' diff --git a/api/blueprints/drbSearch.py b/api/blueprints/drbSearch.py index 0bde12172d..141d3aeb80 100644 --- a/api/blueprints/drbSearch.py +++ b/api/blueprints/drbSearch.py @@ -57,16 +57,19 @@ def query(): filtered_formats = APIUtils.formatFilters(terms) works = db_client.fetchSearchedWorks(results) + + # Depending on the version of elastic search, hits will either be an integer or a dictionary + total_hits = search_result.hits.total if isinstance(search_result.hits.total, int) else search_result.hits.total.value facets = APIUtils.formatAggregationResult(search_result.aggregations.to_dict()) paging = APIUtils.formatPagingOptions( search_page + 1, search_size, - search_result.hits.total.value + total_hits ) data_block = { - 'totalWorks': search_result.hits.total.value, + 'totalWorks': total_hits, 'works': APIUtils.formatWorkOutput( works, results, diff --git a/config/frontend-ci.yaml b/config/frontend-ci.yaml new file mode 100644 index 0000000000..cdac88d26a --- /dev/null +++ b/config/frontend-ci.yaml @@ -0,0 +1,17 @@ +LOG_LEVEL: INFO + +POSTGRES_HOST: sfr-new-metadata-production-cluster.cluster-cvy7z512hcjg.us-east-1.rds.amazonaws.com +POSTGRES_NAME: dcdw_qa +POSTGRES_PORT: '5432' + +REDIS_HOST: localhost +REDIS_PORT: '6379' + +ELASTICSEARCH_INDEX: drb_dcdw_qa +ELASTICSEARCH_SCHEME: https +ELASTICSEARCH_HOST: vpc-drb-search-production-5ptfzwisshcadbbaph35dqilva.us-east-1.es.amazonaws.com +ELASTICSEARCH_PORT: '9200' +ELASTICSEARCH_TIMEOUT: '30' + +READER_VERSION: 'v2' +API_PROXY_CORS_ALLOWED: '*' diff --git a/managers/elasticsearch.py b/managers/elasticsearch.py index 806d507405..e6fc6ff51f 100644 --- a/managers/elasticsearch.py +++ b/managers/elasticsearch.py @@ -1,10 +1,12 @@ import os +import boto3 from elasticsearch.client import IngestClient -from elasticsearch import Elasticsearch +from elasticsearch import Elasticsearch, RequestsHttpConnection from elasticsearch.helpers import bulk from elasticsearch_dsl import connections, Index from elastic_transport import ConnectionTimeout +from requests_aws4auth import AWS4Auth from model import ESWork from logger import create_log @@ -26,12 +28,17 @@ def createElasticConnection(self, scheme=None, host=None, port=None, user=None, user = user or os.environ.get('ELASTICSEARCH_USER', None) pswd = pswd or os.environ.get('ELASTICSEARCH_PSWD', None) timeout = int(os.environ.get('ELASTICSEARCH_TIMEOUT', 5)) + environment = os.environ.get('ENVIRONMENT', None) creds = '{}:{}@'.format(user, pswd) if user and pswd else '' #Allowing multple hosts for a ES connection multHosts = [] + if environment == 'frontend-ci': + self._create_ci_connection(scheme=scheme, host=host, timeout=timeout) + return + if ',' not in host: host = '{}://{}{}:{}'.format(scheme, creds, host, port) @@ -318,6 +325,25 @@ def _deleteGenerator(self, uuids): '_id': uuid } + def _create_ci_connection(self, scheme: str, host: str, timeout: int): + host = '{}://{}'.format(scheme, host) + + session = boto3.Session() + credentials = session.get_credentials() + awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, 'us-east-1', 'es', session_token=credentials.token) + + connection_config = { + 'hosts': [host], + 'timeout': timeout, + 'http_auth': awsauth, + 'connection_class': RequestsHttpConnection, + 'use_ssl': True, + 'verify_certs': True, + } + + self.client = connections.create_connection(**connection_config) + self.es = Elasticsearch(**connection_config) + @staticmethod def _splitWorkBatch(works): workCount = len(works) diff --git a/requirements.txt b/requirements.txt index f5d864dba7..054299a18c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ pyyaml==6.0 redis==4.5.4 requests==2.28.2 requests_oauthlib==1.3.1 +requests-aws4auth==1.3.1 scikit-learn==1.2.2 sqlalchemy==2.0.20 waitress==2.1.2