From bd3c774268a81867cebced3e72a819d78edec002 Mon Sep 17 00:00:00 2001 From: Kyle Villegas <86266231+kylevillegas93@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:02:49 -0500 Subject: [PATCH] SFR-2478: Make MUSE urls constant (#519) --- config/example.yaml | 3 --- config/production.yaml | 4 ---- config/qa.yaml | 4 ---- processes/ingest/muse.py | 10 ++++------ tests/helper.py | 2 -- tests/unit/processes/ingest/test_muse_process.py | 4 ++-- 6 files changed, 6 insertions(+), 21 deletions(-) diff --git a/config/example.yaml b/config/example.yaml index 95ee0666b5..c0215b3c88 100644 --- a/config/example.yaml +++ b/config/example.yaml @@ -55,9 +55,6 @@ GITHUB_API_KEY: xxx # Bardo CCE API URL BARDO_CCE_API: xxx -# Project MUSE MARC endpoint -MUSE_MARC_URL: xxx - # Allowed sources of CORS requests to proxy endpoint API_PROXY_CORS_ALLOWED: (?:Source1:Source2) diff --git a/config/production.yaml b/config/production.yaml index e8c68dd59b..65bedd3ccb 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -64,10 +64,6 @@ DRB_API_PORT: '80' # Bardo CCE API URL BARDO_CCE_API: http://sfr-c-ecsal-14v3injrieoy5-258691445.us-east-1.elb.amazonaws.com/search/ -# Project MUSE MARC endpoint -MUSE_MARC_URL: https://about.muse.jhu.edu/lib/metadata?format=marc&content=book&include=oa&filename=open_access_books&no_auth=1 -MUSE_CSV_URL: https://about.muse.jhu.edu/static/org/local/holdings/muse_book_metadata.csv - # Google Books API # GOOGLE_BOOKS_KEY must be configured as a secret diff --git a/config/qa.yaml b/config/qa.yaml index 976fee13f5..7f7dc4d2f3 100644 --- a/config/qa.yaml +++ b/config/qa.yaml @@ -64,10 +64,6 @@ DRB_API_PORT: '80' # Bardo CCE API URL BARDO_CCE_API: http://sfr-c-ecsal-14v3injrieoy5-258691445.us-east-1.elb.amazonaws.com/search/ -# Project MUSE MARC endpoint -MUSE_MARC_URL: https://about.muse.jhu.edu/lib/metadata?format=marc&content=book&include=oa&filename=open_access_books&no_auth=1 -MUSE_CSV_URL: https://about.muse.jhu.edu/static/org/local/holdings/muse_book_metadata.csv - # Google Books API # GOOGLE_BOOKS_KEY must be configured as a secret diff --git a/processes/ingest/muse.py b/processes/ingest/muse.py index 731d45b6cf..7d20c51143 100644 --- a/processes/ingest/muse.py +++ b/processes/ingest/muse.py @@ -17,6 +17,8 @@ class MUSEProcess(CoreProcess): + MARC_URL = 'https://about.muse.jhu.edu/lib/metadata?format=marc&content=book&include=oa&filename=open_access_books&no_auth=1' + MARC_CSV_URL = 'https://about.muse.jhu.edu/static/org/local/holdings/muse_book_metadata.csv' MUSE_ROOT_URL = 'https://muse.jhu.edu' def __init__(self, *args): @@ -115,10 +117,8 @@ def importMARCRecords(self, full=False, startTimestamp=None, recID=None): logger.debug(e) def downloadMARCRecords(self): - marcURL = os.environ['MUSE_MARC_URL'] - try: - museResponse = requests.get(marcURL, stream=True, timeout=30) + museResponse = requests.get(MUSEProcess.MARC_URL, stream=True, timeout=30) museResponse.raise_for_status() except(ReadTimeout, HTTPError) as e: logger.error('Unable to load MUSE MARC records') @@ -132,10 +132,8 @@ def downloadMARCRecords(self): return BytesIO(content) def downloadRecordUpdates(self): - marcCSVURL = os.environ['MUSE_CSV_URL'] - try: - csvResponse = requests.get(marcCSVURL, stream=True, timeout=30) + csvResponse = requests.get(MUSEProcess.MARC_CSV_URL, stream=True, timeout=30) csvResponse.raise_for_status() except(ReadTimeout, HTTPError) as e: logger.error('Unable to load MUSE CSV records') diff --git a/tests/helper.py b/tests/helper.py index 9bafc3da48..29e3702a61 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -42,8 +42,6 @@ class TestHelpers: 'NYPL_API_CLIENT_TOKEN_URL': 'test_api_token_url', 'GITHUB_API_KEY': 'test_github_key', 'BARDO_CCE_API': 'test_cce_url', - 'MUSE_MARC_URL': 'test_muse_url', - 'MUSE_CSV_URL': 'test_muse_csv', 'ENVIRONMENT': 'test' } diff --git a/tests/unit/processes/ingest/test_muse_process.py b/tests/unit/processes/ingest/test_muse_process.py index 0ed03495ec..6117ded3d7 100644 --- a/tests/unit/processes/ingest/test_muse_process.py +++ b/tests/unit/processes/ingest/test_muse_process.py @@ -187,7 +187,7 @@ def test_downloadMARCRecords_success(self, testProcess, mocker): testRecords = testProcess.downloadMARCRecords() assert testRecords.read() == b'marc' - mockGet.assert_called_once_with('test_muse_url', stream=True, timeout=30) + mockGet.assert_called_once_with(MUSEProcess.MARC_URL, stream=True, timeout=30) def test_downloadMARCRecords_failure(self, testProcess, mocker): mockGet = mocker.patch.object(requests, 'get') @@ -210,7 +210,7 @@ def test_downloadRecordUpdates_success(self, testProcess, testBookCSV, mocker): testProcess.downloadRecordUpdates() assert testProcess.updateDates == {'row1': datetime(2020, 1, 1), 'row3': datetime(2020, 1, 1)} - mockGet.assert_called_once_with('test_muse_csv', stream=True, timeout=30) + mockGet.assert_called_once_with(MUSEProcess.MARC_CSV_URL, stream=True, timeout=30) mockResp.iter_lines.assert_called_once_with(decode_unicode=True) mockCSV.assert_called_once_with('testFile', skipinitialspace=True)