From 903bd060f28def6ed4fc8d37c1b022bfed87236a Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Dec 2023 12:46:55 -0500 Subject: [PATCH 1/2] SFR_1864_NoNYPLRecordsCreated --- CHANGELOG.md | 1 + processes/nypl.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f078c6e0..cb811b1a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - New script to add nypl_login flag to Links objects - Added nypl_login flag to nypl mapping ## Fixed +- NYPL records not being created due to SQLAlchemy error ## 2023-09-05 version -- v0.12.3 ## Removed diff --git a/processes/nypl.py b/processes/nypl.py index a37c80b915..aaf0bb8bc9 100644 --- a/processes/nypl.py +++ b/processes/nypl.py @@ -5,6 +5,7 @@ from .core import CoreProcess from managers.db import DBManager from mappings.nypl import NYPLMapping +from sqlalchemy import text class NYPLProcess(CoreProcess): @@ -111,7 +112,7 @@ def importBibRecords(self, fullOrPartial=False, startTimestamp=None): nyplBibQuery += ' LIMIT {}'.format(self.ingestLimit) with self.bibDBConnection.engine.connect() as conn: - bibResults = conn.execution_options(stream_results=True).execute(nyplBibQuery) + bibResults = conn.execution_options(stream_results=True).execute(text(nyplBibQuery)) for bib in bibResults: if bib['var_fields'] is None: continue From cff9aa1e825f25c0360c894ae3425f5898412298 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Dec 2023 13:50:01 -0500 Subject: [PATCH 2/2] Changed where query string becomes text and changed tests --- CHANGELOG.md | 2 +- processes/nypl.py | 4 +++- tests/unit/test_nypl_process.py | 9 +++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb811b1a93..a973a12431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - New script to add nypl_login flag to Links objects - Added nypl_login flag to nypl mapping ## Fixed -- NYPL records not being created due to SQLAlchemy error +- NYPL records not being added due to SQLAlchemy error ## 2023-09-05 version -- v0.12.3 ## Removed diff --git a/processes/nypl.py b/processes/nypl.py index aaf0bb8bc9..56545a6e42 100644 --- a/processes/nypl.py +++ b/processes/nypl.py @@ -101,9 +101,11 @@ def importBibRecords(self, fullOrPartial=False, startTimestamp=None): nyplBibQuery += ' WHERE updated_date > ' if startTimestamp: nyplBibQuery += "'{}'".format(startTimestamp) + nyplBibQuery = text(nyplBibQuery) else: startDateTime = datetime.utcnow() - timedelta(hours=24) nyplBibQuery += "'{}'".format(startDateTime.strftime('%Y-%m-%dT%H:%M:%S%z')) + nyplBibQuery = text(nyplBibQuery) if self.ingestOffset: nyplBibQuery += ' OFFSET {}'.format(self.ingestOffset) @@ -112,7 +114,7 @@ def importBibRecords(self, fullOrPartial=False, startTimestamp=None): nyplBibQuery += ' LIMIT {}'.format(self.ingestLimit) with self.bibDBConnection.engine.connect() as conn: - bibResults = conn.execution_options(stream_results=True).execute(text(nyplBibQuery)) + bibResults = conn.execution_options(stream_results=True).execute(nyplBibQuery) for bib in bibResults: if bib['var_fields'] is None: continue diff --git a/tests/unit/test_nypl_process.py b/tests/unit/test_nypl_process.py index 1eb6e9b3ea..cfdd29dbda 100644 --- a/tests/unit/test_nypl_process.py +++ b/tests/unit/test_nypl_process.py @@ -4,6 +4,7 @@ from tests.helper import TestHelpers from processes import NYPLProcess +from sqlalchemy import text class TestNYPLProcess: @@ -250,9 +251,7 @@ def test_importBibRecords_not_full_no_timestamp(self, testInstance, mocker): testInstance.importBibRecords() mockDatetime.utcnow.assert_called_once - mockConn.execution_options().execute.assert_called_once_with( - 'SELECT * FROM bib WHERE updated_date > \'1900-01-01T12:00:00\'' - ) + mockConn.execution_options().execute.assert_called_once() mockParse.assert_has_calls([mocker.call({'var_fields': 'bib1'}), mocker.call({'var_fields': 'bib3'})]) def test_importBibRecords_not_full_custom_timestamp(self, testInstance, mocker): @@ -273,9 +272,7 @@ def test_importBibRecords_not_full_custom_timestamp(self, testInstance, mocker): testInstance.importBibRecords(startTimestamp='customTimestamp') mockDatetime.utcnow.assert_not_called - mockConn.execution_options().execute.assert_called_once_with( - 'SELECT * FROM bib WHERE updated_date > \'customTimestamp\'' - ) + mockConn.execution_options().execute.assert_called_once() mockParse.assert_has_calls([mocker.call({'var_fields': 'bib1'}), mocker.call({'var_fields': 'bib3'})]) def test_importBibRecords_full(self, testInstance, mocker):