Skip to content

Commit

Permalink
Changed variables and most methods to snake case
Browse files Browse the repository at this point in the history
  • Loading branch information
Dmitri committed Oct 24, 2024
1 parent 59024d9 commit f806c06
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 76 deletions.
54 changes: 27 additions & 27 deletions processes/ingest/chicago_isac.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,20 @@ class ChicagoISACProcess(CoreProcess):
def __init__(self, *args):
super(ChicagoISACProcess, self).__init__(*args[:4])

self.fullImport = self.process == 'complete'
self.full_import = self.process == 'complete'

self.generateEngine()
self.createSession()

self.s3Bucket = os.environ['FILE_BUCKET']
self.createS3Client()

def run_process(self):
def runProcess(self):
with open('ingestJSONFiles/chicagoISAC_metadata.json') as f:
chicagoISACData = json.load(f)
chicago_isac_data = json.load(f)

for metaDict in chicagoISACData:
self.process_chicago_isac_record(metaDict)
for meta_dict in chicago_isac_data:
self.process_chicago_isac_record(meta_dict)

self.saveRecords()
self.commitChanges()
Expand All @@ -35,40 +35,40 @@ def run_process(self):

def process_chicago_isac_record(self, record):
try:
chicagoISACRec = ChicagoISACMapping(record)
chicagoISACRec.applyMapping()
self.store_pdf_manifest(chicagoISACRec.record)
self.addDCDWToUpdateList(chicagoISACRec)
chicago_isac_rec = ChicagoISACMapping(record)
chicago_isac_rec.applyMapping()
self.store_pdf_manifest(chicago_isac_rec.record)
self.addDCDWToUpdateList(chicago_isac_rec)

except Exception:
logger.exception(ChicagoISACError('Unable to process ISAC record'))


def store_pdf_manifest(self, record):
for link in record.has_part:
itemNo, uri, source, mediaType, flags = link[0].split('|')
item_no, uri, source, media_type, flags = link[0].split('|')

if mediaType == 'application/pdf':
recordID = record.identifiers[0].split('|')[0]
if media_type == 'application/pdf':
record_id = record.identifiers[0].split('|')[0]

manifestPath = 'manifests/{}/{}.json'.format(source, recordID)
manifestURI = 'https://{}.s3.amazonaws.com/{}'.format(
self.s3Bucket, manifestPath
manifest_path = 'manifests/{}/{}.json'.format(source, record_id)
manifest_url = 'https://{}.s3.amazonaws.com/{}'.format(
self.s3Bucket, manifest_path
)

manifestJSON = self.generate_manifest(record, uri, manifestURI)
manifest_json = self.generate_manifest(record, uri, manifest_url)

self.createManifestInS3(manifestPath, manifestJSON)
self.createManifestInS3(manifest_path, manifest_json)

linkString = '|'.join([
itemNo,
manifestURI,
link_string = '|'.join([
item_no,
manifest_url,
source,
'application/webpub+json',
flags
])

record.has_part.insert(0, linkString)
record.has_part.insert(0, link_string)
self.change_has_part_url_array_to_string(record)

break
Expand All @@ -77,27 +77,27 @@ def store_pdf_manifest(self, record):
def change_has_part_url_array_to_string(record):
for i in range(1, len(record.has_part)):
if len(record.has_part[i]) > 1:
urlArray = record.has_part[i]
url_array = record.has_part[i]
record.has_part.pop(i)
for elem in urlArray:
for elem in url_array:
record.has_part.append(elem)
else:
record.has_part[i] = ''.join(record.has_part[i])

@staticmethod
def generate_manifest(record, sourceURI, manifestURI):
manifest = WebpubManifest(sourceURI, 'application/pdf')
def generate_manifest(record, source_url, manifest_url):
manifest = WebpubManifest(source_url, 'application/pdf')

manifest.addMetadata(
record,
conformsTo=os.environ['WEBPUB_PDF_PROFILE']
)

manifest.addChapter(sourceURI, record.title)
manifest.addChapter(source_url, record.title)

manifest.links.append({
'rel': 'self',
'href': manifestURI,
'href': manifest_url,
'type': 'application/webpub+json'
})

Expand Down
98 changes: 49 additions & 49 deletions tests/unit/processes/ingest/test_chicago_isac_process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,91 +14,91 @@ def teardown_class(cls):
TestHelpers.clearEnvVars()

@pytest.fixture
def testProcess(self, mocker):
def test_process(self, mocker):
class TestISAC(ChicagoISACProcess):
def __init__(self):
self.s3Bucket = 'test_aws_bucket'
self.s3Client = mocker.MagicMock(s3Client='testS3Client')
self.session = mocker.MagicMock(session='testSession')
self.records = mocker.MagicMock(record='testRecord')
self.batchSize = mocker.MagicMock(batchSize='testBatchSize')
self.s3_client = mocker.MagicMock(s3_client='test_s3_client')
self.session = mocker.MagicMock(session='test_session')
self.records = mocker.MagicMock(record='test_record')
self.batch_size = mocker.MagicMock(batch_size='test_batch_size')

return TestISAC()

def test_runProcess(self, testProcess, mocker):
runMocks = mocker.patch.multiple(
def test_run_process(self, test_process, mocker):
run_mocks = mocker.patch.multiple(
ChicagoISACProcess,
saveRecords=mocker.DEFAULT,
commitChanges=mocker.DEFAULT
)

testProcess.run_process()
test_process.runProcess()

runMocks['saveRecords'].assert_called_once()
runMocks['commitChanges'].assert_called_once()
run_mocks['saveRecords'].assert_called_once()
run_mocks['commitChanges'].assert_called_once()


def test_processChicagoISACRecord_success(self, testProcess, mocker):
def test_process_chicago_isac_record_success(self, test_process, mocker):
processMocks = mocker.patch.multiple(ChicagoISACProcess,
store_pdf_manifest=mocker.DEFAULT,
addDCDWToUpdateList=mocker.DEFAULT
)

mockMapping = mocker.MagicMock(record='testRecord')
mockMapper = mocker.patch('processes.ingest.chicago_isac.ChicagoISACMapping')
mockMapper.return_value = mockMapping
mock_mapping = mocker.MagicMock(record='test_record')
mock_mapper = mocker.patch('processes.ingest.chicago_isac.ChicagoISACMapping')
mock_mapper.return_value = mock_mapping

testProcess.process_chicago_isac_record(mockMapping)
test_process.process_chicago_isac_record(mock_mapping)

mockMapping.applyMapping.assert_called_once()
mock_mapping.applyMapping.assert_called_once()

processMocks['store_pdf_manifest'].assert_called_once_with('testRecord')
processMocks['addDCDWToUpdateList'].assert_called_once_with(mockMapping)
processMocks['store_pdf_manifest'].assert_called_once_with('test_record')
processMocks['addDCDWToUpdateList'].assert_called_once_with(mock_mapping)

def test_processChicagoISACRecord_error(self, mocker):
def test_process_chicago_isac_record_error(self, mocker):

mockMapper = mocker.patch('processes.ingest.chicago_isac.ChicagoISACMapping')
mockMapper.side_effect = MappingError('testError')
mock_mapper = mocker.patch('processes.ingest.chicago_isac.ChicagoISACMapping')
mock_mapper.side_effect = MappingError('testError')

assert pytest.raises(MappingError)

def test_store_pdf_manifest(self, testProcess, mocker):
mockRecord = mocker.MagicMock(identifiers=['1|isac'])
mockRecord.has_part = [
['1|testURI|isac|application/pdf|{}',
'2|testURIOther|isac|application/pdf|{}'],
def test_store_pdf_manifest(self, test_process, mocker):
mock_record = mocker.MagicMock(identifiers=['1|isac'])
mock_record.has_part = [
['1|test_url|isac|application/pdf|{}',
'2|test_url_other|isac|application/pdf|{}'],
]

mockGenerateMan = mocker.patch.object(ChicagoISACProcess, 'generate_manifest')
mockGenerateMan.return_value = 'testJSON'
mockCreateMan = mocker.patch.object(ChicagoISACProcess, 'createManifestInS3')
mock_generate_man = mocker.patch.object(ChicagoISACProcess, 'generate_manifest')
mock_generate_man.return_value = 'test_json'
mock_create_man = mocker.patch.object(ChicagoISACProcess, 'createManifestInS3')

testProcess.store_pdf_manifest(mockRecord)
test_process.store_pdf_manifest(mock_record)

testManifestURI = 'https://test_aws_bucket.s3.amazonaws.com/manifests/isac/1.json'
assert mockRecord.has_part[0] == '1|{}|isac|application/webpub+json|{{}}'.format(testManifestURI)
test_manifest_url = 'https://test_aws_bucket.s3.amazonaws.com/manifests/isac/1.json'
assert mock_record.has_part[0] == '1|{}|isac|application/webpub+json|{{}}'.format(test_manifest_url)

mockGenerateMan.assert_called_once_with(mockRecord, 'testURI', testManifestURI)
mockCreateMan.assert_called_once_with('manifests/isac/1.json', 'testJSON')
mock_generate_man.assert_called_once_with(mock_record, 'test_url', test_manifest_url)
mock_create_man.assert_called_once_with('manifests/isac/1.json', 'test_json')

def test_createManifestInS3(self, testProcess, mocker):
mockPut = mocker.patch.object(ChicagoISACProcess, 'putObjectInBucket')
def test_create_manifest_in_S3(self, test_process, mocker):
mock_put = mocker.patch.object(ChicagoISACProcess, 'putObjectInBucket')

testProcess.createManifestInS3('testPath', '{"data": "testJSON"}')
test_process.createManifestInS3('test_path', '{"data": "test_json"}')

mockPut.assert_called_once_with(b'{"data": "testJSON"}', 'testPath', 'test_aws_bucket')
mock_put.assert_called_once_with(b'{"data": "test_json"}', 'test_path', 'test_aws_bucket')

def test_generateManifest(self, mocker):
mockManifest = mocker.MagicMock(links=[])
mockManifest.toJson.return_value = 'testJSON'
mockManifestConstructor = mocker.patch('processes.ingest.chicago_isac.WebpubManifest')
mockManifestConstructor.return_value = mockManifest
def test_generate_manifest(self, mocker):
mock_manifest = mocker.MagicMock(links=[])
mock_manifest.toJson.return_value = 'test_json'
mock_manifest_constructor = mocker.patch('processes.ingest.chicago_isac.WebpubManifest')
mock_manifest_constructor.return_value = mock_manifest

mockRecord = mocker.MagicMock(title='testTitle')
testManifest = ChicagoISACProcess.generate_manifest(mockRecord, 'sourceURI', 'manifestURI')
mock_record = mocker.MagicMock(title='test_title')
test_manifest = ChicagoISACProcess.generate_manifest(mock_record, 'source_url', 'manifest_url')

assert testManifest == 'testJSON'
assert mockManifest.links[0] == {'rel': 'self', 'href': 'manifestURI', 'type': 'application/webpub+json'}
assert test_manifest == 'test_json'
assert mock_manifest.links[0] == {'rel': 'self', 'href': 'manifest_url', 'type': 'application/webpub+json'}

mockManifest.addMetadata.assert_called_once_with(mockRecord, conformsTo='test_profile_uri')
mockManifest.addChapter.assert_called_once_with('sourceURI', 'testTitle')
mock_manifest.addMetadata.assert_called_once_with(mock_record, conformsTo='test_profile_uri')
mock_manifest.addChapter.assert_called_once_with('source_url', 'test_title')

0 comments on commit f806c06

Please sign in to comment.