From 05c0bc72ab5954af8afc36f7fd576f05a680191d Mon Sep 17 00:00:00 2001 From: Douglas Cerna Date: Mon, 21 Aug 2023 14:34:42 -0600 Subject: [PATCH] Upgrade Django to 3.2 * Add django-upgrade to pre-commit config * Run pre-commit * Upgrade requirements * Allow compilemessages to write *.mo files * Update test workflow * Replace custom BlobTextField with Django's BinaryField * Fix tests * Make migrations * Update submodules --- .github/workflows/test.yml | 4 +- .pre-commit-config.yaml | 5 + hack/Dockerfile | 6 + .../submodules/archivematica-acceptance-tests | 2 +- hack/submodules/archivematica-storage-service | 2 +- requirements-dev.txt | 34 ++-- requirements.in | 22 +-- requirements.txt | 31 ++-- .../archivematicaCreateMETSMetadataXML.py | 2 +- .../archivematicaCreateMETSReingest.py | 11 +- ...rchivematicaCreateMETSRightsDspaceMDRef.py | 12 +- .../archivematicaCreateMETSTrim.py | 14 +- .../lib/clientScripts/assign_file_uuids.py | 12 +- .../assign_uuids_to_directories.py | 2 +- .../lib/clientScripts/change_object_names.py | 12 +- .../check_for_service_directory.py | 12 +- .../lib/clientScripts/create_mets_v2.py | 33 ++-- .../create_sip_from_transfer_objects.py | 16 +- .../lib/clientScripts/create_transfer_mets.py | 16 +- .../clientScripts/dip_generation_helper.py | 9 +- .../lib/clientScripts/extract_contents.py | 19 ++- .../clientScripts/identify_dspace_files.py | 2 +- .../lib/clientScripts/load_labels_from_csv.py | 2 +- .../load_premis_events_from_xml.py | 14 +- ...ization_create_metadata_and_restructure.py | 8 +- ..._normalization_move_access_files_to_dip.py | 2 +- src/MCPClient/lib/clientScripts/normalize.py | 24 +-- .../lib/clientScripts/normalize_report.py | 10 +- .../lib/clientScripts/parse_dataverse_mets.py | 8 +- .../lib/clientScripts/parse_mets_to_db.py | 2 +- .../lib/clientScripts/pid_declaration.py | 27 ++-- .../lib/clientScripts/policy_check.py | 3 +- .../retry_normalize_remove_normalized.py | 4 +- .../lib/clientScripts/rights_from_csv.py | 3 +- .../store_file_modification_dates.py | 2 +- .../clientScripts/update_size_and_checksum.py | 4 +- .../lib/clientScripts/upload_archivesspace.py | 5 +- src/MCPClient/lib/clientScripts/verify_aip.py | 8 +- .../lib/ensure_no_mutable_globals.py | 1 - .../custom_structmaps/model/files.json | 88 +++++------ .../tests/fixtures/dataverse_sip.json | 44 +++--- .../fixtures/files-transfer-unicode.json | 16 +- .../tests/fixtures/files-transfer.json | 8 +- src/MCPClient/tests/fixtures/files.json | 20 +-- .../tests/fixtures/reingest-preservation.json | 4 +- ...test_archivematicaCreateMETSMetadataXML.py | 2 +- src/MCPClient/tests/test_assign_file_uuids.py | 146 ++++++++++++++++++ .../tests/test_assign_uuids_to_directories.py | 46 ++++++ src/MCPClient/tests/test_create_mets_v2.py | 8 +- .../tests/test_create_transfer_mets.py | 12 +- src/MCPClient/tests/test_filename_change.py | 42 ++--- .../tests/test_load_premis_events_from_xml.py | 6 +- src/MCPClient/tests/test_parse_dataverse.py | 12 +- src/MCPClient/tests/test_parse_mets_to_db.py | 12 +- src/MCPClient/tests/test_pid_components.py | 46 +++--- .../tests/test_store_file_modification.py | 4 +- src/MCPClient/tests/test_upload_qubit.py | 2 +- src/MCPClient/tests/test_verify_checksum.py | 4 +- src/MCPServer/lib/server/packages.py | 12 +- src/MCPServer/tests/test_package.py | 38 ++--- src/MCPServer/tests/test_workflow.py | 8 +- .../lib/databaseFunctions.py | 4 +- src/archivematicaCommon/lib/dicts.py | 8 +- .../lib/elasticSearchFunctions.py | 4 +- src/archivematicaCommon/lib/fileOperations.py | 18 ++- .../fixtures/test_database_functions.json | 8 +- .../tests/test_database_functions.py | 16 +- src/archivematicaCommon/tests/test_dicts.py | 71 +++++---- src/dashboard/src/components/access/urls.py | 27 ++-- .../src/components/accounts/__init__.py | 1 - src/dashboard/src/components/accounts/urls.py | 24 +-- .../src/components/accounts/validators.py | 2 +- .../src/components/accounts/views.py | 2 +- .../src/components/administration/forms.py | 2 +- .../administration/forms_dip_upload.py | 2 +- .../src/components/administration/urls.py | 63 ++++---- .../src/components/administration/views.py | 2 +- .../administration/views_dip_upload.py | 2 +- .../administration/views_processing.py | 2 +- src/dashboard/src/components/api/urls.py | 59 +++---- src/dashboard/src/components/api/views.py | 4 +- .../src/components/appraisal/urls.py | 4 +- .../src/components/archival_storage/forms.py | 2 +- .../src/components/archival_storage/urls.py | 29 ++-- .../src/components/archival_storage/views.py | 2 +- src/dashboard/src/components/backlog/urls.py | 15 +- src/dashboard/src/components/backlog/views.py | 2 +- src/dashboard/src/components/file/urls.py | 8 +- .../src/components/filesystem_ajax/urls.py | 35 +++-- .../src/components/filesystem_ajax/views.py | 53 ++++--- src/dashboard/src/components/helpers.py | 2 +- .../src/components/ingest/pair_matcher.py | 4 +- src/dashboard/src/components/ingest/urls.py | 61 ++++---- src/dashboard/src/components/ingest/views.py | 11 +- .../src/components/ingest/views_as.py | 2 +- src/dashboard/src/components/mcp/urls.py | 6 +- src/dashboard/src/components/rights/forms.py | 2 +- .../src/components/rights/ingest_urls.py | 16 +- .../src/components/rights/transfer_urls.py | 18 +-- src/dashboard/src/components/rights/views.py | 2 +- src/dashboard/src/components/transfer/urls.py | 25 +-- .../src/components/transfer/views.py | 2 +- src/dashboard/src/components/unit/urls.py | 13 +- src/dashboard/src/fpr/forms.py | 2 +- .../src/fpr/migrations/0039_django32.py | 26 ++++ src/dashboard/src/fpr/models.py | 2 +- src/dashboard/src/fpr/urls.py | 95 ++++++------ src/dashboard/src/fpr/utils.py | 2 +- src/dashboard/src/fpr/views.py | 2 +- src/dashboard/src/installer/forms.py | 2 +- src/dashboard/src/installer/urls.py | 6 +- src/dashboard/src/installer/views.py | 2 +- src/dashboard/src/main/__init__.py | 1 - .../commands/rebuild_transfer_backlog.py | 4 +- .../src/main/migrations/0023_blob_fields.py | 7 +- .../main/migrations/0040_directory_model.py | 4 +- .../0049_change_pointer_file_filegrpuse.py | 2 +- .../migrations/0059_siparrange_longblob.py | 6 +- src/dashboard/src/main/models.py | 31 ++-- .../src/main/templatetags/breadcrumb.py | 6 +- src/dashboard/src/main/templatetags/user.py | 2 +- src/dashboard/src/main/urls.py | 31 ++-- src/dashboard/src/main/views.py | 13 +- src/dashboard/src/middleware/locale.py | 2 +- src/dashboard/src/settings/base.py | 2 +- src/dashboard/src/urls.py | 45 +++--- src/dashboard/tests/fixtures/access.json | 6 +- src/dashboard/tests/fixtures/files.json | 16 +- src/dashboard/tests/fixtures/sip_arrange.json | 22 +-- src/dashboard/tests/test_access.py | 3 +- src/dashboard/tests/test_auth.py | 2 +- src/dashboard/tests/test_filesystem_ajax.py | 38 ++--- src/dashboard/tests/test_models.py | 4 +- 133 files changed, 1155 insertions(+), 851 deletions(-) create mode 100644 src/MCPClient/tests/test_assign_file_uuids.py create mode 100755 src/MCPClient/tests/test_assign_uuids_to_directories.py create mode 100644 src/dashboard/src/fpr/migrations/0039_django32.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cc1264a778..f58f64a093 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,10 +129,10 @@ jobs: steps: - name: "Check out repository" uses: "actions/checkout@v3" - - name: "Set up Python 3.7" + - name: "Set up Python 3.8" uses: "actions/setup-python@v4" with: - python-version: "3.7" + python-version: "3.8" - name: "Install tox" run: | python -m pip install --upgrade pip diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61f2145e3c..139d699c4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,11 @@ repos: hooks: - id: reorder-python-imports args: [--py3-plus, --py36-plus] +- repo: https://github.com/adamchainz/django-upgrade + rev: "1.14.0" + hooks: + - id: django-upgrade + args: [--target-version, "3.2"] - repo: https://github.com/ambv/black rev: 22.8.0 hooks: diff --git a/hack/Dockerfile b/hack/Dockerfile index 598c562dc4..481ff0728e 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -94,6 +94,12 @@ RUN set -ex \ COPY . /src +# Allow Django's compilemessages to write *.mo files to the messages subdirectories. +USER root +RUN set -ex \ + && find /src/src/dashboard/src/locale -type d -name 'LC_MESSAGES' -exec chown archivematica:archivematica '{}' \; +USER archivematica + # ----------------------------------------------------------------------------- FROM base AS archivematica-mcp-client diff --git a/hack/submodules/archivematica-acceptance-tests b/hack/submodules/archivematica-acceptance-tests index 96fe940a9e..4a68771ca1 160000 --- a/hack/submodules/archivematica-acceptance-tests +++ b/hack/submodules/archivematica-acceptance-tests @@ -1 +1 @@ -Subproject commit 96fe940a9e18f97aae5868fe4c7c6d0b389814d0 +Subproject commit 4a68771ca1d8fa109f51c7d2cde39cca7307a00c diff --git a/hack/submodules/archivematica-storage-service b/hack/submodules/archivematica-storage-service index 47b2334f0f..048b6f8790 160000 --- a/hack/submodules/archivematica-storage-service +++ b/hack/submodules/archivematica-storage-service @@ -1 +1 @@ -Subproject commit 47b2334f0f4a86288e3625c03a04731db73c8bee +Subproject commit 048b6f8790a0490568a890ee93e2e3b624b52ce8 diff --git a/requirements-dev.txt b/requirements-dev.txt index 081d69a98a..8b5fd510fe 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,6 +10,10 @@ amclient==1.2.3 # via -r requirements.txt ammcpc==0.1.3 # via -r requirements.txt +asgiref==3.4.1 + # via + # -r requirements.txt + # django attrs==22.2.0 # via # -r requirements.txt @@ -19,7 +23,7 @@ bagit==1.8.1 # via -r requirements.txt brotli==1.0.9 # via -r requirements.txt -certifi==2023.5.7 +certifi==2023.7.22 # via # -r requirements.txt # requests @@ -49,28 +53,28 @@ cryptography==40.0.2 # pyopenssl distlib==0.3.7 # via virtualenv -django==1.11.29 +django==3.2.20 # via # -r requirements.txt # django-auth-ldap # django-cas-ng # django-csp # mozilla-django-oidc -django-auth-ldap==2.2.0 +django-auth-ldap==4.0.0 # via -r requirements.txt -django-autoslug==1.9.8 +django-autoslug==1.9.9 # via -r requirements.txt -django-cas-ng==3.6.0 +django-cas-ng==4.3.0 # via -r requirements.txt django-csp==3.7 # via -r requirements.txt django-forms-bootstrap==3.1.0 # via -r requirements.txt -django-prometheus==2.0.0 +django-prometheus==2.2.0 # via -r requirements.txt -django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76 +django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592 # via -r requirements.txt -django-tastypie==0.14.3 +django-tastypie==0.14.5 # via -r requirements.txt elasticsearch==6.8.2 # via -r requirements.txt @@ -88,7 +92,7 @@ greenlet==2.0.2 # via # -r requirements.txt # gevent -gunicorn==21.0.1 +gunicorn==21.2.0 # via -r requirements.txt idna==3.4 # via @@ -134,7 +138,7 @@ metsrw==0.4.0 # via -r requirements.txt mockldap @ git+https://github.com/artefactual-labs/mockldap@v0.3.1 # via -r requirements-dev.in -mozilla-django-oidc==1.2.4 +mozilla-django-oidc==2.0.0 # via -r requirements.txt multidict==5.2.0 # via yarl @@ -191,7 +195,7 @@ pyopenssl==23.2.0 # via # -r requirements.txt # josepy -pyparsing==3.1.0 +pyparsing==3.1.1 # via # -r requirements.txt # packaging @@ -235,7 +239,7 @@ pytz==2023.3 # via # -r requirements.txt # django -pyyaml==6.0 +pyyaml==6.0.1 # via vcrpy requests==2.27.1 # via @@ -250,12 +254,15 @@ six==1.16.0 # -r requirements.txt # amclient # jsonschema - # mozilla-django-oidc # opf-fido # python-cas # python-dateutil # tox # vcrpy +sqlparse==0.4.4 + # via + # -r requirements.txt + # django toml==0.10.2 # via tox tomli==1.2.3 @@ -269,6 +276,7 @@ tox==3.28.0 typing-extensions==4.1.1 # via # -r requirements.txt + # asgiref # importlib-metadata # yarl unidecode==1.3.6 diff --git a/requirements.in b/requirements.in index 7af1fd05ed..012eeccf8f 100644 --- a/requirements.in +++ b/requirements.in @@ -1,16 +1,16 @@ -git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76#egg=django-shibboleth-remoteuser -Django>=1.11,<2 +git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592#egg=django-shibboleth-remoteuser +Django>=3.2,<4 agentarchives amclient ammcpc bagit brotli clamd -django-autoslug==1.9.8 +django-autoslug django-csp -django-forms-bootstrap>=3.0.0,<4.0.0 -django-prometheus>=2.0,<2.1 -django-tastypie==0.14.3 +django-forms-bootstrap +django-prometheus>=2.2,<2.3 +django-tastypie elasticsearch>=6.0.0,<7.0.0 gearman3 gevent @@ -22,21 +22,21 @@ lazy-paged-sequence lxml metsrw mysqlclient -opf-fido==1.6.1 +opf-fido pip pip-tools prometheus_client python-dateutil requests~=2.27 unidecode -whitenoise~=5.3 +whitenoise>=5.3.0,<6.0 # Required by LDAP authentication -django-auth-ldap>=2.2,<2.3 +django-auth-ldap python-ldap # Required by CAS authentication -django-cas-ng~=3.6 +django-cas-ng # Required for OpenID Connect authentication -mozilla-django-oidc~=1.2 +mozilla-django-oidc~=2.0 diff --git a/requirements.txt b/requirements.txt index 9a1aefb89e..279d7fc46c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,13 +10,15 @@ amclient==1.2.3 # via -r requirements.in ammcpc==0.1.3 # via -r requirements.in +asgiref==3.4.1 + # via django attrs==22.2.0 # via jsonschema bagit==1.8.1 # via -r requirements.in brotli==1.0.9 # via -r requirements.in -certifi==2023.5.7 +certifi==2023.7.22 # via requests cffi==1.15.1 # via cryptography @@ -31,28 +33,28 @@ cryptography==40.0.2 # josepy # mozilla-django-oidc # pyopenssl -django==1.11.29 +django==3.2.20 # via # -r requirements.in # django-auth-ldap # django-cas-ng # django-csp # mozilla-django-oidc -django-auth-ldap==2.2.0 +django-auth-ldap==4.0.0 # via -r requirements.in -django-autoslug==1.9.8 +django-autoslug==1.9.9 # via -r requirements.in -django-cas-ng==3.6.0 +django-cas-ng==4.3.0 # via -r requirements.in django-csp==3.7 # via -r requirements.in django-forms-bootstrap==3.1.0 # via -r requirements.in -django-prometheus==2.0.0 +django-prometheus==2.2.0 # via -r requirements.in -django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76 +django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592 # via -r requirements.in -django-tastypie==0.14.3 +django-tastypie==0.14.5 # via -r requirements.in elasticsearch==6.8.2 # via -r requirements.in @@ -62,7 +64,7 @@ gevent==22.10.2 # via -r requirements.in greenlet==2.0.2 # via gevent -gunicorn==21.0.1 +gunicorn==21.2.0 # via -r requirements.in idna==3.4 # via requests @@ -91,7 +93,7 @@ lxml==4.9.3 # python-cas metsrw==0.4.0 # via -r requirements.in -mozilla-django-oidc==1.2.4 +mozilla-django-oidc==2.0.0 # via -r requirements.in mysqlclient==1.4.6 # via @@ -121,7 +123,7 @@ pycparser==2.21 # via cffi pyopenssl==23.2.0 # via josepy -pyparsing==3.1.0 +pyparsing==3.1.1 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -151,14 +153,17 @@ six==1.16.0 # via # amclient # jsonschema - # mozilla-django-oidc # opf-fido # python-cas # python-dateutil +sqlparse==0.4.4 + # via django tomli==1.2.3 # via pep517 typing-extensions==4.1.1 - # via importlib-metadata + # via + # asgiref + # importlib-metadata unidecode==1.3.6 # via -r requirements.in urllib3==1.26.16 diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py index 78a2389a9a..f9f8d3b39b 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py @@ -60,7 +60,7 @@ def process_xml_metadata(mets, sip_dir, sip_uuid, sip_type, xml_validation): try: metadata_file = models.File.objects.get( sip_id=sip_uuid, - currentlocation=f"%SIPDirectory%{xml_rel_path}", + currentlocation=f"%SIPDirectory%{xml_rel_path}".encode(), ) except models.File.DoesNotExist: xml_metadata_errors.append(f"No uuid for file: {xml_rel_path}") diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py index 6fc527f1ef..a3b6d2c5d3 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py @@ -437,7 +437,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir): if rel_path != old_mets_rel_path: job.pyprint(rel_path, "not found in METS, must be new file") f = models.File.objects.get( - currentlocation=current_loc, sip_id=str(sip_uuid) + currentlocation=current_loc.encode(), sip_id=str(sip_uuid) ) new_files.append(f) if rel_path == "objects/metadata/metadata.csv": @@ -476,7 +476,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir): # Create parent directories if needed dirs = os.path.dirname( - f.currentlocation.replace("%SIPDirectory%objects/", "", 1) + f.currentlocation.decode().replace("%SIPDirectory%objects/", "", 1) ).split("/") parent_fsentry = objects_fsentry for dirname in (d for d in dirs if d): @@ -491,7 +491,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir): original_f = f.original_file_set.get().source_file derived_from = mets.get_file(file_uuid=str(original_f.uuid)) entry = metsrw.FSEntry( - path=f.currentlocation.replace("%SIPDirectory%", "", 1), + path=f.currentlocation.decode().replace("%SIPDirectory%", "", 1), use=f.filegrpuse, type="Item", file_uuid=str(f.uuid), @@ -545,7 +545,9 @@ def _get_directory_fsentry(mets, path): def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state): job.pyprint("Parse new metadata.csv") - full_path = metadata_csv.currentlocation.replace("%SIPDirectory%", sip_dir, 1) + full_path = metadata_csv.currentlocation.decode().replace( + "%SIPDirectory%", sip_dir, 1 + ) csvmetadata = createmetscsv.parseMetadataCSV(job, full_path) # FIXME This doesn't support having both DC and non-DC metadata in dmdSecs @@ -607,7 +609,6 @@ def _get_old_mets_rel_path(sip_uuid): def update_mets(job, sip_dir, sip_uuid, state): - old_mets_path = os.path.join(sip_dir, _get_old_mets_rel_path(sip_uuid)) job.pyprint("Looking for old METS at path", old_mets_path) diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py index 063e710a23..ef9bc401bb 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py @@ -61,12 +61,14 @@ def archivematicaCreateMETSRightsDspaceMDRef( # Find the mets file. May find none. path = f"%SIPDirectory%{os.path.dirname(filePath)}/mets.xml" try: - mets = File.objects.get(currentlocation=path, transfer_id=transferUUID) + mets = File.objects.get( + currentlocation=path.encode(), transfer_id=transferUUID + ) except File.DoesNotExist: pass else: metsFileUUID = mets.uuid - metsLoc = mets.currentlocation.replace("%SIPDirectory%", "", 1) + metsLoc = mets.currentlocation.decode().replace("%SIPDirectory%", "", 1) metsLocation = os.path.join(os.path.dirname(itemdirectoryPath), "mets.xml") LABEL = "mets.xml-%s" % (metsFileUUID) ret.append(createMDRefDMDSec(LABEL, metsLocation, metsLoc)) @@ -86,12 +88,14 @@ def archivematicaCreateMETSRightsDspaceMDRef( path = f"%SIPDirectory%{fullDir2}/mets.xml" try: - f = File.objects.get(currentlocation=path, transfer_id=transferUUID) + f = File.objects.get( + currentlocation=path.encode(), transfer_id=transferUUID + ) except File.DoesNotExist: pass else: metsFileUUID = f.uuid - metsLoc = f.currentlocation.replace("%SIPDirectory%", "", 1) + metsLoc = f.currentlocation.decode().replace("%SIPDirectory%", "", 1) metsLocation = os.path.join(fullDir, "mets.xml") job.pyprint(metsLocation) LABEL = "mets.xml-" + metsFileUUID diff --git a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py index d3e312757e..82dfa4bf2d 100755 --- a/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py +++ b/src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py @@ -87,7 +87,7 @@ def getTrimDmdSec(job, baseDirectoryPath, sipUUID): minDateMod = None maxDateMod = None for f in files: - fileMetadataXmlPath = f.currentlocation.replace( + fileMetadataXmlPath = f.currentlocation.decode().replace( "%SIPDirectory%", baseDirectoryPath, 1 ) if os.path.isfile(fileMetadataXmlPath): @@ -128,7 +128,9 @@ def getTrimFileDmdSec(job, baseDirectoryPath, sipUUID, fileUUID): job.pyprint("no metadata for original file: ", fileUUID, file=sys.stderr) return None else: - xmlFilePath = f.currentlocation.replace("%SIPDirectory%", baseDirectoryPath, 1) + xmlFilePath = f.currentlocation.decode().replace( + "%SIPDirectory%", baseDirectoryPath, 1 + ) dublincore = etree.SubElement( xmlData, ns.dctermsBNS + "dublincore", @@ -165,10 +167,11 @@ def getTrimFileAmdSec(job, baseDirectoryPath, sipUUID, fileUUID): job.pyprint("no metadata for original file: ", fileUUID, file=sys.stderr) return None else: - label = os.path.basename(f.currentlocation) + label = os.path.basename(f.currentlocation.decode()) attrib = { "LABEL": label, - ns.xlinkBNS + "href": f.currentlocation.replace("%SIPDirectory%", "", 1), + ns.xlinkBNS + + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1), "MDTYPE": "OTHER", "OTHERMDTYPE": "CUSTOM", "LOCTYPE": "OTHER", @@ -189,7 +192,8 @@ def getTrimAmdSec(job, baseDirectoryPath, sipUUID): for f in files: attrib = { "LABEL": "ContainerMetadata.xml", - ns.xlinkBNS + "href": f.currentlocation.replace("%SIPDirectory%", "", 1), + ns.xlinkBNS + + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1), "MDTYPE": "OTHER", "OTHERMDTYPE": "CUSTOM", "LOCTYPE": "OTHER", diff --git a/src/MCPClient/lib/clientScripts/assign_file_uuids.py b/src/MCPClient/lib/clientScripts/assign_file_uuids.py index 85d1bc789e..8c5393cc6c 100755 --- a/src/MCPClient/lib/clientScripts/assign_file_uuids.py +++ b/src/MCPClient/lib/clientScripts/assign_file_uuids.py @@ -121,9 +121,6 @@ def assign_transfer_file_uuid( We open a database transaction for each chunk of 10 files, in an attempt to balance performance with reasonable transaction lengths. """ - if isinstance(file_path, bytes): - file_path = file_path.decode("utf-8") - file_path_relative_to_sip = file_path.replace( sip_directory, "%transferDirectory%", 1 ) @@ -156,7 +153,9 @@ def assign_transfer_file_uuid( # Update the current location to reflect what's on disk. if transfer.type == Transfer.ARCHIVEMATICA_AIP and mets: job.print_output("Updating current location for", file_uuid, "with", info) - File.objects.filter(uuid=file_uuid).update(currentlocation=info["current_path"]) + File.objects.filter(uuid=file_uuid).update( + currentlocation=info["current_path"].encode() + ) def assign_sip_file_uuid( @@ -175,14 +174,11 @@ def assign_sip_file_uuid( filter_subdir=None, ): """Write SIP file to database with new UUID.""" - if isinstance(file_path, bytes): - file_path = file_path.decode("utf-8") - file_uuid = str(uuid.uuid4()) file_path_relative_to_sip = file_path.replace(sip_directory, "%SIPDirectory%", 1) matching_file = File.objects.filter( - currentlocation=file_path_relative_to_sip, + currentlocation=file_path_relative_to_sip.encode(), sip=sip_uuid, ).first() if matching_file: diff --git a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py index 3563f11a96..af68aa4f97 100755 --- a/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py +++ b/src/MCPClient/lib/clientScripts/assign_uuids_to_directories.py @@ -65,7 +65,7 @@ def exit_on_known_exception(func): @wraps(func) def wrapped(*_args, **kwargs): try: - func(*_args, **kwargs) + return func(*_args, **kwargs) except (DirsUUIDsException, DirsUUIDsWarning) as exc: return exc.exit_code diff --git a/src/MCPClient/lib/clientScripts/change_object_names.py b/src/MCPClient/lib/clientScripts/change_object_names.py index 915e142538..7d69cc67f2 100755 --- a/src/MCPClient/lib/clientScripts/change_object_names.py +++ b/src/MCPClient/lib/clientScripts/change_object_names.py @@ -127,13 +127,15 @@ def apply_file_updates(self): # We pass through _all_ objects here, as they may not be normalized in # the db :( for file_obj in self.file_queryset.iterator(): - old_location = unicodedata.normalize("NFC", file_obj.currentlocation) + old_location = unicodedata.normalize( + "NFC", file_obj.currentlocation.decode() + ) try: changed_location = self.files_index[old_location] except KeyError: continue - file_obj.currentlocation = changed_location + file_obj.currentlocation = changed_location.encode() file_obj.save() change_event = Event( @@ -176,13 +178,15 @@ def apply_dir_updates(self): # We pass through _all_ objects here, as they may not be normalized in # the db :( for dir_obj in self.directory_queryset.iterator(): - old_location = unicodedata.normalize("NFC", dir_obj.currentlocation) + old_location = unicodedata.normalize( + "NFC", dir_obj.currentlocation.decode() + ) try: changed_location = self.dirs_index[old_location] except KeyError: continue - dir_obj.currentlocation = changed_location + dir_obj.currentlocation = changed_location.encode() dir_obj.save() # TODO: Dir name changes don't generate events? diff --git a/src/MCPClient/lib/clientScripts/check_for_service_directory.py b/src/MCPClient/lib/clientScripts/check_for_service_directory.py index b21893d490..8fc4a16b5a 100755 --- a/src/MCPClient/lib/clientScripts/check_for_service_directory.py +++ b/src/MCPClient/lib/clientScripts/check_for_service_directory.py @@ -35,7 +35,7 @@ def something(job, SIPDirectory, serviceDirectory, objectsDirectory, SIPUUID, da exitCode = 0 job.pyprint(SIPDirectory) # For every file, & directory Try to find the matching file & directory in the objects directory - for (path, dirs, files) in os.walk(serviceDirectory): + for path, dirs, files in os.walk(serviceDirectory): for file in files: servicePreExtension = "_me" originalPreExtension = "_m" @@ -59,7 +59,9 @@ def something(job, SIPDirectory, serviceDirectory, objectsDirectory, SIPUUID, da ) # service f = File.objects.get( - currentlocation=file1Full, removedtime__isnull=True, sip_id=SIPUUID + currentlocation=file1Full.encode(), + removedtime__isnull=True, + sip_id=SIPUUID, ) f.filegrpuse = "service" @@ -80,7 +82,7 @@ def regular(SIPDirectory, objectsDirectory, SIPUUID, date): if not searchForRegularExpressions: return - for (path, dirs, files) in os.walk(objectsDirectory): + for path, dirs, files in os.walk(objectsDirectory): for file in files: m = re.search(r"_me\.[a-zA-Z0-9]{2,4}$", file) if m is not None: @@ -93,7 +95,9 @@ def regular(SIPDirectory, objectsDirectory, SIPUUID, date): ) # original f = File.objects.get( - currentlocation=file1Full, removedtime__isnull=True, sip_id=SIPUUID + currentlocation=file1Full.encode(), + removedtime__isnull=True, + sip_id=SIPUUID, ) f.filegrpuse = "service" diff --git a/src/MCPClient/lib/clientScripts/create_mets_v2.py b/src/MCPClient/lib/clientScripts/create_mets_v2.py index cf65c5fe86..e95c04b10c 100755 --- a/src/MCPClient/lib/clientScripts/create_mets_v2.py +++ b/src/MCPClient/lib/clientScripts/create_mets_v2.py @@ -65,7 +65,6 @@ from archivematicaCreateMETSTrim import getTrimFileAmdSec # archivematicaCommon -from archivematicaFunctions import escape from archivematicaFunctions import normalizeNonDcElementName from create_mets_dataverse_v2 import ( create_dataverse_sip_dmdsec, @@ -267,9 +266,9 @@ def getDirDmdSec(dir_mdl, relativeDirectoryPath): for identifier in chain((("UUID", dir_uuid),), _get_mdl_identifiers(dir_mdl)): object_elem = _add_identifier(object_elem, identifier, bns=ns.premisBNS) try: - original_name = escape(dir_mdl.originallocation) + original_name = dir_mdl.originallocation.decode() except AttributeError: # SIP model won't have originallocation - original_name = escape(relativeDirectoryPath) + original_name = relativeDirectoryPath etree.SubElement(object_elem, ns.premisBNS + "originalName").text = original_name return ret @@ -571,9 +570,9 @@ def create_premis_object(fileUUID): for elem in create_premis_object_characteristics_extensions(fileUUID): objectCharacteristics.append(elem) - etree.SubElement(object_elem, ns.premisBNS + "originalName").text = escape( - f.originallocation - ) + etree.SubElement( + object_elem, ns.premisBNS + "originalName" + ).text = f.originallocation.decode() for elem in create_premis_object_derivations(fileUUID): object_elem.append(elem) @@ -776,7 +775,7 @@ def createEvent(event_record): ) etree.SubElement( eventDetailInformation, ns.premisBNS + "eventDetail" - ).text = escape(event_record.event_detail) + ).text = event_record.event_detail eventOutcomeInformation = etree.SubElement( event, ns.premisBNS + "eventOutcomeInformation" @@ -789,7 +788,7 @@ def createEvent(event_record): ) etree.SubElement( eventOutcomeDetail, ns.premisBNS + "eventOutcomeDetailNote" - ).text = escape(event_record.event_outcome_detail) + ).text = event_record.event_outcome_detail # linkingAgentIdentifier for agent in Agent.objects.extend_queryset_with_preservation_system( @@ -1067,7 +1066,7 @@ def createFileSec( kwargs = { "removedtime__isnull": True, "sip_id": sipUUID, - "currentlocation": directoryPathSTR, + "currentlocation": directoryPathSTR.encode(), } try: f = File.objects.get(**kwargs) @@ -1155,7 +1154,9 @@ def createFileSec( if use == "original": DMDIDS = createDMDIDsFromCSVMetadata( job, - f.originallocation.replace("%transferDirectory%", "", 1), + f.originallocation.decode().replace( + "%transferDirectory%", "", 1 + ), state, ) if DMDIDS: @@ -1190,7 +1191,9 @@ def createFileSec( "removedtime__isnull": True, "sip_id": sipUUID, "filegrpuse": "original", - "originallocation__startswith": os.path.dirname(f.originallocation), + "originallocation__startswith": os.path.dirname( + f.originallocation.decode() + ), } original_file = File.objects.filter(**kwargs).first() if original_file is not None: @@ -1252,9 +1255,11 @@ def createFileSec( # Special Dataverse processing. If there's .tab file, check if # there's a Dataverse METS with additional metadata. - if f.originallocation.endswith(".tab"): + if f.originallocation.decode().endswith(".tab"): dv_metadata = create_dataverse_tabfile_dmdsec( - job, baseDirectoryPath, os.path.basename(f.originallocation) + job, + baseDirectoryPath, + os.path.basename(f.originallocation.decode()), ) state.dmdSecs.extend(dv_metadata) ids = " ".join([ds.get("ID") for ds in dv_metadata]) @@ -1730,7 +1735,7 @@ def main( # createSIPfromTransferObjects.py for the association of ``Directory`` # objects to a ``SIP``. directories = { - d.currentlocation.rstrip("/"): d + d.currentlocation.decode().rstrip("/"): d for d in Directory.objects.filter(sip_id=sipUUID).all() } diff --git a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py index ec57ab96ab..b7d5efe0f8 100755 --- a/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py +++ b/src/MCPClient/lib/clientScripts/create_sip_from_transfer_objects.py @@ -145,12 +145,14 @@ def call(jobs): # objects/ directory. For each subdirectory, confirm it's in the SIP # objects/ directory, and update the current location and owning SIP. for dir_mdl in dir_mdls: - currentSIPDirPath = dir_mdl.currentlocation.replace( + currentSIPDirPath = dir_mdl.currentlocation.decode().replace( "%transferDirectory%", tmpSIPDir ) if os.path.isdir(currentSIPDirPath): - dir_mdl.currentlocation = dir_mdl.currentlocation.replace( - "%transferDirectory%", "%SIPDirectory%" + dir_mdl.currentlocation = ( + dir_mdl.currentlocation.decode() + .replace("%transferDirectory%", "%SIPDirectory%") + .encode() ) dir_mdl.sip = sip dir_mdl.save() @@ -168,12 +170,14 @@ def call(jobs): removedtime__isnull=True, ) for f in files: - currentSIPFilePath = f.currentlocation.replace( + currentSIPFilePath = f.currentlocation.decode().replace( "%transferDirectory%", tmpSIPDir ) if os.path.isfile(currentSIPFilePath): - f.currentlocation = f.currentlocation.replace( - "%transferDirectory%", "%SIPDirectory%" + f.currentlocation = ( + f.currentlocation.decode() + .replace("%transferDirectory%", "%SIPDirectory%") + .encode() ) f.sip = sip f.save() diff --git a/src/MCPClient/lib/clientScripts/create_transfer_mets.py b/src/MCPClient/lib/clientScripts/create_transfer_mets.py index 1bb711be81..6093c61d7d 100755 --- a/src/MCPClient/lib/clientScripts/create_transfer_mets.py +++ b/src/MCPClient/lib/clientScripts/create_transfer_mets.py @@ -31,7 +31,7 @@ import metsrw # archivematicaCommon -from archivematicaFunctions import get_dashboard_uuid, escape +from archivematicaFunctions import get_dashboard_uuid from countryCodes import getCodeForCountry # dashboard @@ -84,7 +84,7 @@ def write_mets(mets_path, transfer_dir_path, base_path_placeholder, transfer_uui mets.agents.append(agent) try: - transfer = Transfer.objects.get(uuid=str(transfer_uuid)) + transfer = Transfer.objects.get(uuid=transfer_uuid) except Transfer.DoesNotExist: logger.info("No record in database for transfer: %s", transfer_uuid) raise @@ -225,11 +225,11 @@ def load_file_data_from_db(self): ) for file_obj in self._batch_query(file_objs): try: - fsentry = self.file_index[file_obj.currentlocation] + fsentry = self.file_index[file_obj.currentlocation.decode()] except KeyError: logger.info( "File is no longer present on the filesystem: %s", - file_obj.currentlocation, + file_obj.currentlocation.decode(), ) continue fsentry.file_uuid = str(file_obj.uuid) @@ -278,11 +278,11 @@ def load_dir_uuids_from_db(self): for dir_obj in self._batch_query(dir_objs): try: - fsentry = self.dir_index[dir_obj.currentlocation] + fsentry = self.dir_index[dir_obj.currentlocation.decode()] except KeyError: logger.info( "Directory is no longer present on the filesystem: %s", - dir_obj.currentlocation, + dir_obj.currentlocation.decode(), ) else: premis_intellectual_entity = dir_obj_to_premis(dir_obj) @@ -602,7 +602,7 @@ def file_obj_to_premis(file_obj): """ premis_digest_algorithm = convert_to_premis_hash_function(file_obj.checksumtype) format_data = get_premis_format_data(file_obj.fileid_set.all()) - original_name = escape(file_obj.originallocation) + original_name = file_obj.originallocation.decode() object_identifiers = get_premis_object_identifiers( file_obj.uuid, file_obj.identifiers.all() ) @@ -653,7 +653,7 @@ def dir_obj_to_premis(dir_obj): Returns: lxml.etree._Element """ - original_name = escape(dir_obj.originallocation) + original_name = dir_obj.originallocation.decode() object_identifiers = get_premis_object_identifiers( dir_obj.uuid, dir_obj.identifiers.all() ) diff --git a/src/MCPClient/lib/clientScripts/dip_generation_helper.py b/src/MCPClient/lib/clientScripts/dip_generation_helper.py index 29ec0f4b78..22688bc421 100755 --- a/src/MCPClient/lib/clientScripts/dip_generation_helper.py +++ b/src/MCPClient/lib/clientScripts/dip_generation_helper.py @@ -98,12 +98,11 @@ def parse_archivesspace_ids(sip_path, sip_uuid): # Get file object (for fileUUID, to see if in DIP) logger.debug('Getting file object: filename="%s" ref_id="%s"', filename, ref_id) try: - f = models.File.objects.get( - Q(originallocation="%transferDirectory%" + filename) - | Q(originallocation="%transferDirectory%objects/" + filename) - | Q(originallocation="%SIPDirectory%" + filename) - | Q(originallocation="%SIPDirectory%objects/" + filename), + Q(originallocation=b"%transferDirectory%" + filename.encode()) + | Q(originallocation=b"%transferDirectory%objects/" + filename.encode()) + | Q(originallocation=b"%SIPDirectory%" + filename.encode()) + | Q(originallocation=b"%SIPDirectory%objects/" + filename.encode()), sip_id=sip_uuid, ) except models.File.DoesNotExist: diff --git a/src/MCPClient/lib/clientScripts/extract_contents.py b/src/MCPClient/lib/clientScripts/extract_contents.py index 827ff90c1d..ba67acc8c3 100755 --- a/src/MCPClient/lib/clientScripts/extract_contents.py +++ b/src/MCPClient/lib/clientScripts/extract_contents.py @@ -134,7 +134,7 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): except: job.pyprint( "Not extracting contents from", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - file format not identified", file=sys.stderr, ) @@ -142,7 +142,7 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): if format_id.format_version is None: job.pyprint( "Not extracting contents from", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - file format not identified", file=sys.stderr, ) @@ -158,7 +158,7 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): except FPCommand.DoesNotExist: job.pyprint( "Not extracting contents from", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - No rule found to extract", file=sys.stderr, ) @@ -168,13 +168,13 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): if already_extracted(file_): job.pyprint( "Not extracting contents from", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - extraction already happened.", file=sys.stderr, ) continue - file_to_be_extracted_path = file_.currentlocation.replace( + file_to_be_extracted_path = file_.currentlocation.decode().replace( TRANSFER_DIRECTORY, sip_directory ) extraction_target, file_path_cache = temporary_directory( @@ -219,7 +219,7 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): # extracted files are properly tracked by Archivematica for extracted_file in tree(extraction_target): extracted_file_original_location = extracted_file.replace( - extraction_target, file_.originallocation, 1 + extraction_target, file_.originallocation.decode(), 1 ) assign_uuid( job, @@ -242,7 +242,10 @@ def main(job, transfer_uuid, sip_directory, date, task_uuid, delete=False): # its contents if delete: delete_and_record_package_file( - job, file_to_be_extracted_path, file_.uuid, file_.currentlocation + job, + file_to_be_extracted_path, + file_.uuid, + file_.currentlocation.decode(), ) if extracted: @@ -260,7 +263,7 @@ def create_extracted_dir_uuids( job=job, root_path=extraction_target, path_prefix_to_repl=sip_directory, - original_location=file_.originallocation, + original_location=file_.originallocation.decode(), ), unit_mdl=transfer_mdl, ) diff --git a/src/MCPClient/lib/clientScripts/identify_dspace_files.py b/src/MCPClient/lib/clientScripts/identify_dspace_files.py index ed0f8d71d8..1c07ba3d95 100755 --- a/src/MCPClient/lib/clientScripts/identify_dspace_files.py +++ b/src/MCPClient/lib/clientScripts/identify_dspace_files.py @@ -60,7 +60,7 @@ def identify_dspace_files( continue File.objects.filter( - currentlocation=db_location, transfer_id=transfer_uuid + currentlocation=db_location.encode(), transfer_id=transfer_uuid ).update(filegrpuse=db_use) diff --git a/src/MCPClient/lib/clientScripts/load_labels_from_csv.py b/src/MCPClient/lib/clientScripts/load_labels_from_csv.py index 887bcd2964..34df80eb95 100755 --- a/src/MCPClient/lib/clientScripts/load_labels_from_csv.py +++ b/src/MCPClient/lib/clientScripts/load_labels_from_csv.py @@ -55,5 +55,5 @@ def call(jobs): filePath = row[0] filePath = os.path.join("%transferDirectory%objects/", filePath) File.objects.filter( - originallocation=filePath, transfer_id=transferUUID + originallocation=filePath.encode(), transfer_id=transferUUID ).update(label=label) diff --git a/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py b/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py index 16380170a9..e81e168ee5 100644 --- a/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py +++ b/src/MCPClient/lib/clientScripts/load_premis_events_from_xml.py @@ -453,7 +453,9 @@ def get_invalid_file_identifiers(files, file_queryset, printfn=print): result.add(file_identifier) continue original_location = "".join([TRANSFER_ORIGINAL_LOCATION_PREFIX, original_name]) - if not file_queryset.filter(originallocation=original_location).exists(): + if not file_queryset.filter( + originallocation=original_location.encode() + ).exists(): log_filename_mismatch(file_identifier, original_name, printfn) result.add(file_identifier) return result @@ -676,20 +678,17 @@ def ensure_event_id_is_uuid(event_id, printfn=print): def save_events(valid_events, file_queryset, printfn=print): - for valid_event in valid_events: - event = valid_event["event"] db_agents = get_or_create_agents(valid_event["event_agents"]) for file_ in valid_event["event_files"]: - # get database file from originalName db_file = file_queryset.get( - originallocation="".join( - [TRANSFER_ORIGINAL_LOCATION_PREFIX, file_["original_name"]] - ) + originallocation=( + "".join([TRANSFER_ORIGINAL_LOCATION_PREFIX, file_["original_name"]]) + ).encode() ) # ensure the event identifier is uuid @@ -778,7 +777,6 @@ def get_valid_events(files, agents, events, file_identifiers_to_ignore, printfn= def main(job): - # extract arguments from job transfer_uuid, xsd_path, xml_path = job.args[1:4] diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py index c44745f0b5..91eac31bf8 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_create_metadata_and_restructure.py @@ -53,7 +53,7 @@ def main(job): try: path_condition = Q(currentlocation__startswith=filePathLike1) | Q( - currentlocation=filePathLike2 + currentlocation=filePathLike2.encode() ) original_file = File.objects.get( path_condition, @@ -129,7 +129,9 @@ def main(job): i = basename.rfind(".") dstFile = basename[:i] + "-" + fileUUID + basename[i:] dstDir = os.path.dirname( - original_file.currentlocation.replace("%SIPDirectory%", SIPDirectory, 1) + original_file.currentlocation.decode().replace( + "%SIPDirectory%", SIPDirectory, 1 + ) ) dst = os.path.join(dstDir, dstFile) dstR = dst.replace(SIPDirectory, "%SIPDirectory%", 1) @@ -142,7 +144,7 @@ def main(job): job.print_output("Renaming preservation file", filePath, "to", dst) os.rename(filePath, dst) # Update the preservation file's location - File.objects.filter(uuid=fileUUID).update(currentlocation=dstR) + File.objects.filter(uuid=fileUUID).update(currentlocation=dstR.encode()) try: # Normalization event already exists, so just update it diff --git a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py index ef346620a4..8db6e32afd 100755 --- a/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py +++ b/src/MCPClient/lib/clientScripts/manual_normalization_move_access_files_to_dip.py @@ -70,7 +70,7 @@ def main(job): try: f = File.objects.get(currentlocation__startswith=filePathLike, **kwargs) except (File.DoesNotExist, File.MultipleObjectsReturned): - f = File.objects.get(currentlocation=filePathLike2, **kwargs) + f = File.objects.get(currentlocation=filePathLike2.encode(), **kwargs) except (File.DoesNotExist, File.MultipleObjectsReturned) as e: # Original file was not found, or there is more than one original file with # the same filename (differing extensions) diff --git a/src/MCPClient/lib/clientScripts/normalize.py b/src/MCPClient/lib/clientScripts/normalize.py index ab0e0c0f99..3194af5eae 100755 --- a/src/MCPClient/lib/clientScripts/normalize.py +++ b/src/MCPClient/lib/clientScripts/normalize.py @@ -92,9 +92,11 @@ def check_manual_normalization(job, opts): ) # Get original name of target file, to handle filename changes. file_ = File.objects.get(uuid=opts.file_uuid) - bname = file_.originallocation.replace( - "%transferDirectory%objects/", "", 1 - ).replace("%SIPDirectory%objects/", "", 1) + bname = ( + file_.originallocation.decode() + .replace("%transferDirectory%objects/", "", 1) + .replace("%SIPDirectory%objects/", "", 1) + ) if os.path.isfile(normalization_csv): found = False # use universal newline mode to support unusual newlines, like \r @@ -185,8 +187,8 @@ def check_manual_normalization(job, opts): job.print_output( "Multiple files matching path {} found. Returning the shortest one." ) - ret = sorted(matches, key=lambda f: f.currentlocation)[0] - job.print_output(f"Returning file at {ret.currentlocation}") + ret = sorted(matches, key=lambda f: f.currentlocation.decode())[0] + job.print_output(f"Returning file at {ret.currentlocation.decode()}") return ret return matches[0] @@ -342,13 +344,13 @@ def main(job, opts): except File.DoesNotExist: job.print_error("File with uuid", opts.file_uuid, "does not exist in database.") return NO_RULE_FOUND - job.print_output("File found:", file_.uuid, file_.currentlocation) + job.print_output("File found:", file_.uuid, file_.currentlocation.decode()) # Unless normalization file group use is submissionDocumentation, skip the # submissionDocumentation directory if ( opts.normalize_file_grp_use != "submissionDocumentation" - and file_.currentlocation.startswith( + and file_.currentlocation.decode().startswith( "%SIPDirectory%objects/submissionDocumentation" ) ): @@ -399,7 +401,7 @@ def main(job, opts): job.print_output( os.path.basename(opts.file_path), "was already manually normalized into", - manually_normalized_file.currentlocation, + manually_normalized_file.currentlocation.decode(), ) if "preservation" in opts.purpose: # Add derivation link and associated event @@ -440,7 +442,7 @@ def main(job, opts): try: rule = get_default_rule(opts.purpose) job.print_output( - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), "not identified or without rule", "- Falling back to default", opts.purpose, @@ -449,7 +451,7 @@ def main(job, opts): except FPRule.DoesNotExist: job.print_output( "Not normalizing", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - No rule or default rule found to normalize for", opts.purpose, ) @@ -493,7 +495,7 @@ def main(job, opts): except FPRule.DoesNotExist: job.print_output( "Not retrying normalizing for", - os.path.basename(file_.currentlocation), + os.path.basename(file_.currentlocation.decode()), " - No default rule found to normalize for", opts.purpose, ) diff --git a/src/MCPClient/lib/clientScripts/normalize_report.py b/src/MCPClient/lib/clientScripts/normalize_report.py index e43f19fdcf..1e8984954e 100755 --- a/src/MCPClient/lib/clientScripts/normalize_report.py +++ b/src/MCPClient/lib/clientScripts/normalize_report.py @@ -137,9 +137,11 @@ def report(uuid): failed_tasks[jobtype] = tasks.values("filename", "fileuuid", "exitcode") for item in failed_tasks[jobtype]: try: - item["location"] = File.objects.get( - uuid=item["fileuuid"] - ).currentlocation.replace("%SIPDirectory%", "") + item["location"] = ( + File.objects.get(uuid=item["fileuuid"]) + .currentlocation.decode() + .replace("%SIPDirectory%", "") + ) except File.DoesNotExist: pass @@ -152,7 +154,7 @@ def report(uuid): ctxdict = { "uuid": uuid, "name": os.path.basename(sip.currentpath.rstrip("/")).replace( - "-" + sip.uuid, "" + "-" + str(sip.uuid), "" ), "pipeline_uuid": helpers.get_setting("dashboard_uuid"), "failed_tasks": failed_tasks, diff --git a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py index 0c6083a5f7..2c710f8534 100644 --- a/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py +++ b/src/MCPClient/lib/clientScripts/parse_dataverse_mets.py @@ -58,7 +58,7 @@ def get_db_objects(job, mets, transfer_uuid): ) try: file_entry = File.objects.get( - originallocation=item_path, transfer_id=transfer_uuid + originallocation=item_path.encode(), transfer_id=transfer_uuid ) # If we retrieve the object there is still a chance that the # file has been removed by Archivematica e.g. via extract @@ -93,7 +93,7 @@ def get_db_objects(job, mets, transfer_uuid): base_name = os.path.basename(entry.path) item_path = os.path.join(transfer_objects_directory, base_name) file_entry = File.objects.get( - originallocation=item_path, transfer_id=transfer_uuid + originallocation=item_path.encode(), transfer_id=transfer_uuid ) # If we retrieve the object there is still a chance that the # file has been removed by Archivematica e.g. via extract @@ -220,7 +220,9 @@ def validate_checksums(job, mapping, unit_path): ): logger.info("File: %s removed by extract packages?", entry.label) continue - path_ = file_entry.currentlocation.replace("%transferDirectory%", unit_path) + path_ = file_entry.currentlocation.decode().replace( + "%transferDirectory%", unit_path + ) if os.path.isdir(path_): continue verify_checksum( diff --git a/src/MCPClient/lib/clientScripts/parse_mets_to_db.py b/src/MCPClient/lib/clientScripts/parse_mets_to_db.py index 0873e5b0d7..8342b9eb94 100755 --- a/src/MCPClient/lib/clientScripts/parse_mets_to_db.py +++ b/src/MCPClient/lib/clientScripts/parse_mets_to_db.py @@ -156,7 +156,7 @@ def update_files(sip_uuid, files): checksum=file_info["checksum"], checksumtype=file_info["checksumtype"], size=file_info["size"], - currentlocation=file_info["current_path"], + currentlocation=file_info["current_path"].encode(), ) if file_info["format_version"]: # Add Format ID diff --git a/src/MCPClient/lib/clientScripts/pid_declaration.py b/src/MCPClient/lib/clientScripts/pid_declaration.py index 28f8c1e6bb..4a3ac08c99 100644 --- a/src/MCPClient/lib/clientScripts/pid_declaration.py +++ b/src/MCPClient/lib/clientScripts/pid_declaration.py @@ -56,7 +56,7 @@ def exit_on_known_exception(func): def wrapped(*args, **kwargs): try: func(*args, **kwargs) - except (DeclarePIDsExceptionNonCritical) as err: + except DeclarePIDsExceptionNonCritical as err: return err.exit_code return wrapped @@ -88,14 +88,14 @@ def _log_identifer_to_stdout(self, mdl, identifier_type, identifier): identifier_type, identifier, "Directory", - os.path.basename(os.path.split(mdl.currentlocation)[0]), + os.path.basename(os.path.split(mdl.currentlocation.decode())[0]), ) else: msg = "Identifier {}: {} added for: {}: {}".format( identifier_type, identifier, "File", - os.path.basename(mdl.currentlocation), + os.path.basename(mdl.currentlocation.decode()), ) self.job.pyprint(msg) except KeyError: @@ -158,14 +158,17 @@ def parse_and_attach_identifiers(self, unit_uuid, json_data): self._add_identifier_to_model(mdl, id_) continue try: - mdl = File.objects.get(sip_id=unit_uuid, currentlocation=file_path) + mdl = File.objects.get( + sip_id=unit_uuid, currentlocation=file_path.encode() + ) self._add_identifier_to_model(mdl, id_) continue except File.DoesNotExist: pass try: mdl = Directory.objects.get( - sip_id=unit_uuid, currentlocation=os.path.join(file_path, "") + sip_id=unit_uuid, + currentlocation=os.path.join(file_path, "").encode(), ) self._add_identifier_to_model(mdl, id_) except Directory.DoesNotExist: @@ -179,11 +182,15 @@ def parse_and_attach_identifiers(self, unit_uuid, json_data): def _retrieve_identifiers_path(self, unit_uuid, sip_directory): """Retrieve a path to the identifiers.json file from the database.""" try: - return File.objects.get( - sip_id=unit_uuid, - filegrpuse="metadata", - currentlocation__endswith="identifiers.json", - ).currentlocation.replace(self.SIP_DIRECTORY, sip_directory) + return ( + File.objects.get( + sip_id=unit_uuid, + filegrpuse="metadata", + currentlocation__endswith="identifiers.json", + ) + .currentlocation.decode() + .replace(self.SIP_DIRECTORY, sip_directory) + ) except File.DoesNotExist: self.job.pyprint("No identifiers.json file found", file=sys.stderr) raise DeclarePIDsExceptionNonCritical() diff --git a/src/MCPClient/lib/clientScripts/policy_check.py b/src/MCPClient/lib/clientScripts/policy_check.py index 3a477cbad3..e4fabde64e 100755 --- a/src/MCPClient/lib/clientScripts/policy_check.py +++ b/src/MCPClient/lib/clientScripts/policy_check.py @@ -202,7 +202,8 @@ def _get_manually_normalized_access_derivative_file_uuid(self): ) try: return File.objects.get( - originallocation=manually_normalized_file_path, sip_id=self.sip_uuid + originallocation=manually_normalized_file_path.encode(), + sip_id=self.sip_uuid, ).uuid except (File.DoesNotExist, File.MultipleObjectsReturned): return None diff --git a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py index 9fcdf461c9..09bf351da8 100755 --- a/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py +++ b/src/MCPClient/lib/clientScripts/retry_normalize_remove_normalized.py @@ -64,7 +64,9 @@ def removePreservationFiles(job, SIPDirectory, SIPUUID): file_.removedtime = timezone.now() file_.save() os.remove( - file_.currentlocation.replace("%SIPDirectory%", SIPDirectory, 1) + file_.currentlocation.decode().replace( + "%SIPDirectory%", SIPDirectory, 1 + ) ) except Exception: job.pyprint("Error removing preservation files", file=sys.stderr) diff --git a/src/MCPClient/lib/clientScripts/rights_from_csv.py b/src/MCPClient/lib/clientScripts/rights_from_csv.py index de53c7753d..85a9468300 100755 --- a/src/MCPClient/lib/clientScripts/rights_from_csv.py +++ b/src/MCPClient/lib/clientScripts/rights_from_csv.py @@ -37,7 +37,6 @@ def __init__(self, message, reader): class RightCsvReader: - metadata_applies_to_type = None current_row = None rows_processed = 0 @@ -170,7 +169,7 @@ def generate_rights_statement(self): # Get file data filepath = self.column_value("file") transfer_file = models.File.objects.get( - originallocation="%transferDirectory%" + filepath, + originallocation=("%transferDirectory%" + filepath).encode(), transfer_id=self.transfer_uuid, ) diff --git a/src/MCPClient/lib/clientScripts/store_file_modification_dates.py b/src/MCPClient/lib/clientScripts/store_file_modification_dates.py index 47936ea8e3..e24a88a5b1 100755 --- a/src/MCPClient/lib/clientScripts/store_file_modification_dates.py +++ b/src/MCPClient/lib/clientScripts/store_file_modification_dates.py @@ -46,7 +46,7 @@ def main(transfer_uuid, shared_directory_path): for transfer_file in files: try: file_path_relative_to_shared_directory = ( - transfer_file.currentlocation.replace( + transfer_file.currentlocation.decode().replace( "%transferDirectory%", transfer.currentlocation, 1 ) ) diff --git a/src/MCPClient/lib/clientScripts/update_size_and_checksum.py b/src/MCPClient/lib/clientScripts/update_size_and_checksum.py index 57ec6bf194..51544957f9 100755 --- a/src/MCPClient/lib/clientScripts/update_size_and_checksum.py +++ b/src/MCPClient/lib/clientScripts/update_size_and_checksum.py @@ -136,11 +136,11 @@ def get_size_and_checksum_for_file( """ kw = {} if transfer_uuid: - file_path = file_.currentlocation.replace( + file_path = file_.currentlocation.decode().replace( TRANSFER_REPLACEMENT_PATH_STRING, sip_directory ) else: - file_path = file_.currentlocation.replace( + file_path = file_.currentlocation.decode().replace( SIP_REPLACEMENT_PATH_STRING, sip_directory ) diff --git a/src/MCPClient/lib/clientScripts/upload_archivesspace.py b/src/MCPClient/lib/clientScripts/upload_archivesspace.py index 13e5cd19a9..4c86fcb4c8 100755 --- a/src/MCPClient/lib/clientScripts/upload_archivesspace.py +++ b/src/MCPClient/lib/clientScripts/upload_archivesspace.py @@ -180,7 +180,7 @@ def upload_to_archivesspace( # Set some variables based on the original, we will override most # of these if there is an access derivative size = os.path.getsize(f) - original_name = os.path.basename(original_file.originallocation) + original_name = os.path.basename(original_file.originallocation.decode()) try: access_file = File.objects.get( filegrpuse="access", original_file_set__source_file=uuid @@ -190,7 +190,7 @@ def upload_to_archivesspace( pass else: # HACK remove DIP from the path because create DIP doesn't - access_file_path = access_file.currentlocation.replace( + access_file_path = access_file.currentlocation.decode().replace( "%SIPDirectory%DIP/", dip_location ) size = os.path.getsize(access_file_path) @@ -232,7 +232,6 @@ def upload_to_archivesspace( inherit_notes=inherit_notes, ) except ArchivesSpaceError as error: - logger.error( "Could not upload {} to ArchivesSpace record {}. Error: {}".format( file_name, as_resource, str(error) diff --git a/src/MCPClient/lib/clientScripts/verify_aip.py b/src/MCPClient/lib/clientScripts/verify_aip.py index b97c3cbfaa..ce589a8e69 100755 --- a/src/MCPClient/lib/clientScripts/verify_aip.py +++ b/src/MCPClient/lib/clientScripts/verify_aip.py @@ -144,14 +144,16 @@ def verify_checksums(job, bag, sip_uuid): verification_skipped_because_reingest = 0 for file_ in File.objects.filter(sip_id=sip_uuid): if ( - os.path.basename(file_.originallocation) in removableFiles + os.path.basename(file_.originallocation.decode()) in removableFiles or file_.removedtime - or not file_.currentlocation.startswith("%SIPDirectory%objects/") + or not file_.currentlocation.decode().startswith( + "%SIPDirectory%objects/" + ) or file_.filegrpuse == "manualNormalization" ): continue file_path = os.path.join( - "data", file_.currentlocation.replace("%SIPDirectory%", "", 1) + "data", file_.currentlocation.decode().replace("%SIPDirectory%", "", 1) ) assert_checksum_types_match(job, file_, sip_uuid, checksum_type) expected_checksum = get_expected_checksum( diff --git a/src/MCPClient/lib/ensure_no_mutable_globals.py b/src/MCPClient/lib/ensure_no_mutable_globals.py index 3b9d58cbe2..fb87db97d5 100755 --- a/src/MCPClient/lib/ensure_no_mutable_globals.py +++ b/src/MCPClient/lib/ensure_no_mutable_globals.py @@ -38,7 +38,6 @@ str, tuple, django.conf.LazySettings, - django.db.DefaultConnectionProxy, prometheus_client.Counter, prometheus_client.Gauge, prometheus_client.Histogram, diff --git a/src/MCPClient/tests/fixtures/custom_structmaps/model/files.json b/src/MCPClient/tests/fixtures/custom_structmaps/model/files.json index c2491e1a66..40542738ad 100644 --- a/src/MCPClient/tests/fixtures/custom_structmaps/model/files.json +++ b/src/MCPClient/tests/fixtures/custom_structmaps/model/files.json @@ -5,7 +5,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/test_file.flac", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5mbGFj", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -13,7 +13,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/test_file.flac", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5mbGFj", "size": 2600 } }, @@ -23,7 +23,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/test_file.mp3", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5tcDM=", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -31,7 +31,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/test_file.mp3", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5tcDM=", "size": 2600 } }, @@ -41,7 +41,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/test_file.jpg", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5qcGc=", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -49,7 +49,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/test_file.jpg", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5qcGc=", "size": 2600 } }, @@ -59,7 +59,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/test_file.png", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5wbmc=", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -67,7 +67,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/test_file.png", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3Rlc3RfZmlsZS5wbmc=", "size": 2600 } }, @@ -77,7 +77,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/nested_dir/nested_file.rdata", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL25lc3RlZF9kaXIvbmVzdGVkX2ZpbGUucmRhdGE=", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -85,7 +85,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/nested_dir/nested_file.rdata", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL25lc3RlZF9kaXIvbmVzdGVkX2ZpbGUucmRhdGE=", "size": 2600 } }, @@ -95,7 +95,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/página_de_prueba.png", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3DDoWdpbmFfZGVfcHJ1ZWJhLnBuZw==", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -103,7 +103,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/página_de_prueba.png", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3DDoWdpbmFfZGVfcHJ1ZWJhLnBuZw==", "size": 2600 } }, @@ -113,7 +113,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/página_de_prueba.jpg", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3DDoWdpbmFfZGVfcHJ1ZWJhLmpwZw==", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -121,7 +121,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/página_de_prueba.jpg", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3DDoWdpbmFfZGVfcHJ1ZWJhLmpwZw==", "size": 2600 } }, @@ -131,7 +131,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/duplicate_file_name.png", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2R1cGxpY2F0ZV9maWxlX25hbWUucG5n", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -139,7 +139,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/duplicate_file_name.png", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2R1cGxpY2F0ZV9maWxlX25hbWUucG5n", "size": 2600 } }, @@ -149,7 +149,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/nested_dir/duplicate_file_name.png", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL25lc3RlZF9kaXIvZHVwbGljYXRlX2ZpbGVfbmFtZS5wbmc=", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -157,7 +157,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/nested_dir/duplicate_file_name.png", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL25lc3RlZF9kaXIvZHVwbGljYXRlX2ZpbGVfbmFtZS5wbmc=", "size": 2600 } }, @@ -167,7 +167,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/dir-with-dashes/file with spaces.bin", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2Rpci13aXRoLWRhc2hlcy9maWxlIHdpdGggc3BhY2VzLmJpbg==", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -175,7 +175,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/dir-with-dashes/file with spaces.bin", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2Rpci13aXRoLWRhc2hlcy9maWxlIHdpdGggc3BhY2VzLmJpbg==", "size": 2600 } }, @@ -185,7 +185,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/mets_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9tZXRzX3N0cnVjdG1hcC54bWw=", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -193,7 +193,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/mets_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9tZXRzX3N0cnVjdG1hcC54bWw=", "size": 2600 } }, @@ -203,7 +203,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/simple_book_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9zaW1wbGVfYm9va19zdHJ1Y3RtYXAueG1s", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -211,7 +211,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/simple_book_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9zaW1wbGVfYm9va19zdHJ1Y3RtYXAueG1s", "size": 2600 } }, @@ -221,7 +221,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/mets_area_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9tZXRzX2FyZWFfc3RydWN0bWFwLnhtbA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -229,7 +229,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/mets_area_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9tZXRzX2FyZWFfc3RydWN0bWFwLnhtbA==", "size": 2600 } }, @@ -239,7 +239,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/unicode_simple_book_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS91bmljb2RlX3NpbXBsZV9ib29rX3N0cnVjdG1hcC54bWw=", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -247,7 +247,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/unicode_simple_book_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS91bmljb2RlX3NpbXBsZV9ib29rX3N0cnVjdG1hcC54bWw=", "size": 2600 } }, @@ -257,7 +257,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/nested_file_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9uZXN0ZWRfZmlsZV9zdHJ1Y3RtYXAueG1s", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -265,7 +265,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/nested_file_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9uZXN0ZWRfZmlsZV9zdHJ1Y3RtYXAueG1s", "size": 2600 } }, @@ -275,7 +275,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/path_with_spaces_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9wYXRoX3dpdGhfc3BhY2VzX3N0cnVjdG1hcC54bWw=", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -283,7 +283,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/path_with_spaces_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9wYXRoX3dpdGhfc3BhY2VzX3N0cnVjdG1hcC54bWw=", "size": 2600 } }, @@ -293,7 +293,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/complex_book_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9jb21wbGV4X2Jvb2tfc3RydWN0bWFwLnhtbA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -301,7 +301,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/complex_book_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9jb21wbGV4X2Jvb2tfc3RydWN0bWFwLnhtbA==", "size": 2600 } }, @@ -311,7 +311,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/broken_structmap.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9icm9rZW5fc3RydWN0bWFwLnhtbA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -319,7 +319,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/broken_structmap.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9icm9rZW5fc3RydWN0bWFwLnhtbA==", "size": 2600 } }, @@ -329,7 +329,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/no-contentids.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9uby1jb250ZW50aWRzLnhtbA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -337,7 +337,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/no-contentids.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9uby1jb250ZW50aWRzLnhtbA==", "size": 2600 } }, @@ -347,7 +347,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/file_does_not_exist.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9maWxlX2RvZXNfbm90X2V4aXN0LnhtbA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -355,7 +355,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/file_does_not_exist.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9maWxlX2RvZXNfbm90X2V4aXN0LnhtbA==", "size": 2600 } }, @@ -365,7 +365,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/empty_filenames.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9lbXB0eV9maWxlbmFtZXMueG1s", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -373,7 +373,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/empty_filenames.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9lbXB0eV9maWxlbmFtZXMueG1s", "size": 2600 } }, @@ -383,7 +383,7 @@ "fields": { "filegrpuuid": "", "sip": "3a915449-d1bb-4920-b274-c917c7bb5929", - "originallocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/missing_contentid.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9taXNzaW5nX2NvbnRlbnRpZC54bWw=", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -391,7 +391,7 @@ "checksum": "", "enteredsystem": "2019-04-10T23:13:00Z", "modificationtime": "1970-01-01T00:00:00Z", - "currentlocation": "%SIPDirectory%objects/metadata/transfers/custom-structmap-41ab1f1a-34d0-4a83-a2a3-0ad1b1ee1c51/missing_contentid.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL3RyYW5zZmVycy9jdXN0b20tc3RydWN0bWFwLTQxYWIxZjFhLTM0ZDAtNGE4My1hMmEzLTBhZDFiMWVlMWM1MS9taXNzaW5nX2NvbnRlbnRpZC54bWw=", "size": 2600 } } diff --git a/src/MCPClient/tests/fixtures/dataverse_sip.json b/src/MCPClient/tests/fixtures/dataverse_sip.json index d9dc332a0f..99ebbcd6b1 100644 --- a/src/MCPClient/tests/fixtures/dataverse_sip.json +++ b/src/MCPClient/tests/fixtures/dataverse_sip.json @@ -18,14 +18,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/chelan 052.jpg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvY2hlbGFuIDA1Mi5qcGc=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "ab61f03d25ddd0e638f1ff2a823336dc38dd0c7235ee4b6ab2ec671880f822ea", "enteredsystem": "2015-11-05T22:06:41", - "currentlocation": "%transferDirectory%objects/chelan_052.jpg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvY2hlbGFuXzA1Mi5qcGc=", "size": 76934 }, "model": "main.file", @@ -35,14 +35,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_data.tab", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS50YWI=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c54c464c5efbdb4d6c903043c18d41b690653a38cbfcbc0cc31fabe6cda55a0e", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_data.tab", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS50YWI=", "size": 563285 }, "model": "main.file", @@ -52,7 +52,7 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data.zip", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhLnppcA==", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": "2015-11-05T22:06:49", @@ -69,14 +69,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_datacitation-ris.ris", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YWNpdGF0aW9uLXJpcy5yaXM=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "21b90f678beb353417f0f0d0e93bc7259c3575b336f8dd590ea6d4f8f2acfce8", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_datacitation-ris.ris", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YWNpdGF0aW9uLXJpcy5yaXM=", "size": 325 }, "model": "main.file", @@ -86,14 +86,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/dataset.json", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvZGF0YXNldC5qc29u", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "9ed70885977815b701eb325062fe1360cc8985398e9d4d536e16a0cf53293377", "enteredsystem": "2015-11-05T22:06:41", - "currentlocation": "%transferDirectory%objects/dataset.json", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvZGF0YXNldC5qc29u", "size": 2832 }, "model": "main.file", @@ -103,14 +103,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_data-ddi.xml", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS1kZGkueG1s", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c3fceee71ab02901f01034cfc3b50e611378e04d8e823aa564405360f3913e3a", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_data-ddi.xml", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS1kZGkueG1s", "size": 15337 }, "model": "main.file", @@ -120,14 +120,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_datacitation-endnote.xml", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YWNpdGF0aW9uLWVuZG5vdGUueG1s", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "ba57d36d8721b925117ff475a81f03180cfea0e7f9f20d22efbd0ba76ed639ef", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_datacitation-endnote.xml", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YWNpdGF0aW9uLWVuZG5vdGUueG1s", "size": 619 }, "model": "main.file", @@ -137,14 +137,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_data.RData", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS5SRGF0YQ==", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c54c464c5efbdb4d6c903043c18d41b690653a38cbfcbc0cc31fabe6cda55a0e", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_data.RData", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS5SRGF0YQ==", "size": 563285 }, "model": "main.file", @@ -154,14 +154,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/Weather_data.sav", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS5zYXY=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c54c464c5efbdb4d6c903043c18d41b690653a38cbfcbc0cc31fabe6cda55a0e", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/Weather_data.sav", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL1dlYXRoZXJfZGF0YS5zYXY=", "size": 563285 }, "model": "main.file", @@ -171,14 +171,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/i_am_a_duplicate.original", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL2lfYW1fYV9kdXBsaWNhdGUub3JpZ2luYWw=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c54c464c5efbdb4d6c903043c18d41b690653a38cbfcbc0cc31fabe6cda55a0e", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/i_am_a_duplicate.original", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL2lfYW1fYV9kdXBsaWNhdGUub3JpZ2luYWw=", "size": 563285 }, "model": "main.file", @@ -188,14 +188,14 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/i_am_a_duplicate.original", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL2lfYW1fYV9kdXBsaWNhdGUub3JpZ2luYWw=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": null, "label": "", "checksum": "c54c464c5efbdb4d6c903043c18d41b690653a38cbfcbc0cc31fabe6cda55a0e", "enteredsystem": "2015-11-05T22:06:49", - "currentlocation": "%transferDirectory%objects/Weather_data/i_am_a_duplicate.original", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL2lfYW1fYV9kdXBsaWNhdGUub3JpZ2luYWw=", "size": 563285 }, "model": "main.file", @@ -205,7 +205,7 @@ "fields": { "filegrpuuid": "", "sip": null, - "originallocation": "%transferDirectory%objects/Weather_data/i_have_been_removed", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvV2VhdGhlcl9kYXRhL2lfaGF2ZV9iZWVuX3JlbW92ZWQ=", "transfer": "6741c782-f22b-47b3-8bcf-72fd0c94e195", "filegrpuse": "original", "removedtime": "2019-01-15T23:13:00", diff --git a/src/MCPClient/tests/fixtures/files-transfer-unicode.json b/src/MCPClient/tests/fixtures/files-transfer-unicode.json index f59ecfe90d..640659a3f3 100644 --- a/src/MCPClient/tests/fixtures/files-transfer-unicode.json +++ b/src/MCPClient/tests/fixtures/files-transfer-unicode.json @@ -4,7 +4,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "sha256", - "originallocation": "%transferDirectory%objects/\u305f\u304f\u3055\u3093 directories/need name change/checking here/ev\u00e9lyn's photo.jpg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMv44Gf44GP44GV44KTIGRpcmVjdG9yaWVzL25lZWQgbmFtZSBjaGFuZ2UvY2hlY2tpbmcgaGVyZS9ldsOpbHluJ3MgcGhvdG8uanBn", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -12,7 +12,7 @@ "checksum": "d2bed92b73c7090bb30a0b30016882e7069c437488e1513e9deaacbe29d38d92", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/\u305f\u304f\u3055\u3093 directories/need name change/checking here/ev\u00e9lyn's photo.jpg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMv44Gf44GP44GV44KTIGRpcmVjdG9yaWVzL25lZWQgbmFtZSBjaGFuZ2UvY2hlY2tpbmcgaGVyZS9ldsOpbHluJ3MgcGhvdG8uanBn", "size": 158131 }, "model": "main.file", @@ -23,7 +23,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "sha256", - "originallocation": "%transferDirectory%objects/no_name_change/needed_here/lion.svg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvbm9fbmFtZV9jaGFuZ2UvbmVlZGVkX2hlcmUvbGlvbi5zdmc=", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -31,7 +31,7 @@ "checksum": "f78615cd834f7fb84832177e73f13e3479f5b5b22ae7a9506c7fa0a14fd9df9e", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/no_name_change/needed_here/lion.svg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvbm9fbmFtZV9jaGFuZ2UvbmVlZGVkX2hlcmUvbGlvbi5zdmc=", "size": 18324 }, "model": "main.file", @@ -42,7 +42,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "sha256", - "originallocation": "%transferDirectory%objects/\u305f\u304f\u3055\u3093 directories/need name change/checking here/lion\u5199\u771f.svg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMv44Gf44GP44GV44KTIGRpcmVjdG9yaWVzL25lZWQgbmFtZSBjaGFuZ2UvY2hlY2tpbmcgaGVyZS9saW9u5YaZ55yfLnN2Zw==", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -50,7 +50,7 @@ "checksum": "f78615cd834f7fb84832177e73f13e3479f5b5b22ae7a9506c7fa0a14fd9df9e", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/\u305f\u304f\u3055\u3093 directories/need name change/checking here/lion\u5199\u771f.svg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMv44Gf44GP44GV44KTIGRpcmVjdG9yaWVzL25lZWQgbmFtZSBjaGFuZ2UvY2hlY2tpbmcgaGVyZS9saW9u5YaZ55yfLnN2Zw==", "size": 18324 }, "model": "main.file", @@ -61,7 +61,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "sha256", - "originallocation": "%transferDirectory%objects/has space/lion.svg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvaGFzIHNwYWNlL2xpb24uc3Zn", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -69,7 +69,7 @@ "checksum": "f78615cd834f7fb84832177e73f13e3479f5b5b22ae7a9506c7fa0a14fd9df9e", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/has space/lion.svg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvaGFzIHNwYWNlL2xpb24uc3Zn", "size": 18324 }, "model": "main.file", diff --git a/src/MCPClient/tests/fixtures/files-transfer.json b/src/MCPClient/tests/fixtures/files-transfer.json index ff5152851f..bd9e17dd42 100644 --- a/src/MCPClient/tests/fixtures/files-transfer.json +++ b/src/MCPClient/tests/fixtures/files-transfer.json @@ -4,7 +4,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "", - "originallocation": "%transferDirectory%objects/G31DS.TIF", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvRzMxRFMuVElG", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -12,7 +12,7 @@ "checksum": "", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/G31DS.TIF", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvRzMxRFMuVElG", "size": 125968 }, "model": "main.file", @@ -23,7 +23,7 @@ "filegrpuuid": "", "sip": null, "checksumtype": "", - "originallocation": "%transferDirectory%objects/lion.svg", + "originallocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvbGlvbi5zdmc=", "transfer": "e95ab50f-9c84-45d5-a3ca-1b0b3f58d9b6", "filegrpuse": "original", "removedtime": null, @@ -31,7 +31,7 @@ "checksum": "", "enteredsystem": "2017-01-04T19:35:20Z", "modificationtime": "2017-01-04T19:35:20Z", - "currentlocation": "%transferDirectory%objects/lion.svg", + "currentlocation": "JXRyYW5zZmVyRGlyZWN0b3J5JW9iamVjdHMvbGlvbi5zdmc=", "size": 18324 }, "model": "main.file", diff --git a/src/MCPClient/tests/fixtures/files.json b/src/MCPClient/tests/fixtures/files.json index 88a92ce68c..1280cad126 100644 --- a/src/MCPClient/tests/fixtures/files.json +++ b/src/MCPClient/tests/fixtures/files.json @@ -5,7 +5,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/submissionDocumentation/transfer-no-metadata-46260807-ece1-4a0e-b70a-9814c701146b/METS.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3N1Ym1pc3Npb25Eb2N1bWVudGF0aW9uL3RyYW5zZmVyLW5vLW1ldGFkYXRhLTQ2MjYwODA3LWVjZTEtNGEwZS1iNzBhLTk4MTRjNzAxMTQ2Yi9NRVRTLnhtbA==", "transfer": null, "filegrpuse": "submissionDocumentation", "removedtime": null, @@ -13,7 +13,7 @@ "checksum": "51132e5ce1b5d2c2c363f05495f447ea924ab29c2cda2c11037b5fca2119e45a", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/submissionDocumentation/transfer-no-metadata-46260807-ece1-4a0e-b70a-9814c701146b/METS.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3N1Ym1pc3Npb25Eb2N1bWVudGF0aW9uL3RyYW5zZmVyLW5vLW1ldGFkYXRhLTQ2MjYwODA3LWVjZTEtNGEwZS1iNzBhLTk4MTRjNzAxMTQ2Yi9NRVRTLnhtbA==", "size": 12222 } }, @@ -23,7 +23,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/evelyn_s_photo-8140ebe5-295c-490b-a34a-83955b7c844e.tif", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLTgxNDBlYmU1LTI5NWMtNDkwYi1hMzRhLTgzOTU1YjdjODQ0ZS50aWY=", "transfer": null, "filegrpuse": "preservation", "removedtime": null, @@ -31,7 +31,7 @@ "checksum": "d82448f154b9185bc777ecb0a3602760eb76ba85dd3098f073b2c91a03f571e9", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/evelyn_s_photo-8140ebe5-295c-490b-a34a-83955b7c844e.tif", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLTgxNDBlYmU1LTI5NWMtNDkwYi1hMzRhLTgzOTU1YjdjODQ0ZS50aWY=", "size": 1446772 } }, @@ -41,7 +41,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/evelyn's photo.jpg", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bidzIHBob3RvLmpwZw==", "transfer": null, "filegrpuse": "original", "removedtime": null, @@ -49,7 +49,7 @@ "checksum": "d2bed92b73c7090bb30a0b30016882e7069c437488e1513e9deaacbe29d38d92", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/evelyn_s_photo.jpg", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLmpwZw==", "size": 158131 } }, @@ -59,7 +59,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%metadata/metadata.csv", + "originallocation": "JVNJUERpcmVjdG9yeSVtZXRhZGF0YS9tZXRhZGF0YS5jc3Y=", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -67,7 +67,7 @@ "checksum": "e8121d8a660e2992872f0b67923d2d08dde9a1ba72dfd58e5a31e68fbac3633c", "enteredsystem": "2015-06-24T17:32:09Z", "modificationtime": "2015-06-24T17:32:09Z", - "currentlocation": "%SIPDirectory%objects/metadata/metadata.csv", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL21ldGFkYXRhLmNzdg==", "size": 154 } }, @@ -77,7 +77,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%metadata/foo/foo.txt", + "originallocation": "JVNJUERpcmVjdG9yeSVtZXRhZGF0YS9mb28vZm9vLnR4dA==", "transfer": null, "filegrpuse": "metadata", "removedtime": null, @@ -85,7 +85,7 @@ "checksum": "", "enteredsystem": "2015-06-24T17:32:09Z", "modificationtime": "2015-06-24T17:32:09Z", - "currentlocation": "%SIPDirectory%objects/metadata/foo/foo.txt", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL2Zvby9mb28udHh0", "size": 154 } } diff --git a/src/MCPClient/tests/fixtures/reingest-preservation.json b/src/MCPClient/tests/fixtures/reingest-preservation.json index bb15da04d3..a9073ee378 100644 --- a/src/MCPClient/tests/fixtures/reingest-preservation.json +++ b/src/MCPClient/tests/fixtures/reingest-preservation.json @@ -40,7 +40,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/evelyn_s_photo-d8cc7af7-284a-42f5-b7f4-e181a0efc35f.tif", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLWQ4Y2M3YWY3LTI4NGEtNDJmNS1iN2Y0LWUxODFhMGVmYzM1Zi50aWY=", "transfer": null, "filegrpuse": "preservation", "removedtime": null, @@ -49,7 +49,7 @@ "checksum": "d82448f154b9185bc777ecb0a3602760eb76ba85dd3098f073b2c91a03f571e9", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/evelyn_s_photo-d8cc7af7-284a-42f5-b7f4-e181a0efc35f.tif", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLWQ4Y2M3YWY3LTI4NGEtNDJmNS1iN2Y0LWUxODFhMGVmYzM1Zi50aWY=", "size": 1446772 } }, diff --git a/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py b/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py index 7d8046b77c..1a02fa83a5 100644 --- a/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py +++ b/src/MCPClient/tests/test_archivematicaCreateMETSMetadataXML.py @@ -82,7 +82,7 @@ def _make_metadata_file(rel_path): return File.objects.create( uuid=uuid4(), sip_id=sip.uuid, - currentlocation=f"%SIPDirectory%{rel_path}", + currentlocation=f"%SIPDirectory%{rel_path}".encode(), ) return _make_metadata_file diff --git a/src/MCPClient/tests/test_assign_file_uuids.py b/src/MCPClient/tests/test_assign_file_uuids.py new file mode 100644 index 0000000000..385c1f5bad --- /dev/null +++ b/src/MCPClient/tests/test_assign_file_uuids.py @@ -0,0 +1,146 @@ +import uuid + +import pytest +from assign_file_uuids import call +from job import Job +from main import models + + +@pytest.fixture +def transfer(): + return models.Transfer.objects.create( + currentlocation=r"%transferDirectory%", + ) + + +@pytest.fixture +def sip_dir(tmp_path): + sip_dir = tmp_path / "dir" + sip_dir.mkdir() + + # Create a directory to test subdirectory filtering. + contents_dir = sip_dir / "contents" + contents_dir.mkdir() + (contents_dir / "file-in.txt").touch() + + # Create a file outside the contents directory. + (sip_dir / "file-out.txt").touch() + + # Create a directory to represent a reingest. + (sip_dir / "reingest").mkdir() + + return sip_dir + + +@pytest.mark.django_db +def test_call_creates_transfer_files_and_events(mocker, sip_dir, transfer): + job = mocker.MagicMock( + spec=Job, + args=[ + "assign_file_uuids.py", + "--transferUUID", + str(transfer.uuid), + "--sipDirectory", + str(sip_dir), + "--filterSubdir", + "contents", + ], + ) + + # Verify there are no files or events yet. + assert models.File.objects.filter(transfer=transfer).count() == 0 + assert models.Event.objects.filter().count() == 0 + + call([job]) + + # Verify the job succeeded. + job.set_status.assert_called_once_with(0) + + # Verify a file and an ingestion event were created. + f = models.File.objects.get(transfer=transfer) + assert models.Event.objects.filter(file_uuid=f, event_type="ingestion").count() == 1 + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "job_arguments", + [ + (["assign_file_uuids.py"]), + ( + [ + "assign_file_uuids.py", + "--transferUUID", + str(uuid.uuid4()), + "--sipUUID", + str(uuid.uuid4()), + ] + ), + ], + ids=["no_transfer_nor_sip_provided", "transfer_and_sip_provided"], +) +def test_call_validates_job_arguments(mocker, job_arguments): + job = mocker.MagicMock( + spec=Job, + args=job_arguments, + ) + + call([job]) + + # Verify the job did not succeed. + job.set_status.assert_called_once_with(2) + job.print_error.assert_called_once_with( + "SIP exclusive-or Transfer UUID must be defined" + ) + + +@pytest.mark.django_db +def test_call_updates_transfer_file_on_reingest(mocker, sip_dir, transfer): + # Fake METS parsing and return static file information. + mocker.patch("metsrw.METSDocument.fromfile") + mocker.patch("assign_file_uuids.find_mets_file") + file_info = { + "uuid": str(uuid.uuid4()), + "filegrpuse": "original", + "original_path": str(sip_dir / "contents" / "file-in.txt"), + "current_path": str(sip_dir / "reingest" / "file-in.txt"), + } + mocker.patch("assign_file_uuids.get_file_info_from_mets", return_value=file_info) + + job = mocker.MagicMock( + spec=Job, + args=[ + "assign_file_uuids.py", + "--transferUUID", + str(transfer.uuid), + "--sipDirectory", + str(sip_dir), + "--filterSubdir", + "contents", + ], + ) + + # Mark the transfer as reingested. + transfer.type = models.Transfer.ARCHIVEMATICA_AIP + transfer.save() + + # Verify there are no files or events yet. + assert models.File.objects.filter(transfer=transfer).count() == 0 + assert models.Event.objects.filter().count() == 0 + + call([job]) + + # Verify the job succeeded. + job.set_status.assert_called_once_with(0) + + # Verify a file and a reingestion event were created. + f = models.File.objects.get(transfer=transfer) + assert str(f.uuid) == file_info["uuid"] + assert f.filegrpuse == file_info["filegrpuse"] + assert ( + models.Event.objects.filter(file_uuid=f, event_type="reingestion").count() == 1 + ) + + # Verify the location of the file was updated. + job.print_output.assert_called_with( + "Updating current location for", str(f.uuid), "with", file_info + ) diff --git a/src/MCPClient/tests/test_assign_uuids_to_directories.py b/src/MCPClient/tests/test_assign_uuids_to_directories.py new file mode 100755 index 0000000000..500ff985d5 --- /dev/null +++ b/src/MCPClient/tests/test_assign_uuids_to_directories.py @@ -0,0 +1,46 @@ +import pytest +from assign_uuids_to_directories import main +from job import Job +from main import models + + +@pytest.fixture +def transfer(): + return models.Transfer.objects.create( + currentlocation=r"%transferDirectory%", + ) + + +@pytest.fixture +def transfer_dir(tmp_path): + transfer_dir = tmp_path / "dir" + transfer_dir.mkdir() + + # Create a directory with contents. + contents_dir = transfer_dir / "contents" + contents_dir.mkdir() + (contents_dir / "file-in.txt").touch() + + # Create a file outside the contents directory. + (transfer_dir / "file-out.txt").touch() + + # Create a directory to represent a reingest. + (transfer_dir / "reingest").mkdir() + + return transfer_dir + + +@pytest.mark.django_db +def test_main(mocker, transfer, transfer_dir): + job = mocker.Mock(spec=Job) + include_dirs = True + + # Verify there are no directories. + models.Directory.objects.count() == 0 + + result = main(job, str(transfer_dir), transfer.uuid, include_dirs) + + assert result == 0 + + # Verify two directories were created. + models.Directory.objects.filter(transfer=transfer).count() == 2 diff --git a/src/MCPClient/tests/test_create_mets_v2.py b/src/MCPClient/tests/test_create_mets_v2.py index 04aca38c56..b7cdc78bd4 100644 --- a/src/MCPClient/tests/test_create_mets_v2.py +++ b/src/MCPClient/tests/test_create_mets_v2.py @@ -85,8 +85,8 @@ def metadata_csv(sip, sip_path, objects_path): file_obj = File.objects.create( uuid=uuid.uuid4(), sip=sip, - originallocation=originallocation, - currentlocation=currentlocation, + originallocation=originallocation.encode(), + currentlocation=currentlocation.encode(), size=1024, filegrpuse="metadata", checksum="f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b", @@ -135,8 +135,8 @@ def file_obj(db, sip, sip_path, file_path): file_obj = File.objects.create( uuid=uuid.uuid4(), sip=sip, - originallocation=originallocation, - currentlocation=currentlocation, + originallocation=originallocation.encode(), + currentlocation=currentlocation.encode(), size=113318, filegrpuse="original", checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", diff --git a/src/MCPClient/tests/test_create_transfer_mets.py b/src/MCPClient/tests/test_create_transfer_mets.py index ca2b848eea..b0e03b1060 100644 --- a/src/MCPClient/tests/test_create_transfer_mets.py +++ b/src/MCPClient/tests/test_create_transfer_mets.py @@ -74,8 +74,8 @@ def file_obj(db, transfer, tmp_path, file_path): file_obj = File.objects.create( uuid=uuid.uuid4(), transfer=transfer, - originallocation=file_obj_path, - currentlocation=file_obj_path, + originallocation=file_obj_path.encode(), + currentlocation=file_obj_path.encode(), removedtime=None, size=113318, checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", @@ -94,8 +94,8 @@ def file_obj2(db, transfer, tmp_path, file_path2): return File.objects.create( uuid=uuid.uuid4(), transfer=transfer, - originallocation=file_obj_path, - currentlocation=file_obj_path, + originallocation=file_obj_path.encode(), + currentlocation=file_obj_path.encode(), removedtime=None, size=113318, checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", @@ -111,8 +111,8 @@ def dir_obj(db, transfer, tmp_path, subdir_path): dir_obj = Directory.objects.create( uuid=uuid.uuid4(), transfer=transfer, - originallocation=dir_obj_path, - currentlocation=dir_obj_path, + originallocation=dir_obj_path.encode(), + currentlocation=dir_obj_path.encode(), ) dir_obj.identifiers.create(type="TEST ID", value="12345") diff --git a/src/MCPClient/tests/test_filename_change.py b/src/MCPClient/tests/test_filename_change.py index 4af33cc390..fb0354c58a 100644 --- a/src/MCPClient/tests/test_filename_change.py +++ b/src/MCPClient/tests/test_filename_change.py @@ -73,8 +73,8 @@ def transfer_dir_obj(db, transfer, tmp_path, subdir_path): dir_obj = Directory.objects.create( uuid=uuid.uuid4(), transfer=transfer, - originallocation=dir_obj_path, - currentlocation=dir_obj_path, + originallocation=dir_obj_path.encode(), + currentlocation=dir_obj_path.encode(), ) return dir_obj @@ -92,8 +92,8 @@ def sip_dir_obj(db, sip, tmp_path, subdir_path): dir_obj = Directory.objects.create( uuid=uuid.uuid4(), sip=sip, - originallocation=dir_obj_path, - currentlocation=dir_obj_path, + originallocation=dir_obj_path.encode(), + currentlocation=dir_obj_path.encode(), ) return dir_obj @@ -113,8 +113,8 @@ def sip_file_obj(db, sip, tmp_path, subdir_path): return File.objects.create( uuid=uuid.uuid4(), sip=sip, - originallocation=relative_path, - currentlocation=relative_path, + originallocation=relative_path.encode(), + currentlocation=relative_path.encode(), removedtime=None, size=113318, checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", @@ -147,8 +147,8 @@ def multiple_transfer_file_objs(db, transfer, tmp_path, multiple_file_paths): File( uuid=uuid.uuid4(), transfer=transfer, - originallocation=relative_path, - currentlocation=relative_path, + originallocation=relative_path.encode(), + currentlocation=relative_path.encode(), removedtime=None, size=113318, checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", @@ -212,7 +212,9 @@ def test_change_object_names(self): "%sharedPath%currentlyProcessing", str(self.tmpdir) ) for f in File.objects.filter(transfer_id=self.transfer_uuid): - path = f.currentlocation.replace("%transferDirectory%", transfer_path) + path = f.currentlocation.decode().replace( + "%transferDirectory%", transfer_path + ) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) @@ -245,7 +247,7 @@ def test_change_object_names(self): ) ) assert File.objects.get( - currentlocation="%transferDirectory%objects/takusan_directories/need_name_change/checking_here/evelyn_s_photo.jpg" + currentlocation=b"%transferDirectory%objects/takusan_directories/need_name_change/checking_here/evelyn_s_photo.jpg" ) event = Event.objects.get( file_uuid="47813453-6872-442b-9d65-6515be3c5aa1", @@ -260,7 +262,7 @@ def test_change_object_names(self): ) ) assert File.objects.get( - currentlocation="%transferDirectory%objects/no_name_change/needed_here/lion.svg" + currentlocation=b"%transferDirectory%objects/no_name_change/needed_here/lion.svg" ) assert not Event.objects.filter( file_uuid="60e5c61b-14ef-4e92-89ec-9b9201e68adb", @@ -278,7 +280,7 @@ def test_change_object_names(self): ) ) assert File.objects.get( - currentlocation="%transferDirectory%objects/takusan_directories/need_name_change/checking_here/lionXie_Zhen_.svg" + currentlocation=b"%transferDirectory%objects/takusan_directories/need_name_change/checking_here/lionXie_Zhen_.svg" ) assert Event.objects.filter( file_uuid="791e07ea-ad44-4315-b55b-44ec771e95cf", @@ -289,7 +291,7 @@ def test_change_object_names(self): os.path.join(transfer_path, "objects", "has_space", "lion.svg") ) assert File.objects.get( - currentlocation="%transferDirectory%objects/has_space/lion.svg" + currentlocation=b"%transferDirectory%objects/has_space/lion.svg" ) assert Event.objects.filter( file_uuid="8a1f0b59-cf94-47ef-8078-647b77c8a147", @@ -340,9 +342,9 @@ def test_change_transfer_with_multiple_files( original_location = file_obj.currentlocation file_obj.refresh_from_db() - assert file_obj.currentlocation != original_location - assert subdir_path.as_posix() not in file_obj.currentlocation - assert "bulk-file" in file_obj.currentlocation + assert file_obj.currentlocation.decode() != original_location + assert subdir_path.as_posix() not in file_obj.currentlocation.decode() + assert "bulk-file" in file_obj.currentlocation.decode() verify_event_details( Event.objects.get(file_uuid=file_obj.uuid, event_type="filename change") ) @@ -367,7 +369,7 @@ def test_change_transfer_with_directory_uuids( transfer_dir_obj.refresh_from_db() assert transfer_dir_obj.currentlocation != original_location - assert subdir_path.as_posix() not in transfer_dir_obj.currentlocation + assert subdir_path.as_posix() not in transfer_dir_obj.currentlocation.decode() @pytest.mark.django_db @@ -387,11 +389,11 @@ def test_change_sip(tmp_path, sip, subdir_path, sip_dir_obj, sip_file_obj): sip_dir_obj.refresh_from_db() assert sip_dir_obj.currentlocation != original_dir_location - assert subdir_path.as_posix() not in sip_dir_obj.currentlocation + assert subdir_path.as_posix() not in sip_dir_obj.currentlocation.decode() original_file_location = sip_file_obj.currentlocation sip_file_obj.refresh_from_db() assert sip_file_obj.currentlocation != original_file_location - assert subdir_path.as_posix() not in sip_file_obj.currentlocation - assert "file" in sip_file_obj.currentlocation + assert subdir_path.as_posix() not in sip_file_obj.currentlocation.decode() + assert "file" in sip_file_obj.currentlocation.decode() diff --git a/src/MCPClient/tests/test_load_premis_events_from_xml.py b/src/MCPClient/tests/test_load_premis_events_from_xml.py index 5f35e08993..9178cb328f 100644 --- a/src/MCPClient/tests/test_load_premis_events_from_xml.py +++ b/src/MCPClient/tests/test_load_premis_events_from_xml.py @@ -436,7 +436,7 @@ def test_get_invalid_file_identifiers(mocker): valid_filenames = ["".join([prefix, "object1"]), "".join([prefix, "object2"])] def mock_filter(originallocation): - return_value = originallocation in valid_filenames + return_value = originallocation.decode() in valid_filenames return mocker.Mock(**{"exists.return_value": return_value}) file_queryset = mocker.Mock(**{"filter.side_effect": mock_filter}) @@ -756,8 +756,8 @@ def transfer_file(db, transfer, tmp_path, file_path): result = File.objects.create( uuid=uuid.uuid4(), transfer=transfer, - originallocation=path, - currentlocation=path, + originallocation=path.encode(), + currentlocation=path.encode(), removedtime=None, size=113318, checksum="35e0cc683d75704fc5b04fc3633f6c654e10cd3af57471271f370309c7ff9dba", diff --git a/src/MCPClient/tests/test_parse_dataverse.py b/src/MCPClient/tests/test_parse_dataverse.py index cdc5240b7d..4dfb08520f 100644 --- a/src/MCPClient/tests/test_parse_dataverse.py +++ b/src/MCPClient/tests/test_parse_dataverse.py @@ -104,9 +104,9 @@ def test_mapping(self): for test_case in test_cases: assert self.mets.get_file(file_uuid=test_case["file_uuid"]) in mapping assert models.File.objects.get( - currentlocation=test_case.get("file_location", "").format( - self.transfer_location - ) + currentlocation=test_case.get("file_location", "") + .format(self.transfer_location) + .encode() ) in list(mapping.values()) def test_set_filegroups(self): @@ -162,9 +162,9 @@ def test_set_filegroups(self): for test_case in test_cases: assert ( models.File.objects.get( - currentlocation=test_case.get("file_location", "").format( - self.transfer_location - ) + currentlocation=test_case.get("file_location", "") + .format(self.transfer_location) + .encode() ).filegrpuse == test_case["file_group_use"] ) diff --git a/src/MCPClient/tests/test_parse_mets_to_db.py b/src/MCPClient/tests/test_parse_mets_to_db.py index 81bb3b20a2..3fa687abb3 100644 --- a/src/MCPClient/tests/test_parse_mets_to_db.py +++ b/src/MCPClient/tests/test_parse_mets_to_db.py @@ -582,8 +582,8 @@ def test_insert_file_info(self): orig = models.File.objects.get(uuid=self.ORIG_INFO["uuid"]) assert orig.sip_id == uuid.UUID(self.SIP_UUID) assert orig.transfer is None - assert orig.originallocation == self.ORIG_INFO["original_path"] - assert orig.currentlocation == self.ORIG_INFO["current_path"] + assert orig.originallocation.decode() == self.ORIG_INFO["original_path"] + assert orig.currentlocation.decode() == self.ORIG_INFO["current_path"] assert orig.filegrpuse == self.ORIG_INFO["use"] assert orig.filegrpuuid == "" assert orig.checksum == self.ORIG_INFO["checksum"] @@ -603,8 +603,8 @@ def test_insert_file_info(self): pres = models.File.objects.get(uuid=self.PRES_INFO["uuid"]) assert pres.sip_id == uuid.UUID(self.SIP_UUID) assert pres.transfer is None - assert pres.originallocation == self.PRES_INFO["original_path"] - assert pres.currentlocation == self.PRES_INFO["current_path"] + assert pres.originallocation.decode() == self.PRES_INFO["original_path"] + assert pres.currentlocation.decode() == self.PRES_INFO["current_path"] assert pres.filegrpuse == self.PRES_INFO["use"] assert pres.filegrpuuid == "" assert pres.checksum == self.PRES_INFO["checksum"] @@ -623,8 +623,8 @@ def test_insert_file_info(self): mets = models.File.objects.get(uuid=self.METS_INFO["uuid"]) assert mets.sip_id == uuid.UUID(self.SIP_UUID) assert mets.transfer is None - assert mets.originallocation == self.METS_INFO["original_path"] - assert mets.currentlocation == self.METS_INFO["current_path"] + assert mets.originallocation.decode() == self.METS_INFO["original_path"] + assert mets.currentlocation.decode() == self.METS_INFO["current_path"] assert mets.filegrpuse == self.METS_INFO["use"] assert mets.filegrpuuid == "" assert mets.checksum == self.METS_INFO["checksum"] diff --git a/src/MCPClient/tests/test_pid_components.py b/src/MCPClient/tests/test_pid_components.py index 13ff1b3e95..70e05682aa 100644 --- a/src/MCPClient/tests/test_pid_components.py +++ b/src/MCPClient/tests/test_pid_components.py @@ -64,7 +64,7 @@ def transfer(db): return Transfer.objects.create( uuid="29460c83-957e-481f-9ca2-873bca42229a", type="Standard", - currentlocation="%sharedPath%watchedDirectories/SIPCreation/completedTransfers/test-3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e/", + currentlocation=b"%sharedPath%watchedDirectories/SIPCreation/completedTransfers/test-3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e/", ) @@ -138,11 +138,11 @@ def files(db): uuid="06da9555-dc5b-425c-b967-6f30a740f1c3", checksum="012661d0de5ed60973e5cb0f123df74e95118734345b2bdf35432ac7442dd8c8", checksumtype="sha256", - currentlocation="%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/directory_tree.txt", + currentlocation=b"%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/directory_tree.txt", enteredsystem="2019-05-02T11:16:08.195Z", filegrpuse="metadata", modificationtime="2019-05-02T11:16:08.195Z", - originallocation="%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/directory_tree.txt", + originallocation=b"%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/directory_tree.txt", sip_id=PACKAGE_UUID, size=233, ), @@ -150,11 +150,11 @@ def files(db): uuid="52e4900a-b096-4815-856a-81b3cbe0952d", checksum="cd2b6311b1552b32c9bed90209bb219ab8216fd777b12dd36ad06aba959cba6e", checksumtype="sha256", - currentlocation="%SIPDirectory%objects/images/image_001.jpg", + currentlocation=b"%SIPDirectory%objects/images/image_001.jpg", enteredsystem="2019-05-02T11:15:31.856Z", filegrpuse="original", modificationtime="2019-04-23T12:25:01Z", - originallocation="%transferDirectory%objects/images/image_001.jpg", + originallocation=b"%transferDirectory%objects/images/image_001.jpg", sip_id=PACKAGE_UUID, size=11, transfer_id="29460c83-957e-481f-9ca2-873bca42229a", @@ -163,11 +163,11 @@ def files(db): uuid="697c407f-7a43-43d2-b7e2-fefc956ea5fd", checksum="cd2b6311b1552b32c9bed90209bb219ab8216fd777b12dd36ad06aba959cba6e", checksumtype="sha256", - currentlocation="%SIPDirectory%objects/documents/Document001.doc", + currentlocation=b"%SIPDirectory%objects/documents/Document001.doc", enteredsystem="2019-05-02T11:15:31.869Z", filegrpuse="original", modificationtime="2019-04-23T12:26:18Z", - originallocation="%transferDirectory%objects/documents/Document001.doc", + originallocation=b"%transferDirectory%objects/documents/Document001.doc", sip_id=PACKAGE_UUID, size=11, transfer_id="29460c83-957e-481f-9ca2-873bca42229a", @@ -176,11 +176,11 @@ def files(db): uuid="c91d7725-f363-4e8a-bde1-0f5416b4f7f8", checksum="e4bdbed4732746727a013431d87126a43e99ee7350c4d7ba27eb4927d8d06f15", checksumtype="sha256", - currentlocation="%SIPDirectory%objects/submissionDocumentation/transfer-pid_tests-29460c83-957e-481f-9ca2-873bca42229a/METS.xml", + currentlocation=b"%SIPDirectory%objects/submissionDocumentation/transfer-pid_tests-29460c83-957e-481f-9ca2-873bca42229a/METS.xml", enteredsystem="2019-05-02T11:16:00.917Z", filegrpuse="submissionDocumentation", modificationtime="2019-05-02T11:16:00.917Z", - originallocation="%SIPDirectory%objects/submissionDocumentation/transfer-pid_tests-29460c83-957e-481f-9ca2-873bca42229a/METS.xml", + originallocation=b"%SIPDirectory%objects/submissionDocumentation/transfer-pid_tests-29460c83-957e-481f-9ca2-873bca42229a/METS.xml", sip_id=PACKAGE_UUID, size=37119, ), @@ -188,11 +188,11 @@ def files(db): uuid="d94dbb6b-1082-4747-9d4c-4ddea8ce85b8", checksum="28045d295e79a3666bd4c9a09536e2b6c65e2f917972e15c46c6eb2cad1e5700", checksumtype="sha256", - currentlocation="%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/identifiers.json", + currentlocation=b"%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/identifiers.json", enteredsystem="2019-05-02T11:16:08.217Z", filegrpuse="metadata", modificationtime="2019-05-02T11:16:08.217Z", - originallocation="%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/identifiers.json", + originallocation=b"%SIPDirectory%objects/metadata/transfers/pid_tests-29460c83-957e-481f-9ca2-873bca42229a/identifiers.json", sip_id=PACKAGE_UUID, size=1768, ), @@ -206,46 +206,46 @@ def directories(db): [ Directory( uuid="3cc124e1-4e5c-433b-8f15-a7831d70145a", - originallocation="%transferDirectory%metadata/", + originallocation=b"%transferDirectory%metadata/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%transferDirectory%metadata/", + currentlocation=b"%transferDirectory%metadata/", enteredsystem="2019-04-23T12:45:34.925Z", ), Directory( uuid="41b3fcfd-36ed-4cf7-b5af-56127238b362", - originallocation="%transferDirectory%logs/", + originallocation=b"%transferDirectory%logs/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%transferDirectory%logs/", + currentlocation=b"%transferDirectory%logs/", enteredsystem="2019-04-23T12:45:34.925Z", ), Directory( uuid="966755bd-0ae3-4f85-b4ec-b359fefeff33", sip_id=PACKAGE_UUID, - originallocation="%transferDirectory%objects/images/", + originallocation=b"%transferDirectory%objects/images/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%SIPDirectory%objects/images/", + currentlocation=b"%SIPDirectory%objects/images/", enteredsystem="2019-04-23T12:45:34.925Z", ), Directory( uuid="d298dd3f-c5d1-4445-99fe-09123fba8b30", sip_id=PACKAGE_UUID, - originallocation="%transferDirectory%objects/documents/", + originallocation=b"%transferDirectory%objects/documents/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%SIPDirectory%objects/documents/", + currentlocation=b"%SIPDirectory%objects/documents/", enteredsystem="2019-04-23T12:45:34.925Z", ), Directory( uuid="e7ae9b01-4c07-4915-a7a0-1833b1078a5b", - originallocation="%transferDirectory%logs/fileMeta/", + originallocation=b"%transferDirectory%logs/fileMeta/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%transferDirectory%logs/fileMeta/", + currentlocation=b"%transferDirectory%logs/fileMeta/", enteredsystem="2019-04-23T12:45:34.925Z", ), Directory( uuid="f9e8c91a-bfbc-48f0-8ff0-eed210ff6fde", - originallocation="%transferDirectory%metadata/submissionDocumentation/", + originallocation=b"%transferDirectory%metadata/submissionDocumentation/", transfer_id="29460c83-957e-481f-9ca2-873bca42229a", - currentlocation="%transferDirectory%metadata/submissionDocumentation/", + currentlocation=b"%transferDirectory%metadata/submissionDocumentation/", enteredsystem="2019-04-23T12:45:34.925Z", ), ] diff --git a/src/MCPClient/tests/test_store_file_modification.py b/src/MCPClient/tests/test_store_file_modification.py index 548d9da4c7..264d406397 100644 --- a/src/MCPClient/tests/test_store_file_modification.py +++ b/src/MCPClient/tests/test_store_file_modification.py @@ -41,7 +41,9 @@ def test_store_file_modification_dates(self): transfer.save() for f in models.File.objects.filter(transfer_id=self.transfer_uuid): - path = f.currentlocation.replace("%transferDirectory%", transfer_path) + path = f.currentlocation.decode().replace( + "%transferDirectory%", transfer_path + ) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) diff --git a/src/MCPClient/tests/test_upload_qubit.py b/src/MCPClient/tests/test_upload_qubit.py index 14962bc902..8e89da4fe1 100644 --- a/src/MCPClient/tests/test_upload_qubit.py +++ b/src/MCPClient/tests/test_upload_qubit.py @@ -44,7 +44,7 @@ def job(sip, tmp_path): def access(db, sip): return models.Access.objects.create( sipuuid=sip.uuid, - target=pickle.dumps({"target": "atom-description-id"}, protocol=0), + target=pickle.dumps({"target": "atom-description-id"}, protocol=0).decode(), ) diff --git a/src/MCPClient/tests/test_verify_checksum.py b/src/MCPClient/tests/test_verify_checksum.py index 8bbddae90e..f39cda9f5f 100644 --- a/src/MCPClient/tests/test_verify_checksum.py +++ b/src/MCPClient/tests/test_verify_checksum.py @@ -71,8 +71,8 @@ def transfer(self, db, user): checksum="f78615cd834f7fb84832177e73f13e3479f5b5b22ae7a9506c7fa0a14fd9df9e", enteredsystem="2017-01-04T19:35:20Z", modificationtime="2017-01-04T19:35:20Z", - originallocation="%transferDirectory%objects/has space/lion.svg", - currentlocation="%transferDirectory%objects/has space/lion.svg", + originallocation=b"%transferDirectory%objects/has space/lion.svg", + currentlocation=b"%transferDirectory%objects/has space/lion.svg", size=18324, ) return transfer diff --git a/src/MCPServer/lib/server/packages.py b/src/MCPServer/lib/server/packages.py index 637fb5b1eb..2b561c3dc8 100644 --- a/src/MCPServer/lib/server/packages.py +++ b/src/MCPServer/lib/server/packages.py @@ -500,18 +500,20 @@ def parts(self): def get_file_replacement_mapping(file_obj, unit_directory): mapping = BASE_REPLACEMENTS.copy() - dirname = os.path.dirname(file_obj.currentlocation) - name, ext = os.path.splitext(file_obj.currentlocation) + dirname = os.path.dirname(file_obj.currentlocation.decode()) + name, ext = os.path.splitext(file_obj.currentlocation.decode()) name = os.path.basename(name) - absolute_path = file_obj.currentlocation.replace(r"%SIPDirectory%", unit_directory) + absolute_path = file_obj.currentlocation.decode().replace( + r"%SIPDirectory%", unit_directory + ) absolute_path = absolute_path.replace(r"%transferDirectory%", unit_directory) mapping.update( { r"%fileUUID%": str(file_obj.pk), - r"%originalLocation%": file_obj.originallocation, - r"%currentLocation%": file_obj.currentlocation, + r"%originalLocation%": file_obj.originallocation.decode(), + r"%currentLocation%": file_obj.currentlocation.decode(), r"%fileGrpUse%": file_obj.filegrpuse, r"%fileDirectory%": dirname, r"%fileName%": name, diff --git a/src/MCPServer/tests/test_package.py b/src/MCPServer/tests/test_package.py index 9ddb1a936c..85e81b8512 100644 --- a/src/MCPServer/tests/test_package.py +++ b/src/MCPServer/tests/test_package.py @@ -163,11 +163,10 @@ def test_reload_file_list(tmp_path): first_file = objects_path / "file.txt" first_file.touch() + current_location = Path(transfer.REPLACEMENT_PATH_STRING, "objects", "file.txt") kwargs = { "uuid": uuid.uuid4(), - "currentlocation": "".join( - [transfer.REPLACEMENT_PATH_STRING, str(Path("/objects/file.txt"))] - ), + "currentlocation": bytes(current_location), "filegrpuse": "original", "transfer_id": transfer_uuid, } @@ -188,16 +187,14 @@ def test_reload_file_list(tmp_path): if file_info.get("%fileUUID%") != "None": continue assert file_info.get("%relativeLocation%") == str(new_file) - file_path = "".join( - [ - transfer.REPLACEMENT_PATH_STRING, - "/objects", - file_info.get("%relativeLocation%").split("objects")[1], - ] + file_path = Path( + transfer.REPLACEMENT_PATH_STRING, + "objects", + Path(file_info.get("%relativeLocation%")).relative_to(objects_path), ) kwargs = { "uuid": uuid.uuid4(), - "currentlocation": file_path, + "currentlocation": bytes(file_path), "filegrpuse": "original", "transfer_id": transfer_uuid, } @@ -214,18 +211,16 @@ def test_reload_file_list(tmp_path): new_file = sub_dir / "another_new_file.txt" new_file.touch() for file_count, file_info in enumerate(transfer.files(None, None, "/objects"), 1): - file_path = "".join( - [ - transfer.REPLACEMENT_PATH_STRING, - "/objects", - file_info.get("%relativeLocation%").split("objects")[1], - ] - ) if file_info.get("%fileUUID%") != "None": continue + file_path = Path( + transfer.REPLACEMENT_PATH_STRING, + "objects", + Path(file_info.get("%relativeLocation%")).relative_to(Path(objects_path)), + ) kwargs = { "uuid": uuid.uuid4(), - "currentlocation": file_path, + "currentlocation": bytes(file_path), "filegrpuse": "original", "transfer_id": transfer_uuid, } @@ -279,11 +274,10 @@ def test_package_files_with_non_ascii_names(tmp_path): file_.touch() # Create a File model instance for the transfer file + current_location = Path(transfer.REPLACEMENT_PATH_STRING, "objects", "montréal.txt") kwargs = { "uuid": uuid.uuid4(), - "currentlocation": "".join( - [transfer.REPLACEMENT_PATH_STRING, "/objects/montréal.txt"] - ), + "currentlocation": bytes(current_location), "filegrpuse": "original", "transfer_id": transfer_uuid, } @@ -295,7 +289,7 @@ def test_package_files_with_non_ascii_names(tmp_path): # And it is the file we just created assert result[0]["%fileUUID%"] == str(kwargs["uuid"]) - assert result[0]["%currentLocation%"] == kwargs["currentlocation"] + assert result[0]["%currentLocation%"] == current_location.as_posix() assert result[0]["%fileGrpUse%"] == kwargs["filegrpuse"] diff --git a/src/MCPServer/tests/test_workflow.py b/src/MCPServer/tests/test_workflow.py index 8adbce0d0b..0bbb9196de 100644 --- a/src/MCPServer/tests/test_workflow.py +++ b/src/MCPServer/tests/test_workflow.py @@ -2,7 +2,7 @@ from io import StringIO import pytest -from django.utils.translation import ugettext_lazy +from django.utils.translation import gettext_lazy from server import translation from server import workflow @@ -18,9 +18,9 @@ def test_invert_job_statuses(mocker): mocker.patch( "server.jobs.Job.STATUSES", ( - (1, ugettext_lazy("Uno")), - (2, ugettext_lazy("Dos")), - (3, ugettext_lazy("Tres")), + (1, gettext_lazy("Uno")), + (2, gettext_lazy("Dos")), + (3, gettext_lazy("Tres")), ), ) ret = workflow._invert_job_statuses() diff --git a/src/archivematicaCommon/lib/databaseFunctions.py b/src/archivematicaCommon/lib/databaseFunctions.py index 06073f4a0c..0fe9866544 100644 --- a/src/archivematicaCommon/lib/databaseFunctions.py +++ b/src/archivematicaCommon/lib/databaseFunctions.py @@ -97,8 +97,8 @@ def insertIntoFiles( kwargs = { "uuid": str(fileUUID), - "originallocation": originalLocation, - "currentlocation": filePath, + "originallocation": originalLocation.encode(), + "currentlocation": filePath.encode(), "enteredsystem": enteredSystem, "filegrpuse": use, } diff --git a/src/archivematicaCommon/lib/dicts.py b/src/archivematicaCommon/lib/dicts.py index cece83ddbd..b1bbe063e8 100644 --- a/src/archivematicaCommon/lib/dicts.py +++ b/src/archivematicaCommon/lib/dicts.py @@ -143,16 +143,16 @@ def frommodel(type_="file", sip=None, file_=None, expand_path=True): if expand_path and sipdir is not None: base_location = base_location.replace("%sharedPath%", shared_path) - origin = file_.originallocation.replace( + origin = file_.originallocation.decode().replace( "%transferDirectory%", base_location ) - current_location = file_.currentlocation.replace( + current_location = file_.currentlocation.decode().replace( "%transferDirectory%", base_location ) current_location = current_location.replace("%SIPDirectory%", sipdir) else: - origin = file_.originallocation - current_location = file_.currentlocation + origin = file_.originallocation.decode() + current_location = file_.currentlocation.decode() rd["%originalLocation%"] = origin rd["%currentLocation%"] = current_location rd["%fileDirectory%"] = os.path.dirname(current_location) diff --git a/src/archivematicaCommon/lib/elasticSearchFunctions.py b/src/archivematicaCommon/lib/elasticSearchFunctions.py index be4702bbfa..49d33da7d8 100644 --- a/src/archivematicaCommon/lib/elasticSearchFunctions.py +++ b/src/archivematicaCommon/lib/elasticSearchFunctions.py @@ -808,7 +808,9 @@ def _index_transfer_files( filepath, path ).lstrip("data/") try: - f = File.objects.get(currentlocation=currentlocation, transfer_id=uuid) + f = File.objects.get( + currentlocation=currentlocation.encode(), transfer_id=uuid + ) file_uuid = str(f.uuid) formats = _get_file_formats(f) bulk_extractor_reports = _list_bulk_extractor_reports(path, file_uuid) diff --git a/src/archivematicaCommon/lib/fileOperations.py b/src/archivematicaCommon/lib/fileOperations.py index c5d0ad25f0..dd7a6256a0 100644 --- a/src/archivematicaCommon/lib/fileOperations.py +++ b/src/archivematicaCommon/lib/fileOperations.py @@ -190,7 +190,7 @@ def updateDirectoryLocation( files = File.objects.filter(**kwargs) for f in files: - f.currentlocation = f.currentlocation.replace(srcDB, dstDB) + f.currentlocation = f.currentlocation.decode().replace(srcDB, dstDB).encode() f.save() if os.path.isdir(dst): if dst.endswith("/"): @@ -221,7 +221,7 @@ def updateFileLocation2( # Fetch the file UUID kwargs = { "removedtime__isnull": True, - "currentlocation": srcDB, + "currentlocation": srcDB.encode(), unitIdentifierType: unitIdentifier, } @@ -245,7 +245,7 @@ def updateFileLocation2( printfn("Moving", src, "to", dst) shutil.move(src, dst) # Update the DB - f.currentlocation = dstDB + f.currentlocation = dstDB.encode() f.save() @@ -285,7 +285,7 @@ def updateFileLocation( f = File.objects.get(uuid=fileUUID) # UPDATE THE CURRENT FILE PATH - f.currentlocation = dst + f.currentlocation = dst.encode() f.save() if not createEvent: @@ -314,7 +314,7 @@ def getFileUUIDLike( "currentlocation__contains": srcDB, unitIdentifierType: unitIdentifier, } - return {f.currentlocation: f.uuid for f in File.objects.filter(**kwargs)} + return {f.currentlocation.decode(): f.uuid for f in File.objects.filter(**kwargs)} def updateFileGrpUsefileGrpUUID(fileUUID, fileGrpUse, fileGrpUUID): @@ -371,9 +371,11 @@ def findFileInNormalizationCSV( file=sys.stderr, ) raise FindFileInNormalizatonCSVError(2) - target_file = f.originallocation.replace( - "%transferDirectory%objects/", "", 1 - ).replace("%SIPDirectory%objects/", "", 1) + target_file = ( + f.originallocation.decode() + .replace("%transferDirectory%objects/", "", 1) + .replace("%SIPDirectory%objects/", "", 1) + ) try: for row in reader: if not row: diff --git a/src/archivematicaCommon/tests/fixtures/test_database_functions.json b/src/archivematicaCommon/tests/fixtures/test_database_functions.json index dcf58d1a82..67edebdae3 100644 --- a/src/archivematicaCommon/tests/fixtures/test_database_functions.json +++ b/src/archivematicaCommon/tests/fixtures/test_database_functions.json @@ -4,7 +4,7 @@ "model": "main.file", "fields": { "checksum": "", - "currentlocation": "file_with_sip_agent", + "currentlocation": "ZmlsZV93aXRoX3NpcF9hZ2VudA==", "enteredsystem": "2014-10-30T00:00:00", "modificationtime": "2014-10-30T00:00:00", "filegrpuse": "original", @@ -47,7 +47,7 @@ "model": "main.file", "fields": { "checksum": "", - "currentlocation": "file_with_transfer_agent", + "currentlocation": "ZmlsZV93aXRoX3RyYW5zZmVyX2FnZW50", "enteredsystem": "2014-10-30T00:00:00", "modificationtime": "2014-10-30T00:00:00", "filegrpuse": "original", @@ -95,7 +95,7 @@ "model": "main.file", "fields": { "checksum": "", - "currentlocation": "file_with_sip_agent", + "currentlocation": "ZmlsZV93aXRoX3NpcF9hZ2VudA==", "enteredsystem": "2014-10-30T00:00:00", "modificationtime": "2014-10-30T00:00:00", "filegrpuse": "original", @@ -167,7 +167,7 @@ "model": "main.file", "fields": { "checksum": "", - "currentlocation": "file_with_no_unit_var", + "currentlocation": "ZmlsZV93aXRoX25vX3VuaXRfdmFy", "enteredsystem": "2014-10-30T00:00:00", "modificationtime": "2014-10-30T00:00:00", "filegrpuse": "original", diff --git a/src/archivematicaCommon/tests/test_database_functions.py b/src/archivematicaCommon/tests/test_database_functions.py index 720769a69c..2d6e246345 100644 --- a/src/archivematicaCommon/tests/test_database_functions.py +++ b/src/archivematicaCommon/tests/test_database_functions.py @@ -11,21 +11,20 @@ class TestDatabaseFunctions(TestCase): - fixture_files = ["agents.json", "test_database_functions.json"] fixtures = [os.path.join(THIS_DIR, "fixtures", p) for p in fixture_files] # insertIntoFiles def test_insert_into_files_with_sip(self): path = "%sharedDirectory%/" - assert File.objects.filter(currentlocation=path).count() == 0 + assert File.objects.filter(currentlocation=path.encode()).count() == 0 databaseFunctions.insertIntoFiles( "690c2fb5-7fee-4c29-a8b2-e3758ab9871e", path, sipUUID="0049fa6c-152f-44a0-93b0-c5e856a02292", ) - assert File.objects.filter(currentlocation=path).count() == 1 + assert File.objects.filter(currentlocation=path.encode()).count() == 1 def test_insert_into_files_raises_if_no_sip_or_transfer_provided(self): with pytest.raises(Exception) as excinfo: @@ -63,9 +62,14 @@ def test_insert_into_files_with_original_path(self): sipUUID="0049fa6c-152f-44a0-93b0-c5e856a02292", originalLocation=original_location, ) - assert File.objects.filter(originallocation=original_location).count() == 1 assert ( - File.objects.get(originallocation=original_location).currentlocation + File.objects.filter(originallocation=original_location.encode()).count() + == 1 + ) + assert ( + File.objects.get( + originallocation=original_location.encode() + ).currentlocation.decode() != original_location ) @@ -80,7 +84,7 @@ def test_insert_into_files_with_original_path(self): assert ( File.objects.filter(uuid="554661f1-b331-452c-a583-0c582ebcb298")[ 0 - ].originallocation + ].originallocation.decode() == file_path ) diff --git a/src/archivematicaCommon/tests/test_dicts.py b/src/archivematicaCommon/tests/test_dicts.py index 97160c3ed8..1e2862dc72 100644 --- a/src/archivematicaCommon/tests/test_dicts.py +++ b/src/archivematicaCommon/tests/test_dicts.py @@ -6,27 +6,36 @@ from dicts import setup as setup_dicts from main import models -TRANSFER = models.Transfer( - uuid="fb0aa04d-8547-46fc-bb7f-288ea5827d2c", - currentlocation="%sharedDirectory%foo", - type="Standard", - accessionid="accession1", - hidden=True, -) - -SIP = models.SIP( - uuid="c58794fd-4fb8-42a0-b9be-e75191696ab8", - currentpath="%sharedDirectory%bar", - hidden=True, -) - -FILE = models.File( - uuid="ee61d09b-2790-4980-827a-135346657eec", - transfer=TRANSFER, - originallocation="%sharedDirectory%orig", - currentlocation="%sharedDirectory%new", - filegrpuse="original", -) + +@pytest.fixture +def TRANSFER(db): + return models.Transfer.objects.create( + uuid="fb0aa04d-8547-46fc-bb7f-288ea5827d2c", + currentlocation="%sharedDirectory%foo", + type="Standard", + accessionid="accession1", + hidden=True, + ) + + +@pytest.fixture +def SIP(): + return models.SIP( + uuid="c58794fd-4fb8-42a0-b9be-e75191696ab8", + currentpath="%sharedDirectory%bar", + hidden=True, + ) + + +@pytest.fixture +def FILE(db, TRANSFER): + return models.File.objects.create( + uuid="ee61d09b-2790-4980-827a-135346657eec", + transfer=TRANSFER, + originallocation=b"%sharedDirectory%orig", + currentlocation=b"%sharedDirectory%new", + filegrpuse="original", + ) @pytest.fixture(scope="module", autouse=True) @@ -59,7 +68,7 @@ def test_replacementdict_replace(): assert d.replace("%PREFIX%/bin/") == ["/usr/local/bin/"] -def test_replacementdict_model_constructor_transfer(): +def test_replacementdict_model_constructor_transfer(TRANSFER, FILE): rd = ReplacementDict.frommodel(sip=TRANSFER, file_=FILE, type_="transfer") # Transfer-specific variables @@ -78,12 +87,12 @@ def test_replacementdict_model_constructor_transfer(): # File-specific variables assert rd["%fileUUID%"] == FILE.uuid - assert rd["%originalLocation%"] == FILE.originallocation - assert rd["%currentLocation%"] == FILE.currentlocation + assert rd["%originalLocation%"] == FILE.originallocation.decode() + assert rd["%currentLocation%"] == FILE.currentlocation.decode() assert rd["%fileGrpUse%"] == FILE.filegrpuse -def test_replacementdict_model_constructor_sip(): +def test_replacementdict_model_constructor_sip(SIP, FILE): rd = ReplacementDict.frommodel(sip=SIP, file_=FILE, type_="sip") # SIP-specific variables @@ -99,18 +108,18 @@ def test_replacementdict_model_constructor_sip(): # File-specific variables assert rd["%fileUUID%"] == FILE.uuid - assert rd["%originalLocation%"] == FILE.originallocation - assert rd["%currentLocation%"] == FILE.currentlocation + assert rd["%originalLocation%"] == FILE.originallocation.decode() + assert rd["%currentLocation%"] == FILE.currentlocation.decode() assert rd["%fileGrpUse%"] == FILE.filegrpuse -def test_replacementdict_model_constructor_file_only(): +def test_replacementdict_model_constructor_file_only(FILE): rd = ReplacementDict.frommodel(file_=FILE, type_="file") assert rd["%fileUUID%"] == FILE.uuid - assert rd["%originalLocation%"] == FILE.originallocation - assert rd["%currentLocation%"] == FILE.currentlocation - assert rd["%relativeLocation%"] == FILE.currentlocation + assert rd["%originalLocation%"] == FILE.originallocation.decode() + assert rd["%currentLocation%"] == FILE.currentlocation.decode() + assert rd["%relativeLocation%"] == FILE.currentlocation.decode() assert rd["%fileGrpUse%"] == FILE.filegrpuse diff --git a/src/dashboard/src/components/access/urls.py b/src/dashboard/src/components/access/urls.py index 9639d4fb89..3cb41408ff 100644 --- a/src/dashboard/src/components/access/urls.py +++ b/src/dashboard/src/components/access/urls.py @@ -1,39 +1,40 @@ from components.access import views -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "access" urlpatterns = [ - url(r"archivesspace/$", views.all_records), - url(r"archivesspace/levels/$", views.get_levels_of_description), - url(r"archivesspace/(?P[A-Za-z0-9-_]+)/$", views.record), - url( + path("archivesspace/", views.all_records), + path("archivesspace/levels/", views.get_levels_of_description), + re_path(r"archivesspace/(?P[A-Za-z0-9-_]+)/$", views.record), + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/contents/arrange/$", views.access_arrange_contents, name="access_arrange_contents", ), - url( + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/copy_to_arrange/$", views.access_copy_to_arrange, ), - url( + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/copy_from_arrange/$", views.access_arrange_start_sip, ), - url( + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/create_directory_within_arrange/$", views.access_create_directory, name="access_create_directory", ), - url( + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/rights/$", views.access_sip_rights ), - url( + re_path( r"archivesspace/(?P[A-Za-z0-9-_]+)/metadata/$", views.access_sip_metadata, ), - url(r"archivesspace/(?P.+)/children/$", views.record_children), - url( - r"archivesspace/(?P.+)/digital_object_components/$", + path("archivesspace//children/", views.record_children), + path( + "archivesspace//digital_object_components/", views.digital_object_components, ), ] diff --git a/src/dashboard/src/components/accounts/__init__.py b/src/dashboard/src/components/accounts/__init__.py index b9418f5a67..e69de29bb2 100644 --- a/src/dashboard/src/components/accounts/__init__.py +++ b/src/dashboard/src/components/accounts/__init__.py @@ -1 +0,0 @@ -default_app_config = "components.accounts.apps.AccountsConfig" diff --git a/src/dashboard/src/components/accounts/urls.py b/src/dashboard/src/components/accounts/urls.py index b6f9d5ee29..4926a22d54 100644 --- a/src/dashboard/src/components/accounts/urls.py +++ b/src/dashboard/src/components/accounts/urls.py @@ -17,35 +17,35 @@ import django.contrib.auth.views from components.accounts import views from django.conf import settings -from django.conf.urls import url +from django.urls import path app_name = "accounts" urlpatterns = [ - url(r"^$", views.list, name="accounts_index"), - url(r"add/$", views.add, name="add"), - url(r"(?P\d+)/delete/$", views.delete, name="delete"), - url(r"(?P\d+)/edit/$", views.edit, name="edit"), - url(r"profile/$", views.profile, name="profile"), - url(r"list/$", views.list), + path("", views.list, name="accounts_index"), + path("add/", views.add, name="add"), + path("/delete/", views.delete, name="delete"), + path("/edit/", views.edit, name="edit"), + path("profile/", views.profile, name="profile"), + path("list/", views.list), ] if "django_cas_ng" in settings.INSTALLED_APPS: import django_cas_ng.views urlpatterns += [ - url(r"login/$", django_cas_ng.views.LoginView.as_view(), name="login"), - url(r"logout/$", django_cas_ng.views.LogoutView.as_view(), name="logout"), + path("login/", django_cas_ng.views.LoginView.as_view(), name="login"), + path("logout/", django_cas_ng.views.LogoutView.as_view(), name="logout"), ] else: urlpatterns += [ - url( - r"login/$", + path( + "login/", django.contrib.auth.views.LoginView.as_view( template_name="accounts/login.html" ), name="login", ), - url(r"logout/$", django.contrib.auth.views.logout_then_login, name="logout"), + path("logout/", django.contrib.auth.views.logout_then_login, name="logout"), ] diff --git a/src/dashboard/src/components/accounts/validators.py b/src/dashboard/src/components/accounts/validators.py index d08eaa5925..1950ed2d9f 100644 --- a/src/dashboard/src/components/accounts/validators.py +++ b/src/dashboard/src/components/accounts/validators.py @@ -1,5 +1,5 @@ from django.core.exceptions import ValidationError -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ class PasswordComplexityValidator: diff --git a/src/dashboard/src/components/accounts/views.py b/src/dashboard/src/components/accounts/views.py index b8ccb1c6b0..b44f74fde6 100644 --- a/src/dashboard/src/components/accounts/views.py +++ b/src/dashboard/src/components/accounts/views.py @@ -29,7 +29,7 @@ from django.shortcuts import redirect from django.shortcuts import render from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main.models import UserProfile from tastypie.models import ApiKey diff --git a/src/dashboard/src/components/administration/forms.py b/src/dashboard/src/components/administration/forms.py index ec44e208cd..e28be8c02f 100644 --- a/src/dashboard/src/components/administration/forms.py +++ b/src/dashboard/src/components/administration/forms.py @@ -22,7 +22,7 @@ from django.conf import settings from django.forms.widgets import Select from django.forms.widgets import TextInput -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from installer.forms import load_site_url from installer.forms import site_url_field from lxml import etree diff --git a/src/dashboard/src/components/administration/forms_dip_upload.py b/src/dashboard/src/components/administration/forms_dip_upload.py index c1927136a4..394125f87c 100644 --- a/src/dashboard/src/components/administration/forms_dip_upload.py +++ b/src/dashboard/src/components/administration/forms_dip_upload.py @@ -1,5 +1,5 @@ from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ PREMIS_CHOICES = [("yes", _("Yes")), ("no", _("No")), ("premis", _("Base on PREMIS"))] diff --git a/src/dashboard/src/components/administration/urls.py b/src/dashboard/src/components/administration/urls.py index d75ce9cfac..b88d334af3 100644 --- a/src/dashboard/src/components/administration/urls.py +++ b/src/dashboard/src/components/administration/urls.py @@ -19,81 +19,82 @@ from components.administration import views_processing from components.administration.forms import ProcessingConfigurationForm from django.conf import settings -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path +from django.urls import re_path from django.views.generic import TemplateView app_name = "administration" urlpatterns = [ - url(r"^$", views.administration, name="administration_index"), - url( + path("", views.administration, name="administration_index"), + re_path( r"^reports/failures/delete/(?P\w+)/$", views.failure_report_delete, name="failure_report_delete", ), - url( + re_path( r"^reports/failures/(?P\w+)/$", views.failure_report, name="failure_report", ), - url(r"^reports/failures/$", views.failure_report, name="reports_failures_index"), - url(r"^dips/as/$", views_dip_upload.admin_as, name="dips_as"), - url(r"^dips/atom/$", views_dip_upload.admin_atom, name="dips_atom_index"), - url( - r"^dips/atom/edit_levels/$", + path("reports/failures/", views.failure_report, name="reports_failures_index"), + path("dips/as/", views_dip_upload.admin_as, name="dips_as"), + path("dips/atom/", views_dip_upload.admin_atom, name="dips_atom_index"), + path( + "dips/atom/edit_levels/", views.atom_levels_of_description, name="atom_levels_of_description", ), - url(r"^i18n/", include("django.conf.urls.i18n", namespace="django_i18n")), - url( - r"^language/$", + path("i18n/", include(("django.conf.urls.i18n", "i18n"), namespace="i18n")), + path( + "language/", TemplateView.as_view(template_name="administration/language.html"), name="admin_set_language", ), - url(r"^storage/$", views.storage, name="storage"), - url(r"^usage/$", views.usage, name="usage"), - url(r"^usage/clear/(?P\w+)/$", views.usage_clear, name="usage_clear"), - url(r"^processing/$", views_processing.list, name="processing"), - url(r"^processing/add/$", views_processing.edit, name="processing_add"), - url( + path("storage/", views.storage, name="storage"), + path("usage/", views.usage, name="usage"), + re_path(r"^usage/clear/(?P\w+)/$", views.usage_clear, name="usage_clear"), + path("processing/", views_processing.list, name="processing"), + path("processing/add/", views_processing.edit, name="processing_add"), + re_path( rf"^processing/edit/{ProcessingConfigurationForm.NAME_URL_REGEX}/$", views_processing.edit, name="processing_edit", ), - url( + re_path( rf"^processing/reset/{ProcessingConfigurationForm.NAME_URL_REGEX}/$", views_processing.reset, name="processing_reset", ), - url( + re_path( rf"^processing/delete/{ProcessingConfigurationForm.NAME_URL_REGEX}/$", views_processing.delete, name="processing_delete", ), - url( + re_path( rf"^processing/download/{ProcessingConfigurationForm.NAME_URL_REGEX}/$", views_processing.download, name="processing_download", ), - url(r"^premis/agent/$", views.premis_agent, name="premis_agent"), - url(r"^handle/$", views.handle_config, name="handle"), - url(r"^api/$", views.api, name="api"), - url(r"^general/$", views.general, name="general"), - url(r"^version/$", views.version, name="version"), - url( + path("premis/agent/", views.premis_agent, name="premis_agent"), + path("handle/", views.handle_config, name="handle"), + path("api/", views.api, name="api"), + path("general/", views.general, name="general"), + path("version/", views.version, name="version"), + re_path( r"^taxonomy/term/(?P" + settings.UUID_REGEX + ")/$", views.term_detail, name="term", ), - url( + re_path( r"^taxonomy/term/(?P" + settings.UUID_REGEX + ")/delete/$", views.term_delete, ), - url( + re_path( r"^taxonomy/(?P" + settings.UUID_REGEX + ")/$", views.terms, name="terms", ), - url(r"^taxonomy/$", views.taxonomy, name="taxonomy"), + path("taxonomy/", views.taxonomy, name="taxonomy"), ] diff --git a/src/dashboard/src/components/administration/views.py b/src/dashboard/src/components/administration/views.py index 6d0e4edab6..ad80c42bfd 100644 --- a/src/dashboard/src/components/administration/views.py +++ b/src/dashboard/src/components/administration/views.py @@ -42,7 +42,7 @@ from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from installer.steps import setup_pipeline_in_ss from main import models from version import get_full_version diff --git a/src/dashboard/src/components/administration/views_dip_upload.py b/src/dashboard/src/components/administration/views_dip_upload.py index ca22626dab..a750151f3a 100644 --- a/src/dashboard/src/components/administration/views_dip_upload.py +++ b/src/dashboard/src/components/administration/views_dip_upload.py @@ -2,7 +2,7 @@ from components.administration.forms_dip_upload import AtomConfigForm from django.contrib import messages from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main.models import DashboardSetting diff --git a/src/dashboard/src/components/administration/views_processing.py b/src/dashboard/src/components/administration/views_processing.py index 8aff39947b..1c588bc336 100644 --- a/src/dashboard/src/components/administration/views_processing.py +++ b/src/dashboard/src/components/administration/views_processing.py @@ -24,7 +24,7 @@ from django.http import Http404 from django.shortcuts import redirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from processing import install_builtin_config from .forms import ProcessingConfigurationForm diff --git a/src/dashboard/src/components/api/urls.py b/src/dashboard/src/components/api/urls.py index 98e79bb503..29babdd004 100644 --- a/src/dashboard/src/components/api/urls.py +++ b/src/dashboard/src/components/api/urls.py @@ -16,67 +16,74 @@ # along with Archivematica. If not, see . from components.api import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "api" urlpatterns = [ - url(r"transfer/approve", views.approve_transfer), - url(r"transfer/unapproved", views.unapproved_transfers), - url(r"transfer/completed", views.completed_transfers, name="completed_transfers"), - url( + re_path(r"transfer/approve", views.approve_transfer), + re_path(r"transfer/unapproved", views.unapproved_transfers), + re_path( + r"transfer/completed", views.completed_transfers, name="completed_transfers" + ), + re_path( r"transfer/status/(?P" + settings.UUID_REGEX + ")", views.status, {"unit_type": "unitTransfer"}, name="transfer_status", ), - url(r"transfer/start_transfer/", views.start_transfer_api, name="start_transfer"), - url(r"transfer/reingest", views.reingest, {"target": "transfer"}), - url( + re_path( + r"transfer/start_transfer/", views.start_transfer_api, name="start_transfer" + ), + re_path(r"transfer/reingest", views.reingest, {"target": "transfer"}), + re_path( r"ingest/status/(?P" + settings.UUID_REGEX + ")", views.status, {"unit_type": "unitSIP"}, ), - url(r"ingest/waiting", views.waiting_for_user_input), - url( + re_path(r"ingest/waiting", views.waiting_for_user_input), + re_path( r"^(?Ptransfer|ingest)/(?P" + settings.UUID_REGEX + ")/delete/", views.mark_hidden, ), - url(r"^(?Ptransfer|ingest)/delete/", views.mark_completed_hidden), - url(r"^ingest/reingest/approve", views.reingest_approve, name="reingest_approve"), - url(r"^ingest/reingest", views.reingest, {"target": "ingest"}), - url(r"^ingest/completed", views.completed_ingests, name="completed_ingests"), - url(r"^ingest/copy_metadata_files/$", views.copy_metadata_files_api), - url( - r"administration/dips/atom/levels/$", + re_path(r"^(?Ptransfer|ingest)/delete/", views.mark_completed_hidden), + re_path( + r"^ingest/reingest/approve", views.reingest_approve, name="reingest_approve" + ), + re_path(r"^ingest/reingest", views.reingest, {"target": "ingest"}), + re_path(r"^ingest/completed", views.completed_ingests, name="completed_ingests"), + path("ingest/copy_metadata_files/", views.copy_metadata_files_api), + path( + "administration/dips/atom/levels/", views.get_levels_of_description, name="get_levels_of_description", ), - url( - r"administration/dips/atom/fetch_levels/$", + path( + "administration/dips/atom/fetch_levels/", views.fetch_levels_of_description_from_atom, name="fetch_atom_lods", ), - url(r"filesystem/metadata/$", views.path_metadata), - url( + path("filesystem/metadata/", views.path_metadata), + re_path( r"processing-configuration/(?P\w{1,16})", views.processing_configuration, name="processing_configuration", ), - url( + re_path( r"processing-configuration", views.processing_configurations, name="processing_configuration_list", ), - url(r"v2beta/package", views.package), - url(r"v2beta/validate/([-\w]+)", views.validate, name="v2beta_validate"), - url( + re_path(r"v2beta/package", views.package), + re_path(r"v2beta/validate/([-\w]+)", views.validate, name="v2beta_validate"), + re_path( r"v2beta/jobs/(?P" + settings.UUID_REGEX + ")", views.unit_jobs, name="v2beta_jobs", ), - url( + re_path( r"v2beta/task/(?P" + settings.UUID_REGEX + ")", views.task, name="v2beta_task", diff --git a/src/dashboard/src/components/api/views.py b/src/dashboard/src/components/api/views.py index c38f6e9f25..4f388244e5 100644 --- a/src/dashboard/src/components/api/views.py +++ b/src/dashboard/src/components/api/views.py @@ -769,7 +769,7 @@ def processing_configuration(request, name): {"success": False, "error": msg}, status_code=404 ) else: - accepted_types = request.META.get("HTTP_ACCEPT", "").lower() + accepted_types = request.headers.get("accept", "").lower() if accepted_types != "*/*" and "xml" not in accepted_types: return django.http.HttpResponse(status=415) @@ -863,7 +863,7 @@ def validate(request, validator_name): # We could leverage Content-Type so a validator knows the type of document # and encoding that it's dealing with. For now, we're just enforcing that # "text/csv; charset=utf-8" is used, which is used by Avalon's validator. - mime_type, props = parse_header(request.META.get("CONTENT_TYPE", "")) + mime_type, props = parse_header(request.headers.get("content-type", "")) if mime_type != "text/csv" or props.get("charset") != "utf-8": return _error_response('Content type should be "text/csv; charset=utf-8"') diff --git a/src/dashboard/src/components/appraisal/urls.py b/src/dashboard/src/components/appraisal/urls.py index b95a04863d..f9aea1b859 100644 --- a/src/dashboard/src/components/appraisal/urls.py +++ b/src/dashboard/src/components/appraisal/urls.py @@ -1,5 +1,5 @@ from components.appraisal import views -from django.conf.urls import url +from django.urls import path app_name = "appraisal" -urlpatterns = [url(r"^$", views.appraisal, name="appraisal_index")] +urlpatterns = [path("", views.appraisal, name="appraisal_index")] diff --git a/src/dashboard/src/components/archival_storage/forms.py b/src/dashboard/src/components/archival_storage/forms.py index 31788ff868..abb9c69f2e 100644 --- a/src/dashboard/src/components/archival_storage/forms.py +++ b/src/dashboard/src/components/archival_storage/forms.py @@ -17,7 +17,7 @@ from agentarchives.atom.client import CommunicationError from components.archival_storage.atom import get_atom_client from django import forms -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from requests import ConnectionError from requests import Timeout diff --git a/src/dashboard/src/components/archival_storage/urls.py b/src/dashboard/src/components/archival_storage/urls.py index b594e5586c..b2bb936b41 100644 --- a/src/dashboard/src/components/archival_storage/urls.py +++ b/src/dashboard/src/components/archival_storage/urls.py @@ -16,39 +16,42 @@ # along with Archivematica. If not, see . from components.archival_storage import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "archival_storage" urlpatterns = [ - url(r"^(?P" + settings.UUID_REGEX + ")/$", views.view_aip, name="view_aip"), - url( + re_path( + r"^(?P" + settings.UUID_REGEX + ")/$", views.view_aip, name="view_aip" + ), + re_path( r"^download/aip/file/(?P" + settings.UUID_REGEX + ")/$", views.aip_file_download, name="aip_file_download", ), - url( + re_path( r"^download/aip/(?P" + settings.UUID_REGEX + ")/mets_download/$", views.aip_mets_file_download, name="aip_mets_file_download", ), - url( + re_path( r"^download/aip/(?P" + settings.UUID_REGEX + ")/pointer_file/$", views.aip_pointer_file_download, name="aip_pointer_file_download", ), - url( + re_path( r"^download/aip/(?P" + settings.UUID_REGEX + ")/$", views.aip_download, name="aip_download", ), - url(r"^search/json/file/(?P\w+)/$", views.file_json), - url(r"^search/create_aic/$", views.create_aic, name="create_aic"), - url(r"^search/$", views.search, name="archival_storage_search"), - url( + re_path(r"^search/json/file/(?P\w+)/$", views.file_json), + path("search/create_aic/", views.create_aic, name="create_aic"), + path("search/", views.search, name="archival_storage_search"), + re_path( r"^thumbnail/(?P" + settings.UUID_REGEX + ")/$", views.send_thumbnail ), - url(r"^save_state/(?P[-\w]+)/$", views.save_state, name="save_state"), - url(r"^load_state/(?P
[-\w]+)/$", views.load_state, name="load_state"), - url(r"^$", views.execute, name="archival_storage_index"), + re_path(r"^save_state/(?P
[-\w]+)/$", views.save_state, name="save_state"), + re_path(r"^load_state/(?P
[-\w]+)/$", views.load_state, name="load_state"), + path("", views.execute, name="archival_storage_index"), ] diff --git a/src/dashboard/src/components/archival_storage/views.py b/src/dashboard/src/components/archival_storage/views.py index 97cc667d29..63c67f598b 100644 --- a/src/dashboard/src/components/archival_storage/views.py +++ b/src/dashboard/src/components/archival_storage/views.py @@ -42,7 +42,7 @@ from django.template.defaultfilters import filesizeformat from django.utils.timezone import get_current_timezone from django.utils.timezone import make_aware -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from elasticsearch import ElasticsearchException logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/backlog/urls.py b/src/dashboard/src/components/backlog/urls.py index 82219420c6..a5ef37e8e7 100644 --- a/src/dashboard/src/components/backlog/urls.py +++ b/src/dashboard/src/components/backlog/urls.py @@ -16,22 +16,23 @@ # along with Archivematica. If not, see . from components.backlog import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "backlog" urlpatterns = [ - url(r"^$", views.execute, name="backlog_index"), - url(r"search/$", views.search, name="backlog_search"), - url( + path("", views.execute, name="backlog_index"), + path("search/", views.search, name="backlog_search"), + re_path( r"delete/(?P" + settings.UUID_REGEX + ")/$", views.delete, name="backlog_delete", ), - url( + re_path( r"download/(?P" + settings.UUID_REGEX + ")/$", views.download, name="backlog_download", ), - url(r"save_state/(?P
[-\w]+)/$", views.save_state, name="save_state"), - url(r"load_state/(?P
[-\w]+)/$", views.load_state, name="load_state"), + re_path(r"save_state/(?P
[-\w]+)/$", views.save_state, name="save_state"), + re_path(r"load_state/(?P
[-\w]+)/$", views.load_state, name="load_state"), ] diff --git a/src/dashboard/src/components/backlog/views.py b/src/dashboard/src/components/backlog/views.py index 82aa379900..5d6c4bb8c3 100644 --- a/src/dashboard/src/components/backlog/views.py +++ b/src/dashboard/src/components/backlog/views.py @@ -33,7 +33,7 @@ from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/file/urls.py b/src/dashboard/src/components/file/urls.py index ec37eb9046..f93d9a2632 100644 --- a/src/dashboard/src/components/file/urls.py +++ b/src/dashboard/src/components/file/urls.py @@ -1,15 +1,15 @@ from components.file import views from django.conf import settings -from django.conf.urls import url +from django.urls import re_path app_name = "file" urlpatterns = [ - url(r"(?P" + settings.UUID_REGEX + ")/$", views.file_details), - url( + re_path(r"(?P" + settings.UUID_REGEX + ")/$", views.file_details), + re_path( r"(?P" + settings.UUID_REGEX + ")/tags/$", views.TransferFileTags.as_view(), ), - url( + re_path( r"(?P" + settings.UUID_REGEX + ")/bulk_extractor/$", views.bulk_extractor, ), diff --git a/src/dashboard/src/components/filesystem_ajax/urls.py b/src/dashboard/src/components/filesystem_ajax/urls.py index de6f4eddcb..5ca1ae0583 100644 --- a/src/dashboard/src/components/filesystem_ajax/urls.py +++ b/src/dashboard/src/components/filesystem_ajax/urls.py @@ -16,33 +16,34 @@ # along with Archivematica. If not, see . from components.filesystem_ajax import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "filesystem_ajax" urlpatterns = [ - url(r"^download_ss/$", views.download_ss), - url(r"^download_fs/$", views.download_fs), - url(r"^(?P" + settings.UUID_REGEX + ")/download/$", views.download_by_uuid), - url(r"^(?P" + settings.UUID_REGEX + ")/preview/$", views.preview_by_uuid), - url(r"^contents/arrange/$", views.arrange_contents, name="contents_arrange"), - url(r"^contents/$", views.contents, name="contents"), - url( + path("download_ss/", views.download_ss), + path("download_fs/", views.download_fs), + re_path( + r"^(?P" + settings.UUID_REGEX + ")/download/$", views.download_by_uuid + ), + re_path(r"^(?P" + settings.UUID_REGEX + ")/preview/$", views.preview_by_uuid), + path("contents/arrange/", views.arrange_contents, name="contents_arrange"), + path("contents/", views.contents, name="contents"), + re_path( r"^children/location/(?P" + settings.UUID_REGEX + ")/$", views.directory_children_proxy_to_storage_server, ), - url(r"^delete/arrange/$", views.delete_arrange, name="delete_arrange"), - url( - r"^create_directory_within_arrange/$", + path("delete/arrange/", views.delete_arrange, name="delete_arrange"), + path( + "create_directory_within_arrange/", views.create_directory_within_arrange, name="create_directory_within_arrange", ), - url(r"^copy_to_arrange/$", views.copy_to_arrange, name="copy_to_arrange"), - url( - r"^copy_from_arrange/$", + path("copy_to_arrange/", views.copy_to_arrange, name="copy_to_arrange"), + path( + "copy_from_arrange/", views.copy_from_arrange_to_completed, name="copy_from_arrange", ), - url( - r"^copy_metadata_files/$", views.copy_metadata_files, name="copy_metadata_files" - ), + path("copy_metadata_files/", views.copy_metadata_files, name="copy_metadata_files"), ] diff --git a/src/dashboard/src/components/filesystem_ajax/views.py b/src/dashboard/src/components/filesystem_ajax/views.py index 22699bf012..073b91ce33 100644 --- a/src/dashboard/src/components/filesystem_ajax/views.py +++ b/src/dashboard/src/components/filesystem_ajax/views.py @@ -33,8 +33,8 @@ from archivematicaFunctions import b64encode_string from components import helpers from django.conf import settings as django_settings -from django.utils.translation import ugettext as _ -from django.utils.translation import ungettext +from django.utils.translation import gettext as _ +from django.utils.translation import ngettext from main import models @@ -89,7 +89,7 @@ def _prepare_browse_response(response): prop["display_string"] = prop["verbose name"].strip() elif "object count" in prop: try: - prop["display_string"] = ungettext( + prop["display_string"] = ngettext( "%(count)d object", "%(count)d objects", prop["object count"] ) % {"count": prop["object count"]} except TypeError: # 'object_count' val can be a string, see SS:space.py @@ -154,12 +154,11 @@ def arrange_contents(request, path=None): # Query SIP Arrangement for results # Get all the paths that are not in SIPs and start with base_path. We don't # need the objects, just the arrange_path - paths = ( - models.SIPArrange.objects.filter(sip_created=False) - .filter(aip_created=False) - .filter(arrange_path__startswith=base_path) - .order_by("arrange_path") - ) + paths = models.SIPArrange.objects.filter( + sip_created=False, + aip_created=False, + arrange_path__startswith=base_path, + ).order_by("arrange_path") if len(paths) == 0 and base_path != DEFAULT_ARRANGE_PATH: response = { @@ -176,7 +175,7 @@ def arrange_contents(request, path=None): properties = {} for item in paths: # Strip common prefix - path_parts = item.arrange_path.replace(base_path, "", 1).split("/") + path_parts = item.arrange_path.decode().replace(base_path, "", 1).split("/") entry = path_parts[0] if not entry: continue @@ -411,7 +410,7 @@ def _create_arranged_sip(staging_sip_path, files, sip_uuid): in_sip_path = "/".join(file_["destination"].split("/")[2:]) currentlocation = "%SIPDirectory%" + in_sip_path models.File.objects.filter(uuid=file_["uuid"]).update( - sip=sip_uuid, currentlocation=currentlocation + sip=sip_uuid, currentlocation=currentlocation.encode() ) # Get all ancestor directory paths of the file's destination. subdir = os.path.dirname(currentlocation) @@ -523,28 +522,30 @@ def copy_from_arrange_to_completed_common(filepath, sip_uuid, sip_name): "copy_from_arrange_to_completed: staging_sip_path: %s", staging_sip_path ) # Fetch all files with 'filepath' as prefix, and have a source path - arrange = models.SIPArrange.objects.filter(sip_created=False).filter( - arrange_path__startswith=filepath + arrange = models.SIPArrange.objects.filter( + sip_created=False, arrange_path__startswith=filepath ) arrange_files = arrange.filter(original_path__isnull=False) # Collect file and directory information. Change path to be in staging, not arrange files = [] for arranged_file in arrange_files: - destination = arranged_file.arrange_path.replace( + destination = arranged_file.arrange_path.decode().replace( filepath, staging_sip_path, 1 ) files.append( { - "source": arranged_file.original_path.lstrip("/"), + "source": arranged_file.original_path.decode().lstrip("/"), "destination": destination, "uuid": str(arranged_file.file_uuid), } ) # Get transfer folder name - transfer_parts = arranged_file.original_path.replace( - DEFAULT_BACKLOG_PATH, "", 1 - ).split("/", 1) + transfer_parts = ( + arranged_file.original_path.decode() + .replace(DEFAULT_BACKLOG_PATH, "", 1) + .split("/", 1) + ) transfer_name = transfer_parts[0] # Determine if the transfer is a BagIt package is_bagit = transfer_parts[1].startswith("data/") @@ -588,7 +589,9 @@ def copy_from_arrange_to_completed_common(filepath, sip_uuid, sip_name): for arranged_entry in arrange: # Update arrange_path to be relative to new SIP's objects # Use normpath to strip trailing / from directories - relative_path = arranged_entry.arrange_path.replace(filepath, "", 1) + relative_path = arranged_entry.arrange_path.decode().replace( + filepath, "", 1 + ) if relative_path == "objects/": # If objects directory created manually, delete it as we # don't want the LoD for it, and it's not needed elsewhere @@ -596,7 +599,7 @@ def copy_from_arrange_to_completed_common(filepath, sip_uuid, sip_name): continue relative_path = relative_path.replace("objects/", "", 1) relative_path = os.path.normpath(relative_path) - arranged_entry.arrange_path = relative_path + arranged_entry.arrange_path = relative_path.encode() arranged_entry.sip_id = sip_uuid arranged_entry.sip_created = True arranged_entry.save() @@ -618,7 +621,7 @@ def create_arrange_directories(paths): arranges = [ models.SIPArrange( original_path=None, - arrange_path=os.path.join(path, ""), # ensure ends with / + arrange_path=os.path.join(path, "").encode(), # ensure ends with / file_uuid=None, ) for path in paths @@ -670,13 +673,15 @@ def _move_files_within_arrange(sourcepath, destination): # we retain the folder name when we move the files. source_parent = "/".join(sourcepath.split("/")[:-2]) + "/" for entry in folder_contents: - entry.arrange_path = entry.arrange_path.replace( - source_parent, destination, 1 + entry.arrange_path = ( + entry.arrange_path.decode() + .replace(source_parent, destination, 1) + .encode() ) entry.save() else: # source is a file models.SIPArrange.objects.filter(arrange_path=sourcepath).update( - arrange_path=destination + os.path.basename(sourcepath) + arrange_path=(destination + os.path.basename(sourcepath)).encode() ) else: # destination is a file (this should have been caught by JS) raise ValueError(_("You cannot drag and drop onto a file.")) diff --git a/src/dashboard/src/components/helpers.py b/src/dashboard/src/components/helpers.py index 9b484115cd..09b64af811 100644 --- a/src/dashboard/src/components/helpers.py +++ b/src/dashboard/src/components/helpers.py @@ -37,7 +37,7 @@ from django.http import StreamingHttpResponse from django.urls import reverse from django.utils.dateformat import format -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main import models from tastypie.models import ApiKey diff --git a/src/dashboard/src/components/ingest/pair_matcher.py b/src/dashboard/src/components/ingest/pair_matcher.py index 7122eca5c6..72f6cf0007 100644 --- a/src/dashboard/src/components/ingest/pair_matcher.py +++ b/src/dashboard/src/components/ingest/pair_matcher.py @@ -240,7 +240,7 @@ def remove_review_matches_prefixes(path): def review_matches(client, request, template, uuid, matches=[]): object_paths = { - file_.uuid: remove_review_matches_prefixes(file_.currentlocation) + file_.uuid: remove_review_matches_prefixes(file_.currentlocation.decode()) for file_ in models.File.objects.filter(sip=uuid) } for match in matches: @@ -279,7 +279,7 @@ def ingest_upload_atk_get_dip_object_paths(uuid): for item2 in item.findall("{http://www.loc.gov/METS/}FLocat"): object_path = item2.attrib["{http://www.w3.org/1999/xlink}href"] file = models.File.objects.get( - sip=uuid, currentlocation="%SIPDirectory%" + object_path + sip=uuid, currentlocation=("%SIPDirectory%" + object_path).encode() ) object_path = remove_objects_prefix(object_path) paths.append(object_path) diff --git a/src/dashboard/src/components/ingest/urls.py b/src/dashboard/src/components/ingest/urls.py index 766199cb56..77d8c8f864 100644 --- a/src/dashboard/src/components/ingest/urls.py +++ b/src/dashboard/src/components/ingest/urls.py @@ -17,129 +17,130 @@ from components.ingest import views from components.ingest import views_as from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "ingest" urlpatterns = [ - url(r"^$", views.ingest_grid, name="ingest_index"), - url(r"^sips/$", views.SipsView.as_view()), - url( + path("", views.ingest_grid, name="ingest_index"), + path("sips/", views.SipsView.as_view()), + re_path( r"^aic/(?P" + settings.UUID_REGEX + ")/metadata/add/$", views.aic_metadata_add, name="aic_metadata_add", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/$", views.ingest_metadata_list, name="ingest_metadata_list", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/add/$", views.ingest_metadata_edit, name="ingest_metadata_add", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + r")/metadata/(?P\d+)/$", views.ingest_metadata_edit, name="ingest_metadata_edit", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + r")/metadata/delete/(?P\d+)/$", views.ingest_metadata_delete, name="ingest_metadata_delete", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/event_detail/$", views.ingest_metadata_event_detail, name="ingest_metadata_event_detail", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/add_files/$", views.ingest_metadata_add_files, name="ingest_metadata_add_files", ), - url(r"^upload/url/check/$", views.ingest_upload_destination_url_check), - url(r"^(?P" + settings.UUID_REGEX + ")/upload/$", views.ingest_upload), - url(r"^status/$", views.ingest_status), - url(r"^status/(?P" + settings.UUID_REGEX + ")/$", views.ingest_status), - url( + path("upload/url/check/", views.ingest_upload_destination_url_check), + re_path(r"^(?P" + settings.UUID_REGEX + ")/upload/$", views.ingest_upload), + path("status/", views.ingest_status), + re_path(r"^status/(?P" + settings.UUID_REGEX + ")/$", views.ingest_status), + re_path( r"^normalization-report/(?P" + settings.UUID_REGEX + r")/(?P\d+)/$", views.ingest_normalization_report, name="ingest_normalization_report_page", ), - url( + re_path( r"^normalization-report/(?P" + settings.UUID_REGEX + ")/$", views.ingest_normalization_report, name="ingest_normalization_report", ), - url( + re_path( r"^preview/(?P[\w-]+)/(?P" + settings.UUID_REGEX + ")/$", views.ingest_browse, ), - url( + re_path( r"^backlog/file/download/(?P" + settings.UUID_REGEX + ")/", views.transfer_file_download, ), - url(r"^backlog/$", views.transfer_backlog, {"ui": "legacy"}), - url(r"^appraisal_list/$", views.transfer_backlog, {"ui": "appraisal"}), + path("backlog/", views.transfer_backlog, {"ui": "legacy"}), + path("appraisal_list/", views.transfer_backlog, {"ui": "appraisal"}), ] # ArchivesSpace urlpatterns += [ - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/match/resource/(?P.+)/$", views_as.ingest_upload_as_match_dip_objects_to_resource_levels, name="ingest_upload_as_match_dip_objects_to_resource_levels", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/match/resourcecomponent/(?P.+)/$", views_as.ingest_upload_as_match_dip_objects_to_resource_component_levels, name="ingest_upload_as_match_dip_objects_to_resource_component_levels", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/resource/(?P.+)/$", views_as.ingest_upload_as_resource, name="ingest_upload_as_resource", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/resourcecomponent/(?P.+)/$", views_as.ingest_upload_as_resource_component, name="ingest_upload_as_resource_component", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/save/$", views_as.ingest_upload_as_save, ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/match/$", views_as.ingest_upload_as_match, ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/reset/$", views_as.ingest_upload_as_reset, name="ingest_upload_as_reset", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/review/$", views_as.ingest_upload_as_review_matches, name="ingest_upload_as_review_matches", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/complete/$", views_as.complete_matching, name="complete_matching", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/upload/as/$", views_as.ingest_upload_as, name="ingest_upload_as", diff --git a/src/dashboard/src/components/ingest/views.py b/src/dashboard/src/components/ingest/views.py index 2edba58a56..a0c842c1d4 100644 --- a/src/dashboard/src/components/ingest/views.py +++ b/src/dashboard/src/components/ingest/views.py @@ -44,7 +44,7 @@ from django.shortcuts import render from django.urls import reverse from django.utils.text import slugify -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.generic import View from main import forms from main import models @@ -254,7 +254,7 @@ def ingest_metadata_event_detail(request, uuid): # Add path for original and derived files to each form for form in formset: - form.original_file = form.instance.file_uuid.originallocation.replace( + form.original_file = form.instance.file_uuid.originallocation.decode().replace( "%transferDirectory%objects/", "", 1 ) form.derived_file = ( @@ -262,7 +262,8 @@ def ingest_metadata_event_detail(request, uuid): derived_file__filegrpuse="preservation" ) .get() - .derived_file.originallocation.replace("%transferDirectory%objects/", "", 1) + .derived_file.originallocation.decode() + .replace("%transferDirectory%objects/", "", 1) ) # Get name of SIP from directory name of most recent job @@ -697,5 +698,7 @@ def transfer_file_download(request, uuid): path_to_transfer = transfer.currentlocation.replace( "%sharedPath%", shared_directory_path ) - path_to_file = file.currentlocation.replace("%transferDirectory%", path_to_transfer) + path_to_file = file.currentlocation.decode().replace( + "%transferDirectory%", path_to_transfer + ) return helpers.send_file(request, path_to_file) diff --git a/src/dashboard/src/components/ingest/views_as.py b/src/dashboard/src/components/ingest/views_as.py index 41acbac627..69998a3460 100644 --- a/src/dashboard/src/components/ingest/views_as.py +++ b/src/dashboard/src/components/ingest/views_as.py @@ -15,7 +15,7 @@ from django.http import HttpResponseRedirect from django.http import HttpResponseServerError from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main import models logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/mcp/urls.py b/src/dashboard/src/components/mcp/urls.py index 127e74187c..78dd73a717 100644 --- a/src/dashboard/src/components/mcp/urls.py +++ b/src/dashboard/src/components/mcp/urls.py @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . from components.mcp import views -from django.conf.urls import url +from django.urls import path app_name = "mcp" urlpatterns = [ - url(r"execute/$", views.execute, name="execute"), - url(r"list/$", views.list, name="list"), + path("execute/", views.execute, name="execute"), + path("list/", views.list, name="list"), ] diff --git a/src/dashboard/src/components/rights/forms.py b/src/dashboard/src/components/rights/forms.py index e229e0b1f5..b474b80be7 100644 --- a/src/dashboard/src/components/rights/forms.py +++ b/src/dashboard/src/components/rights/forms.py @@ -16,7 +16,7 @@ # along with Archivematica. If not, see . from django import forms from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main import models diff --git a/src/dashboard/src/components/rights/ingest_urls.py b/src/dashboard/src/components/rights/ingest_urls.py index e00bddc95c..d1715e96ed 100644 --- a/src/dashboard/src/components/rights/ingest_urls.py +++ b/src/dashboard/src/components/rights/ingest_urls.py @@ -15,18 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . from components.rights import views -from django.conf.urls import url +from django.urls import path app_name = "rights_ingest" urlpatterns = [ - url(r"^$", views.ingest_rights_list, name="index"), - url(r"^add/$", views.ingest_rights_edit, name="add"), - url(r"^delete/(?P\d+)/$", views.ingest_rights_delete), - url( - r"^grants/(?P\d+)/delete/$", + path("", views.ingest_rights_list, name="index"), + path("add/", views.ingest_rights_edit, name="add"), + path("delete//", views.ingest_rights_delete), + path( + "grants//delete/", views.ingest_rights_grant_delete, name="grant_delete", ), - url(r"^grants/(?P\d+)/$", views.ingest_rights_grants_edit, name="grants_edit"), - url(r"^(?P\d+)/$", views.ingest_rights_edit, name="edit"), + path("grants//", views.ingest_rights_grants_edit, name="grants_edit"), + path("/", views.ingest_rights_edit, name="edit"), ] diff --git a/src/dashboard/src/components/rights/transfer_urls.py b/src/dashboard/src/components/rights/transfer_urls.py index ef0814d994..dfed6f5c1e 100644 --- a/src/dashboard/src/components/rights/transfer_urls.py +++ b/src/dashboard/src/components/rights/transfer_urls.py @@ -15,20 +15,18 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . from components.rights import views -from django.conf.urls import url +from django.urls import path app_name = "rights_transfer" urlpatterns = [ - url(r"^$", views.transfer_rights_list, name="index"), - url(r"^add/$", views.transfer_rights_edit, name="add"), - url(r"^delete/(?P\d+)/$", views.transfer_rights_delete), - url( - r"^grants/(?P\d+)/delete/$", + path("", views.transfer_rights_list, name="index"), + path("add/", views.transfer_rights_edit, name="add"), + path("delete//", views.transfer_rights_delete), + path( + "grants//delete/", views.transfer_rights_grant_delete, name="grant_delete", ), - url( - r"^grants/(?P\d+)/$", views.transfer_rights_grants_edit, name="grants_edit" - ), - url(r"^(?P\d+)/$", views.transfer_rights_edit, name="edit"), + path("grants//", views.transfer_rights_grants_edit, name="grants_edit"), + path("/", views.transfer_rights_edit, name="edit"), ] diff --git a/src/dashboard/src/components/rights/views.py b/src/dashboard/src/components/rights/views.py index 9a2aa14c72..c07261069b 100644 --- a/src/dashboard/src/components/rights/views.py +++ b/src/dashboard/src/components/rights/views.py @@ -25,7 +25,7 @@ from django.shortcuts import redirect from django.shortcuts import render from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main import models LOGGER = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/transfer/urls.py b/src/dashboard/src/components/transfer/urls.py index 511ad10489..0417cb7c34 100644 --- a/src/dashboard/src/components/transfer/urls.py +++ b/src/dashboard/src/components/transfer/urls.py @@ -16,42 +16,43 @@ # along with Archivematica. If not, see . from components.transfer import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "transfer" urlpatterns = [ - url(r"^$", views.grid, name="transfer_index"), + path("", views.grid, name="transfer_index"), # Transfer metadata set functions - url(r"^create_metadata_set_uuid/$", views.create_metadata_set_uuid), - url( + path("create_metadata_set_uuid/", views.create_metadata_set_uuid), + re_path( r"^rename_metadata_set/(?P" + settings.UUID_REGEX + r")/(?P[\w\-]+)/$", views.rename_metadata_set, ), - url( + re_path( r"^cleanup_metadata_set/(?P" + settings.UUID_REGEX + ")/$", views.cleanup_metadata_set, ), - url(r"^locations/$", views.transfer_source_locations), - url( + path("locations/", views.transfer_source_locations), + re_path( r"^component/(?P" + settings.UUID_REGEX + ")/$", views.component, name="component", ), - url(r"^status/$", views.status), - url(r"^status/(?P" + settings.UUID_REGEX + ")/$", views.status), - url( + path("status/", views.status), + re_path(r"^status/(?P" + settings.UUID_REGEX + ")/$", views.status), + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/$", views.transfer_metadata_list, name="transfer_metadata_list", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/metadata/add/$", views.transfer_metadata_edit, name="transfer_metadata_add", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + r")/metadata/(?P\d+)/$", views.transfer_metadata_edit, name="transfer_metadata_edit", diff --git a/src/dashboard/src/components/transfer/views.py b/src/dashboard/src/components/transfer/views.py index f1ddb19fca..d5fb8cbd0f 100644 --- a/src/dashboard/src/components/transfer/views.py +++ b/src/dashboard/src/components/transfer/views.py @@ -27,7 +27,7 @@ from django.http import HttpResponse from django.shortcuts import redirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from main import models logger = logging.getLogger("archivematica.dashboard") diff --git a/src/dashboard/src/components/unit/urls.py b/src/dashboard/src/components/unit/urls.py index f7fdcc6530..3a88d62102 100644 --- a/src/dashboard/src/components/unit/urls.py +++ b/src/dashboard/src/components/unit/urls.py @@ -16,22 +16,25 @@ # along with Archivematica. If not, see . from components.unit import views from django.conf import settings -from django.conf.urls import url +from django.urls import path +from django.urls import re_path app_name = "unit" # The first segment of these urls is '^(?Ptransfer|ingest)/' # All views should expect a first parameter of unit_type, with a value of 'transfer' or 'ingest' urlpatterns = [ - url(r"^(?P" + settings.UUID_REGEX + ")/$", views.detail, name="detail"), - url( + re_path( + r"^(?P" + settings.UUID_REGEX + ")/$", views.detail, name="detail" + ), + re_path( r"^(?P" + settings.UUID_REGEX + ")/microservices/$", views.microservices, name="microservices", ), - url( + re_path( r"^(?P" + settings.UUID_REGEX + ")/delete/$", views.mark_hidden, name="mark_hidden", ), - url(r"^delete/$", views.mark_completed_hidden, name="mark_all_hidden"), + path("delete/", views.mark_completed_hidden, name="mark_all_hidden"), ] diff --git a/src/dashboard/src/fpr/forms.py b/src/dashboard/src/fpr/forms.py index 89add52512..09f31fc2d9 100644 --- a/src/dashboard/src/fpr/forms.py +++ b/src/dashboard/src/fpr/forms.py @@ -1,5 +1,5 @@ from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from fpr import models as fprmodels diff --git a/src/dashboard/src/fpr/migrations/0039_django32.py b/src/dashboard/src/fpr/migrations/0039_django32.py new file mode 100644 index 0000000000..a16e43acf6 --- /dev/null +++ b/src/dashboard/src/fpr/migrations/0039_django32.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.20 on 2023-08-18 00:48 +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("fpr", "0038_use_uuidfield"), + ] + + operations = [ + migrations.AlterField( + model_name="fpcommand", + name="tool", + field=models.ForeignKey( + limit_choices_to={"enabled": True}, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="fpr.fptool", + to_field="uuid", + verbose_name="the related tool", + ), + ), + ] diff --git a/src/dashboard/src/fpr/models.py b/src/dashboard/src/fpr/models.py index dd1e55ec94..1f02fc2e1a 100644 --- a/src/dashboard/src/fpr/models.py +++ b/src/dashboard/src/fpr/models.py @@ -12,7 +12,7 @@ from django.core.validators import ValidationError from django.db import connection from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from main.models import UUIDField diff --git a/src/dashboard/src/fpr/urls.py b/src/dashboard/src/fpr/urls.py index 24fdbbb622..a56494e961 100644 --- a/src/dashboard/src/fpr/urls.py +++ b/src/dashboard/src/fpr/urls.py @@ -1,12 +1,13 @@ -from django.conf.urls import url +from django.urls import path +from django.urls import re_path from fpr import views UUID_REGEX = r"[\w]{8}(-[\w]{4}){3}-[\w]{12}" app_name = "fpr" urlpatterns = [ - url(r"^$", views.home, name="fpr_index"), - url( + path("", views.home, name="fpr_index"), + re_path( r"^(?Pformat|formatgroup|idrule|idcommand|fprule|fpcommand)/(?P" + UUID_REGEX + ")/toggle_enabled/$", @@ -14,130 +15,132 @@ name="toggle_enabled", ), # Formats - url(r"^format/$", views.format_list, name="format_list"), - url(r"^format/create/$", views.format_edit, name="format_create"), - url(r"^format/(?P[-\w]+)/$", views.format_detail, name="format_detail"), - url(r"^format/(?P[-\w]+)/edit/$", views.format_edit, name="format_edit"), + path("format/", views.format_list, name="format_list"), + path("format/create/", views.format_edit, name="format_create"), + re_path(r"^format/(?P[-\w]+)/$", views.format_detail, name="format_detail"), + re_path(r"^format/(?P[-\w]+)/edit/$", views.format_edit, name="format_edit"), # Format Versions - url( + re_path( r"^format/(?P[-\w]+)/create/$", views.formatversion_edit, name="formatversion_create", ), - url( + re_path( r"^format/(?P[-\w]+)/(?P[-\w]+)/$", views.formatversion_detail, name="formatversion_detail", ), - url( + re_path( r"^format/(?P[-\w]+)/(?P[-\w]+)/edit/$", views.formatversion_edit, name="formatversion_edit", ), - url( + re_path( r"^format/(?P[-\w]+)/(?P[-\w]+)/delete/$", views.formatversion_delete, name="formatversion_delete", ), # Format groups - url(r"^formatgroup/$", views.formatgroup_list, name="formatgroup_list"), - url(r"^formatgroup/create/$", views.formatgroup_edit, name="formatgroup_create"), - url( + path("formatgroup/", views.formatgroup_list, name="formatgroup_list"), + path("formatgroup/create/", views.formatgroup_edit, name="formatgroup_create"), + re_path( r"^formatgroup/(?P[-\w]+)/$", views.formatgroup_edit, name="formatgroup_edit", ), - url( + re_path( r"^formatgroup/delete/(?P[-\w]+)/$", views.formatgroup_delete, name="formatgroup_delete", ), # ID Tools - url(r"^idtool/$", views.idtool_list, name="idtool_list"), - url(r"^idtool/create/$", views.idtool_edit, name="idtool_create"), - url(r"^idtool/(?P[-\w]+)/$", views.idtool_detail, name="idtool_detail"), - url(r"^idtool/(?P[-\w]+)/edit/$", views.idtool_edit, name="idtool_edit"), + path("idtool/", views.idtool_list, name="idtool_list"), + path("idtool/create/", views.idtool_edit, name="idtool_create"), + re_path(r"^idtool/(?P[-\w]+)/$", views.idtool_detail, name="idtool_detail"), + re_path(r"^idtool/(?P[-\w]+)/edit/$", views.idtool_edit, name="idtool_edit"), # ID Rules - url(r"^idrule/$", views.idrule_list, name="idrule_list"), - url(r"^idrule/create/$", views.idrule_edit, name="idrule_create"), - url( + path("idrule/", views.idrule_list, name="idrule_list"), + path("idrule/create/", views.idrule_edit, name="idrule_create"), + re_path( r"^idrule/(?P" + UUID_REGEX + ")/edit/$", views.idrule_edit, name="idrule_edit", ), - url( + re_path( r"^idrule/(?P" + UUID_REGEX + ")/$", views.idrule_detail, name="idrule_detail", ), - url( + re_path( r"^idrule/(?P" + UUID_REGEX + ")/delete/$", views.idrule_delete, name="idrule_delete", ), # ID Commands - url(r"^idcommand/$", views.idcommand_list, name="idcommand_list"), - url(r"^idcommand/create/$", views.idcommand_edit, name="idcommand_create"), - url( + path("idcommand/", views.idcommand_list, name="idcommand_list"), + path("idcommand/create/", views.idcommand_edit, name="idcommand_create"), + re_path( r"^idcommand/(?P" + UUID_REGEX + ")/$", views.idcommand_detail, name="idcommand_detail", ), - url( + re_path( r"^idcommand/(?P" + UUID_REGEX + ")/edit/$", views.idcommand_edit, name="idcommand_edit", ), - url( + re_path( r"^idcommand/(?P" + UUID_REGEX + ")/delete/$", views.idcommand_delete, name="idcommand_delete", ), # FP Rules - url( + re_path( r"^fprule/(?P" + UUID_REGEX + ")/$", views.fprule_detail, name="fprule_detail", ), - url(r"^fprule/create/$", views.fprule_edit, name="fprule_create"), - url(r"^fprule/$", views.fprule_list, name="fprule_list"), - url(r"^fprule/(?P[-\w]+)/$", views.fprule_list, name="fprule_list"), - url( + path("fprule/create/", views.fprule_edit, name="fprule_create"), + path("fprule/", views.fprule_list, name="fprule_list"), + re_path(r"^fprule/(?P[-\w]+)/$", views.fprule_list, name="fprule_list"), + re_path( r"^fprule/(?P" + UUID_REGEX + ")/edit/$", views.fprule_edit, name="fprule_edit", ), - url( + re_path( r"^fprule/(?P" + UUID_REGEX + ")/delete/$", views.fprule_delete, name="fprule_delete", ), # FP Tools - url(r"^fptool/$", views.fptool_list, name="fptool_list"), - url(r"^fptool/create/$", views.fptool_edit, name="fptool_create"), - url(r"^fptool/(?P[-\w]+)/$", views.fptool_detail, name="fptool_detail"), - url(r"^fptool/(?P[-\w]+)/edit/$", views.fptool_edit, name="fptool_edit"), + path("fptool/", views.fptool_list, name="fptool_list"), + path("fptool/create/", views.fptool_edit, name="fptool_create"), + re_path(r"^fptool/(?P[-\w]+)/$", views.fptool_detail, name="fptool_detail"), + re_path(r"^fptool/(?P[-\w]+)/edit/$", views.fptool_edit, name="fptool_edit"), # FP Commands - url( + re_path( r"^fpcommand/(?P" + UUID_REGEX + ")/$", views.fpcommand_detail, name="fpcommand_detail", ), - url(r"^fpcommand/create/$", views.fpcommand_edit, name="fpcommand_create"), - url(r"^fpcommand/(?P[-\w]+)/$", views.fpcommand_list, name="fpcommand_list"), - url(r"^fpcommand/$", views.fpcommand_list, name="fpcommand_list"), - url( + path("fpcommand/create/", views.fpcommand_edit, name="fpcommand_create"), + re_path( + r"^fpcommand/(?P[-\w]+)/$", views.fpcommand_list, name="fpcommand_list" + ), + path("fpcommand/", views.fpcommand_list, name="fpcommand_list"), + re_path( r"^fpcommand/(?P" + UUID_REGEX + ")/edit/$", views.fpcommand_edit, name="fpcommand_edit", ), - url( + re_path( r"^fpcommand/(?P" + UUID_REGEX + ")/delete/$", views.fpcommand_delete, name="fpcommand_delete", ), # Revisions - url( + re_path( r"^revisions/(?P[-\w]+)/(?P" + UUID_REGEX + ")/$", views.revision_list, name="revision_list", diff --git a/src/dashboard/src/fpr/utils.py b/src/dashboard/src/fpr/utils.py index d941d0f136..0c604f2457 100644 --- a/src/dashboard/src/fpr/utils.py +++ b/src/dashboard/src/fpr/utils.py @@ -1,6 +1,6 @@ from django.apps import apps from django.contrib import messages -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ # ########## DEPENDENCIES ############ diff --git a/src/dashboard/src/fpr/views.py b/src/dashboard/src/fpr/views.py index 368edb3df2..c06453e853 100644 --- a/src/dashboard/src/fpr/views.py +++ b/src/dashboard/src/fpr/views.py @@ -5,7 +5,7 @@ from django.shortcuts import redirect from django.shortcuts import render from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from fpr import forms as fprforms from fpr import models as fprmodels from fpr import utils diff --git a/src/dashboard/src/installer/forms.py b/src/dashboard/src/installer/forms.py index 19981017e6..6419f69054 100644 --- a/src/dashboard/src/installer/forms.py +++ b/src/dashboard/src/installer/forms.py @@ -19,7 +19,7 @@ from django.contrib.auth.forms import UserCreationForm from django.contrib.auth.models import User from django.forms.widgets import TextInput -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from main.models import DashboardSetting diff --git a/src/dashboard/src/installer/urls.py b/src/dashboard/src/installer/urls.py index 171cedc690..62f2bfcd5c 100644 --- a/src/dashboard/src/installer/urls.py +++ b/src/dashboard/src/installer/urls.py @@ -14,11 +14,11 @@ # # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . -from django.conf.urls import url +from django.urls import path from installer import views app_name = "installer" urlpatterns = [ - url(r"welcome/$", views.welcome, name="welcome"), - url(r"storagesetup/$", views.storagesetup, name="storagesetup"), + path("welcome/", views.welcome, name="welcome"), + path("storagesetup/", views.storagesetup, name="storagesetup"), ] diff --git a/src/dashboard/src/installer/views.py b/src/dashboard/src/installer/views.py index d8a116b1bf..7254a9586e 100644 --- a/src/dashboard/src/installer/views.py +++ b/src/dashboard/src/installer/views.py @@ -22,7 +22,7 @@ from django.contrib.auth.models import User from django.shortcuts import redirect from django.shortcuts import render -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from installer.forms import OrganizationForm from installer.forms import SuperUserCreationForm from installer.steps import setup_pipeline diff --git a/src/dashboard/src/main/__init__.py b/src/dashboard/src/main/__init__.py index 965f56572a..e69de29bb2 100644 --- a/src/dashboard/src/main/__init__.py +++ b/src/dashboard/src/main/__init__.py @@ -1 +0,0 @@ -default_app_config = "main.apps.MainAppConfig" diff --git a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py index cd2387f544..1725f01a50 100644 --- a/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py +++ b/src/dashboard/src/main/management/commands/rebuild_transfer_backlog.py @@ -372,8 +372,8 @@ def _import_dir_from_fsentry(cmd, fsentry, transfer_uuid): Directory.objects.create( uuid=uuid.uuid4(), # TODO: what goes here? transfer_id=transfer_uuid, - originallocation="TODO", - currentlocation="TODO") + originallocation=b"TODO", + currentlocation=b"TODO") """ diff --git a/src/dashboard/src/main/migrations/0023_blob_fields.py b/src/dashboard/src/main/migrations/0023_blob_fields.py index 91ea4ea62a..06e5bac577 100644 --- a/src/dashboard/src/main/migrations/0023_blob_fields.py +++ b/src/dashboard/src/main/migrations/0023_blob_fields.py @@ -1,22 +1,21 @@ -import main.models from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [("main", "0022_email_report_args")] operations = [ migrations.AlterField( model_name="file", name="currentlocation", - field=main.models.BlobTextField(null=True, db_column="currentLocation"), + field=models.BinaryField(null=True, db_column="currentLocation"), preserve_default=True, ), migrations.AlterField( model_name="file", name="originallocation", - field=main.models.BlobTextField(db_column="originalLocation"), + field=models.BinaryField(db_column="originalLocation"), preserve_default=True, ), ] diff --git a/src/dashboard/src/main/migrations/0040_directory_model.py b/src/dashboard/src/main/migrations/0040_directory_model.py index b0a6f4b64a..fea4e3b518 100644 --- a/src/dashboard/src/main/migrations/0040_directory_model.py +++ b/src/dashboard/src/main/migrations/0040_directory_model.py @@ -240,11 +240,11 @@ class Migration(migrations.Migration): ), ( "originallocation", - main.models.BlobTextField(db_column="originalLocation"), + models.BinaryField(db_column="originalLocation"), ), ( "currentlocation", - main.models.BlobTextField(null=True, db_column="currentLocation"), + models.BinaryField(null=True, db_column="currentLocation"), ), ( "enteredsystem", diff --git a/src/dashboard/src/main/migrations/0049_change_pointer_file_filegrpuse.py b/src/dashboard/src/main/migrations/0049_change_pointer_file_filegrpuse.py index eda9f2c3f5..81946304c4 100644 --- a/src/dashboard/src/main/migrations/0049_change_pointer_file_filegrpuse.py +++ b/src/dashboard/src/main/migrations/0049_change_pointer_file_filegrpuse.py @@ -23,7 +23,7 @@ def data_migration_down(apps, schema_editor): def update_pointer_files_filegrpuse(apps, filegrpuse): """Change filegrpuse of existing pointer files.""" File = apps.get_model("main", "File") - location = "%SIPDirectory%pointer.xml" + location = b"%SIPDirectory%pointer.xml" pointer_files = File.objects.filter( originallocation=location, currentlocation=location ) diff --git a/src/dashboard/src/main/migrations/0059_siparrange_longblob.py b/src/dashboard/src/main/migrations/0059_siparrange_longblob.py index 32b21f4052..ef8d038d5e 100644 --- a/src/dashboard/src/main/migrations/0059_siparrange_longblob.py +++ b/src/dashboard/src/main/migrations/0059_siparrange_longblob.py @@ -2,6 +2,7 @@ import main.models from django.db import migrations +from django.db import models from django.db import OperationalError @@ -34,7 +35,6 @@ def drop_original_path_unique_key(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [("main", "0058_fix_unit_variable_pull_link")] operations = [ @@ -42,7 +42,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="siparrange", name="arrange_path", - field=main.models.BlobTextField(), + field=models.BinaryField(), ), migrations.AlterField( model_name="siparrange", @@ -59,6 +59,6 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="siparrange", name="original_path", - field=main.models.BlobTextField(default=None, null=True, blank=True), + field=models.BinaryField(default=None, null=True, blank=True), ), ] diff --git a/src/dashboard/src/main/models.py b/src/dashboard/src/main/models.py index b6b1ff3169..13043ba361 100644 --- a/src/dashboard/src/main/models.py +++ b/src/dashboard/src/main/models.py @@ -30,7 +30,7 @@ from django.db import transaction from django.db.models.signals import post_save from django.dispatch import receiver -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from version import get_preservation_system_identifier # Core Django, alphabetical by import source @@ -82,19 +82,6 @@ def get_db_prep_value(self, value, connection, prepared=False): return str(value) -class BlobTextField(models.TextField): - """ - Text field backed by `longblob` instead of `longtext`. - - Used for storing strings that need to match paths on disk with filenames unchanged. - - BLOBs are byte strings (bynary character set and collation). - """ - - def db_type(self, connection): - return "longblob" - - # SIGNALS @@ -625,8 +612,8 @@ def active(self): class SIPArrange(models.Model): """Information about arranged files: original and arranged location, current status.""" - original_path = BlobTextField(null=True, blank=True, default=None) - arrange_path = BlobTextField() + original_path = models.BinaryField(null=True, blank=True, default=None) + arrange_path = models.BinaryField() file_uuid = UUIDField(null=True, blank=True, default=uuid.uuid4, unique=True) transfer_uuid = UUIDField(null=True, blank=True, default=uuid.uuid4) sip = models.ForeignKey( @@ -731,8 +718,8 @@ class File(models.Model): ) # both actually `longblob` in the database - originallocation = BlobTextField(db_column="originalLocation") - currentlocation = BlobTextField(db_column="currentLocation", null=True) + originallocation = models.BinaryField(db_column="originalLocation") + currentlocation = models.BinaryField(db_column="currentLocation", null=True) filegrpuse = models.CharField( max_length=50, db_column="fileGrpUse", default="Original" ) @@ -811,8 +798,8 @@ class Directory(models.Model): blank=True, on_delete=models.CASCADE, ) - originallocation = BlobTextField(db_column="originalLocation") - currentlocation = BlobTextField(db_column="currentLocation", null=True) + originallocation = models.BinaryField(db_column="originalLocation") + currentlocation = models.BinaryField(db_column="currentLocation", null=True) enteredsystem = models.DateTimeField(db_column="enteredSystem", auto_now_add=True) identifiers = models.ManyToManyField("Identifier") @@ -854,8 +841,8 @@ def create_many(cls, dir_paths_uuids, unit_mdl, unit_type="transfer"): **{ "uuid": dir_uuid, unit_type: unit_mdl, - "originallocation": orig_path, - "currentlocation": dir_path, + "originallocation": orig_path.encode(), + "currentlocation": dir_path.encode(), } ) ) diff --git a/src/dashboard/src/main/templatetags/breadcrumb.py b/src/dashboard/src/main/templatetags/breadcrumb.py index f471dc3c7b..51b68f3e7a 100644 --- a/src/dashboard/src/main/templatetags/breadcrumb.py +++ b/src/dashboard/src/main/templatetags/breadcrumb.py @@ -21,7 +21,7 @@ from django.template import Variable from django.template import VariableDoesNotExist from django.template.defaulttags import url -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str logger = logging.getLogger("archivematica.dashboard") register = Library() @@ -88,7 +88,7 @@ def render(self, context): else: title = title.strip("'").strip('"') - title = smart_text(title) + title = smart_str(title) url = None @@ -119,7 +119,7 @@ def render(self, context): title = "" else: title = title.strip("'").strip('"') - title = smart_text(title) + title = smart_str(title) url = self.url_node.render(context) return create_crumb(title, url) diff --git a/src/dashboard/src/main/templatetags/user.py b/src/dashboard/src/main/templatetags/user.py index 0a1d77b269..cfbd03df4f 100644 --- a/src/dashboard/src/main/templatetags/user.py +++ b/src/dashboard/src/main/templatetags/user.py @@ -1,6 +1,6 @@ from django import template from django.urls import reverse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from tastypie.models import ApiKey register = template.Library() diff --git a/src/dashboard/src/main/urls.py b/src/dashboard/src/main/urls.py index de66815ed5..23410830f4 100644 --- a/src/dashboard/src/main/urls.py +++ b/src/dashboard/src/main/urls.py @@ -15,37 +15,38 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . from django.conf import settings -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path +from django.urls import re_path from main import views app_name = "main" urlpatterns = [ # Index - url(r"^$", views.home, name="main_index"), + path("", views.home, name="main_index"), # JavaScript i18n catalog - url( - r"^jsi18n/$", + path( + "jsi18n/", views.cached_javascript_catalog, {"domain": "djangojs"}, name="javascript-catalog", ), # Forbidden - url(r"forbidden/$", views.forbidden, name="forbidden"), + path("forbidden/", views.forbidden, name="forbidden"), # Jobs and tasks (is part of ingest) - url(r"tasks/(?P" + settings.UUID_REGEX + ")/$", views.tasks), - url(r"task/(?P" + settings.UUID_REGEX + ")/$", views.task, name="task"), + re_path(r"tasks/(?P" + settings.UUID_REGEX + ")/$", views.tasks), + re_path(r"task/(?P" + settings.UUID_REGEX + ")/$", views.task, name="task"), # Access - url(r"access/$", views.access_list, name="access_index"), - url(r"access/(?P\d+)/delete/$", views.access_delete, name="access_delete"), + path("access/", views.access_list, name="access_index"), + path("access//delete/", views.access_delete, name="access_delete"), # JSON feeds - url(r"status/$", views.status), - url( + path("status/", views.status), + re_path( r"formdata/(?P\w+)/(?P\d+)/(?P\d+)/$", views.formdata_delete, ), - url(r"formdata/(?P\w+)/(?P\d+)/$", views.formdata), + re_path(r"formdata/(?P\w+)/(?P\d+)/$", views.formdata), ] if "shibboleth" in settings.INSTALLED_APPS: @@ -54,7 +55,7 @@ # the shibboleth lib) class shibboleth_urls: urlpatterns = [ - url(r"^logout/$", views.CustomShibbolethLogoutView.as_view(), name="logout") + path("logout/", views.CustomShibbolethLogoutView.as_view(), name="logout") ] - urlpatterns += [url(r"^shib/", include(shibboleth_urls, namespace="shibboleth"))] + urlpatterns += [path("shib/", include(shibboleth_urls, namespace="shibboleth"))] diff --git a/src/dashboard/src/main/views.py b/src/dashboard/src/main/views.py index 7538de59ca..05e54040d0 100644 --- a/src/dashboard/src/main/views.py +++ b/src/dashboard/src/main/views.py @@ -24,13 +24,12 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import get_language -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import cache_page from django.views.decorators.http import last_modified from django.views.i18n import JavaScriptCatalog from lxml import etree from main import models -from shibboleth.views import LOGOUT_SESSION_KEY from shibboleth.views import ShibbolethLogoutView @@ -322,12 +321,4 @@ def formdata(request, type, parent_id, delete_id=None): class CustomShibbolethLogoutView(ShibbolethLogoutView): - def get(self, request, *args, **kwargs): - response = super().get(request, *args, **kwargs) - # LOGOUT_SESSION_KEY is set by the standard logout to prevent re-login - # which is useful to prevent bouncing straight back to login under - # certain setups, but not here where we want the Django session state - # to reflect the SP session state - if LOGOUT_SESSION_KEY in request.session: - del request.session[LOGOUT_SESSION_KEY] - return response + pass diff --git a/src/dashboard/src/middleware/locale.py b/src/dashboard/src/middleware/locale.py index 191f312ebe..e5026f5eae 100644 --- a/src/dashboard/src/middleware/locale.py +++ b/src/dashboard/src/middleware/locale.py @@ -31,5 +31,5 @@ class ForceDefaultLanguageMiddleware(MiddlewareMixin): """ def process_request(self, request): - if "HTTP_ACCEPT_LANGUAGE" in request.META: + if "accept-language" in request.headers: del request.META["HTTP_ACCEPT_LANGUAGE"] diff --git a/src/dashboard/src/settings/base.py b/src/dashboard/src/settings/base.py index e3d619fc50..91d976fc13 100644 --- a/src/dashboard/src/settings/base.py +++ b/src/dashboard/src/settings/base.py @@ -24,7 +24,7 @@ from appconfig import Config from appconfig import process_search_enabled from django.core.exceptions import ImproperlyConfigured -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def _get_settings_from_file(path): diff --git a/src/dashboard/src/urls.py b/src/dashboard/src/urls.py index f13d3b316c..9c4296bc07 100644 --- a/src/dashboard/src/urls.py +++ b/src/dashboard/src/urls.py @@ -15,42 +15,43 @@ # You should have received a copy of the GNU General Public License # along with Archivematica. If not, see . from django.conf import settings -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path +from django.urls import re_path urlpatterns = [ - url(r"^mcp/", include("components.mcp.urls")), - url(r"^installer/", include("installer.urls")), - url(r"^administration/accounts/", include("components.accounts.urls")), - url(r"^archival-storage/", include("components.archival_storage.urls")), - url(r"^fpr/", include("fpr.urls")), - url( + path("mcp/", include("components.mcp.urls")), + path("installer/", include("installer.urls")), + path("administration/accounts/", include("components.accounts.urls")), + path("archival-storage/", include("components.archival_storage.urls")), + path("fpr/", include("fpr.urls")), + re_path( r"^(?Ptransfer|ingest)/", include("components.unit.urls") ), # URLs common to transfer & ingest - url( + re_path( r"^transfer/(?P" + settings.UUID_REGEX + ")/rights/", include("components.rights.transfer_urls"), ), - url(r"^transfer/", include("components.transfer.urls")), - url(r"^appraisal/", include("components.appraisal.urls")), - url( + path("transfer/", include("components.transfer.urls")), + path("appraisal/", include("components.appraisal.urls")), + re_path( r"^ingest/(?P" + settings.UUID_REGEX + ")/rights/", include("components.rights.ingest_urls"), ), - url(r"^ingest/", include("components.ingest.urls")), - url(r"^administration/", include("components.administration.urls")), - url(r"^filesystem/", include("components.filesystem_ajax.urls")), - url(r"^api/", include("components.api.urls")), - url(r"^file/", include("components.file.urls")), - url(r"^access/", include("components.access.urls")), - url(r"^backlog/", include("components.backlog.urls")), - url(r"", include("main.urls")), + path("ingest/", include("components.ingest.urls")), + path("administration/", include("components.administration.urls")), + path("filesystem/", include("components.filesystem_ajax.urls")), + path("api/", include("components.api.urls")), + path("file/", include("components.file.urls")), + path("access/", include("components.access.urls")), + path("backlog/", include("components.backlog.urls")), + path("", include("main.urls")), ] if settings.PROMETHEUS_ENABLED: # Include prometheus metrics at /metrics - urlpatterns.append(url("", include("django_prometheus.urls"))) + urlpatterns.append(path("", include("django_prometheus.urls"))) if settings.OIDC_AUTHENTICATION: - urlpatterns.append(url(r"^oidc/", include("mozilla_django_oidc.urls"))) + urlpatterns.append(path("oidc/", include("mozilla_django_oidc.urls"))) diff --git a/src/dashboard/tests/fixtures/access.json b/src/dashboard/tests/fixtures/access.json index c6288f0de4..331d6688ff 100644 --- a/src/dashboard/tests/fixtures/access.json +++ b/src/dashboard/tests/fixtures/access.json @@ -14,7 +14,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/-repositories-2-archival_objects-1-9634b79b-a7be-48ec-9961-a91aee35b53a/", + "arrange_path": "L2FycmFuZ2UvLXJlcG9zaXRvcmllcy0yLWFyY2hpdmFsX29iamVjdHMtMS05NjM0Yjc5Yi1hN2JlLTQ4ZWMtOTk2MS1hOTFhZWUzNWI1M2Ev", "original_path": null, "file_uuid": null, "sip_created": false @@ -26,8 +26,8 @@ "fields": { "transfer_uuid": "a29e7e86-eca9-43b6-b059-6f23a9802dc8", "aip_created": false, - "arrange_path": "/arrange/-repositories-2-archival_objects-1-9634b79b-a7be-48ec-9961-a91aee35b53a/evelyn_s_photo.jpg", - "original_path": "originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/objects/evelyn_s_photo.jpg", + "arrange_path": "L2FycmFuZ2UvLXJlcG9zaXRvcmllcy0yLWFyY2hpdmFsX29iamVjdHMtMS05NjM0Yjc5Yi1hN2JlLTQ4ZWMtOTk2MS1hOTFhZWUzNWI1M2EvZXZlbHluX3NfcGhvdG8uanBn", + "original_path": "b3JpZ2luYWxzL25ld3NpcC1hMjllN2U4Ni1lY2E5LTQzYjYtYjA1OS02ZjIzYTk4MDJkYzgvb2JqZWN0cy9ldmVseW5fc19waG90by5qcGc=", "file_uuid": "4fa8f739-b633-4c0f-8833-d108a4f4e88d", "sip_created": false } diff --git a/src/dashboard/tests/fixtures/files.json b/src/dashboard/tests/fixtures/files.json index 215f19afee..bc2af6a67f 100644 --- a/src/dashboard/tests/fixtures/files.json +++ b/src/dashboard/tests/fixtures/files.json @@ -5,7 +5,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/submissionDocumentation/transfer-no-metadata-46260807-ece1-4a0e-b70a-9814c701146b/METS.xml", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3N1Ym1pc3Npb25Eb2N1bWVudGF0aW9uL3RyYW5zZmVyLW5vLW1ldGFkYXRhLTQ2MjYwODA3LWVjZTEtNGEwZS1iNzBhLTk4MTRjNzAxMTQ2Yi9NRVRTLnhtbA==", "transfer": "3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e", "filegrpuse": "submissionDocumentation", "removedtime": null, @@ -13,7 +13,7 @@ "checksum": "51132e5ce1b5d2c2c363f05495f447ea924ab29c2cda2c11037b5fca2119e45a", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/submissionDocumentation/transfer-no-metadata-46260807-ece1-4a0e-b70a-9814c701146b/METS.xml", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL3N1Ym1pc3Npb25Eb2N1bWVudGF0aW9uL3RyYW5zZmVyLW5vLW1ldGFkYXRhLTQ2MjYwODA3LWVjZTEtNGEwZS1iNzBhLTk4MTRjNzAxMTQ2Yi9NRVRTLnhtbA==", "size": 12222 } }, @@ -23,7 +23,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/evelyn_s_photo-6383b731-99e0-432d-a911-a0d2dfd1ce76.tif", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLTYzODNiNzMxLTk5ZTAtNDMyZC1hOTExLWEwZDJkZmQxY2U3Ni50aWY=", "transfer": "3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e", "filegrpuse": "preservation", "removedtime": null, @@ -31,7 +31,7 @@ "checksum": "d82448f154b9185bc777ecb0a3602760eb76ba85dd3098f073b2c91a03f571e9", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/evelyn_s_photo-6383b731-99e0-432d-a911-a0d2dfd1ce76.tif", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLTYzODNiNzMxLTk5ZTAtNDMyZC1hOTExLWEwZDJkZmQxY2U3Ni50aWY=", "size": 1446772 } }, @@ -41,7 +41,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%objects/evelyn's photo.jpg", + "originallocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bidzIHBob3RvLmpwZw==", "transfer": "3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e", "filegrpuse": "original", "removedtime": null, @@ -49,7 +49,7 @@ "checksum": "d2bed92b73c7090bb30a0b30016882e7069c437488e1513e9deaacbe29d38d92", "enteredsystem": "2015-06-24T17:22:09Z", "modificationtime": "2015-06-24T17:22:09Z", - "currentlocation": "%SIPDirectory%objects/evelyn_s_photo.jpg", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL2V2ZWx5bl9zX3Bob3RvLmpwZw==", "size": 158131 } }, @@ -59,7 +59,7 @@ "fields": { "filegrpuuid": "", "sip": "4060ee97-9c3f-4822-afaf-ebdf838284c3", - "originallocation": "%SIPDirectory%metadata/metadata.csv", + "originallocation": "JVNJUERpcmVjdG9yeSVtZXRhZGF0YS9tZXRhZGF0YS5jc3Y=", "transfer": "3e1e56ed-923b-4b53-84fe-c5c1c0b0cf8e", "filegrpuse": "metadata", "removedtime": null, @@ -67,7 +67,7 @@ "checksum": "e8121d8a660e2992872f0b67923d2d08dde9a1ba72dfd58e5a31e68fbac3633c", "enteredsystem": "2015-06-24T17:32:09Z", "modificationtime": "2015-06-24T17:32:09Z", - "currentlocation": "%SIPDirectory%objects/metadata/metadata.csv", + "currentlocation": "JVNJUERpcmVjdG9yeSVvYmplY3RzL21ldGFkYXRhL21ldGFkYXRhLmNzdg==", "size": 154 } } diff --git a/src/dashboard/tests/fixtures/sip_arrange.json b/src/dashboard/tests/fixtures/sip_arrange.json index dd474fc079..c653926e2b 100644 --- a/src/dashboard/tests/fixtures/sip_arrange.json +++ b/src/dashboard/tests/fixtures/sip_arrange.json @@ -5,7 +5,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/newsip/", + "arrange_path": "L2FycmFuZ2UvbmV3c2lwLw==", "original_path": null, "file_uuid": null, "sip_created": false @@ -17,7 +17,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/newsip/objects/", + "arrange_path": "L2FycmFuZ2UvbmV3c2lwL29iamVjdHMv", "original_path": null, "file_uuid": null, "sip_created": false @@ -29,8 +29,8 @@ "fields": { "transfer_uuid": "a29e7e86-eca9-43b6-b059-6f23a9802dc8", "aip_created": false, - "arrange_path": "/arrange/newsip/objects/evelyn_s_photo.jpg", - "original_path": "originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/objects/evelyn_s_photo.jpg", + "arrange_path": "L2FycmFuZ2UvbmV3c2lwL29iamVjdHMvZXZlbHluX3NfcGhvdG8uanBn", + "original_path": "b3JpZ2luYWxzL25ld3NpcC1hMjllN2U4Ni1lY2E5LTQzYjYtYjA1OS02ZjIzYTk4MDJkYzgvb2JqZWN0cy9ldmVseW5fc19waG90by5qcGc=", "file_uuid": "4fa8f739-b633-4c0f-8833-d108a4f4e88d", "sip_created": false } @@ -41,7 +41,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/newsip/objects/evelyn_s_second_photo/", + "arrange_path": "L2FycmFuZ2UvbmV3c2lwL29iamVjdHMvZXZlbHluX3Nfc2Vjb25kX3Bob3RvLw==", "original_path": null, "file_uuid": null, "sip_created": false @@ -53,8 +53,8 @@ "fields": { "transfer_uuid": "a29e7e86-eca9-43b6-b059-6f23a9802dc8", "aip_created": false, - "arrange_path": "/arrange/newsip/objects/evelyn_s_second_photo/evelyn_s_second_photo.jpg", - "original_path": "originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/objects/evelyn_s_second_photo/evelyn_s_second_photo.jpg", + "arrange_path": "L2FycmFuZ2UvbmV3c2lwL29iamVjdHMvZXZlbHluX3Nfc2Vjb25kX3Bob3RvL2V2ZWx5bl9zX3NlY29uZF9waG90by5qcGc=", + "original_path": "b3JpZ2luYWxzL25ld3NpcC1hMjllN2U4Ni1lY2E5LTQzYjYtYjA1OS02ZjIzYTk4MDJkYzgvb2JqZWN0cy9ldmVseW5fc19zZWNvbmRfcGhvdG8vZXZlbHluX3Nfc2Vjb25kX3Bob3RvLmpwZw==", "file_uuid": "7f889d5d-7849-490e-a8e6-ccb9595445d7", "sip_created": false } @@ -65,7 +65,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/toplevel/subsip/", + "arrange_path": "L2FycmFuZ2UvdG9wbGV2ZWwvc3Vic2lwLw==", "original_path": null, "file_uuid": null, "sip_created": false @@ -77,7 +77,7 @@ "fields": { "transfer_uuid": null, "aip_created": false, - "arrange_path": "/arrange/toplevel/", + "arrange_path": "L2FycmFuZ2UvdG9wbGV2ZWwv", "original_path": null, "file_uuid": null, "sip_created": false @@ -89,8 +89,8 @@ "fields": { "transfer_uuid": "a29e7e86-eca9-43b6-b059-6f23a9802dc8", "aip_created": false, - "arrange_path": "/arrange/toplevel/subsip/evelyn_s_third_photo.jpg", - "original_path": "originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/objects/evelyn_s_third_photo/evelyn_s_third_photo.jpg", + "arrange_path": "L2FycmFuZ2UvdG9wbGV2ZWwvc3Vic2lwL2V2ZWx5bl9zX3RoaXJkX3Bob3RvLmpwZw==", + "original_path": "b3JpZ2luYWxzL25ld3NpcC1hMjllN2U4Ni1lY2E5LTQzYjYtYjA1OS02ZjIzYTk4MDJkYzgvb2JqZWN0cy9ldmVseW5fc190aGlyZF9waG90by9ldmVseW5fc190aGlyZF9waG90by5qcGc=", "file_uuid": "0fa7d499-24b9-4cd1-afb2-997eba5351bd", "sip_created": false } diff --git a/src/dashboard/tests/test_access.py b/src/dashboard/tests/test_access.py index 6ad9ef67f0..34d9d4e694 100644 --- a/src/dashboard/tests/test_access.py +++ b/src/dashboard/tests/test_access.py @@ -32,7 +32,8 @@ def test_creating_arrange_directory(self): mapping = models.SIPArrangeAccessMapping.objects.get( system=models.SIPArrangeAccessMapping.ARCHIVESSPACE, identifier=record_id ) - assert models.SIPArrange.objects.get(arrange_path=mapping.arrange_path + "/") + arrange_path = mapping.arrange_path.encode() + b"/" + assert models.SIPArrange.objects.get(arrange_path=arrange_path) def test_arrange_contents(self): record_id = "/repositories/2/archival_objects/1" diff --git a/src/dashboard/tests/test_auth.py b/src/dashboard/tests/test_auth.py index 2b7284519f..168bf496d4 100644 --- a/src/dashboard/tests/test_auth.py +++ b/src/dashboard/tests/test_auth.py @@ -4,9 +4,9 @@ from components.helpers import generate_api_key from django.conf import settings from django.contrib.auth import get_user_model -from django.core.urlresolvers import reverse from django.test import TestCase from django.test.client import Client +from django.urls import reverse from tastypie.models import ApiKey diff --git a/src/dashboard/tests/test_filesystem_ajax.py b/src/dashboard/tests/test_filesystem_ajax.py index c675013df1..bb75e7544e 100644 --- a/src/dashboard/tests/test_filesystem_ajax.py +++ b/src/dashboard/tests/test_filesystem_ajax.py @@ -290,24 +290,24 @@ def test_copy_from_arrange_to_completed_from_bagit_transfer(self): sip_uuid = "a29e7e86-eca9-43b6-b059-6f23a9802dc8" models.SIP.objects.create(uuid=sip_uuid) models.SIPArrange.objects.all().delete() - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/") - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/data/") - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/data/objects/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/data/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/data/objects/") models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/objects/MARBLES.TGA", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/objects/MARBLES.TGA", + arrange_path=b"/arrange/testsip/data/objects/MARBLES.TGA", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/objects/MARBLES.TGA", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", file_uuid="fe077212-094f-4df9-ac51-1bc35be0d95e", ) models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/logs/BagIt/bagit.txt", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/logs/BagIt/bagit.txt", + arrange_path=b"/arrange/testsip/data/logs/BagIt/bagit.txt", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/logs/BagIt/bagit.txt", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", file_uuid="0cf82e9a-0a54-45e8-bdff-700c2fed513c", ) models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/metadata/manifest-md5.txt", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/metadata/manifest-md5.txt", + arrange_path=b"/arrange/testsip/data/metadata/manifest-md5.txt", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/metadata/manifest-md5.txt", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", file_uuid="33c7b9a5-6031-4c89-847d-708fc285b1c1", ) @@ -409,7 +409,7 @@ def test_copy_from_arrange_to_completed_rejects_nondir_filepath(self): def test_copy_from_arrange_to_completed_rejects_empty_arrangements(self): models.SIP.objects.create(uuid="607df760-a0be-4fef-875a-74ea00c61bf9") models.SIPArrange.objects.all().delete() - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/") response = self.client.post( reverse("filesystem_ajax:copy_from_arrange"), @@ -432,22 +432,22 @@ def test_copy_from_arrange_to_completed_handles_name_clashes(self): """ sip_uuid = "a29e7e86-eca9-43b6-b059-6f23a9802dc8" models.SIPArrange.objects.all().delete() - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/") - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/data/") - models.SIPArrange.objects.create(arrange_path="/arrange/testsip/data/objects/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/data/") + models.SIPArrange.objects.create(arrange_path=b"/arrange/testsip/data/objects/") models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/objects/MARBLES.TGA", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/objects/MARBLES.TGA", + arrange_path=b"/arrange/testsip/data/objects/MARBLES.TGA", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/objects/MARBLES.TGA", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", ) models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/logs/BagIt/bagit.txt", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/logs/BagIt/bagit.txt", + arrange_path=b"/arrange/testsip/data/logs/BagIt/bagit.txt", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/logs/BagIt/bagit.txt", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", ) models.SIPArrange.objects.create( - arrange_path="/arrange/testsip/data/metadata/manifest-md5.txt", - original_path="originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/metadata/manifest-md5.txt", + arrange_path=b"/arrange/testsip/data/metadata/manifest-md5.txt", + original_path=b"originals/newsip-a29e7e86-eca9-43b6-b059-6f23a9802dc8/data/metadata/manifest-md5.txt", transfer_uuid="a29e7e86-eca9-43b6-b059-6f23a9802dc8", ) diff --git a/src/dashboard/tests/test_models.py b/src/dashboard/tests/test_models.py index 9f101c94eb..a37d2defbc 100644 --- a/src/dashboard/tests/test_models.py +++ b/src/dashboard/tests/test_models.py @@ -83,8 +83,8 @@ def test_create_user_agent(agent_mock): def test_sip_arrange_create_many(db): arranges = [ - models.SIPArrange(original_path=None, arrange_path="a.txt", file_uuid=None), - models.SIPArrange(original_path=None, arrange_path="b.txt", file_uuid=None), + models.SIPArrange(original_path=None, arrange_path=b"a.txt", file_uuid=None), + models.SIPArrange(original_path=None, arrange_path=b"b.txt", file_uuid=None), ] assert not models.SIPArrange.objects.count() models.SIPArrange.create_many(arranges)