Upgrade Django to 3.2

* Add django-upgrade to pre-commit config
* Run pre-commit
* Upgrade requirements
* Allow compilemessages to write *.mo files
* Update test workflow
* Replace custom BlobTextField with Django's BinaryField
* Fix tests
* Make migrations
* Update submodules

.github/workflows/test.yml

@@ -129,10 +129,10 @@ jobs:
     steps:
       - name: "Check out repository"
         uses: "actions/checkout@v3"
-      - name: "Set up Python 3.7"
+      - name: "Set up Python 3.8"
         uses: "actions/setup-python@v4"
         with:
-          python-version: "3.7"
+          python-version: "3.8"
       - name: "Install tox"
         run: |
           python -m pip install --upgrade pip

.pre-commit-config.yaml

@@ -19,6 +19,11 @@ repos:
   hooks:
   - id: reorder-python-imports
     args: [--py3-plus, --py36-plus]
+- repo: https://github.com/adamchainz/django-upgrade
+  rev: "1.14.0"
+  hooks:
+  - id: django-upgrade
+    args: [--target-version, "3.2"]
 - repo: https://github.com/ambv/black
   rev: 22.8.0
   hooks:

hack/Dockerfile

@@ -94,6 +94,12 @@ RUN set -ex \
 
 COPY . /src
 
+# Allow Django's compilemessages to write *.mo files to the messages subdirectories.
+USER root
+RUN set -ex \
+	&& find /src/src/dashboard/src/locale -type d -name 'LC_MESSAGES' -exec chown archivematica:archivematica '{}' \;
+USER archivematica
+
 # -----------------------------------------------------------------------------
 
 FROM base AS archivematica-mcp-client

requirements-dev.txt

@@ -10,6 +10,10 @@ amclient==1.2.3
     # via -r requirements.txt
 ammcpc==0.1.3
     # via -r requirements.txt
+asgiref==3.4.1
+    # via
+    #   -r requirements.txt
+    #   django
 attrs==22.2.0
     # via
     #   -r requirements.txt
@@ -19,7 +23,7 @@ bagit==1.8.1
     # via -r requirements.txt
 brotli==1.0.9
     # via -r requirements.txt
-certifi==2023.5.7
+certifi==2023.7.22
     # via
     #   -r requirements.txt
     #   requests
@@ -49,28 +53,28 @@ cryptography==40.0.2
     #   pyopenssl
 distlib==0.3.7
     # via virtualenv
-django==1.11.29
+django==3.2.20
     # via
     #   -r requirements.txt
     #   django-auth-ldap
     #   django-cas-ng
     #   django-csp
     #   mozilla-django-oidc
-django-auth-ldap==2.2.0
+django-auth-ldap==4.0.0
     # via -r requirements.txt
-django-autoslug==1.9.8
+django-autoslug==1.9.9
     # via -r requirements.txt
-django-cas-ng==3.6.0
+django-cas-ng==4.3.0
     # via -r requirements.txt
 django-csp==3.7
     # via -r requirements.txt
 django-forms-bootstrap==3.1.0
     # via -r requirements.txt
-django-prometheus==2.0.0
+django-prometheus==2.2.0
     # via -r requirements.txt
-django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76
+django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592
     # via -r requirements.txt
-django-tastypie==0.14.3
+django-tastypie==0.14.5
     # via -r requirements.txt
 elasticsearch==6.8.2
     # via -r requirements.txt
@@ -88,7 +92,7 @@ greenlet==2.0.2
     # via
     #   -r requirements.txt
     #   gevent
-gunicorn==21.0.1
+gunicorn==21.2.0
     # via -r requirements.txt
 idna==3.4
     # via
@@ -134,7 +138,7 @@ metsrw==0.4.0
     # via -r requirements.txt
 mockldap @ git+https://github.com/artefactual-labs/[email protected]
     # via -r requirements-dev.in
-mozilla-django-oidc==1.2.4
+mozilla-django-oidc==2.0.0
     # via -r requirements.txt
 multidict==5.2.0
     # via yarl
@@ -191,7 +195,7 @@ pyopenssl==23.2.0
     # via
     #   -r requirements.txt
     #   josepy
-pyparsing==3.1.0
+pyparsing==3.1.1
     # via
     #   -r requirements.txt
     #   packaging
@@ -235,7 +239,7 @@ pytz==2023.3
     # via
     #   -r requirements.txt
     #   django
-pyyaml==6.0
+pyyaml==6.0.1
     # via vcrpy
 requests==2.27.1
     # via
@@ -250,12 +254,15 @@ six==1.16.0
     #   -r requirements.txt
     #   amclient
     #   jsonschema
-    #   mozilla-django-oidc
     #   opf-fido
     #   python-cas
     #   python-dateutil
     #   tox
     #   vcrpy
+sqlparse==0.4.4
+    # via
+    #   -r requirements.txt
+    #   django
 toml==0.10.2
     # via tox
 tomli==1.2.3
@@ -269,6 +276,7 @@ tox==3.28.0
 typing-extensions==4.1.1
     # via
     #   -r requirements.txt
+    #   asgiref
     #   importlib-metadata
     #   yarl
 unidecode==1.3.6

requirements.in

@@ -1,16 +1,16 @@
-git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76#egg=django-shibboleth-remoteuser
-Django>=1.11,<2
+git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592#egg=django-shibboleth-remoteuser
+Django>=3.2,<4
 agentarchives
 amclient
 ammcpc
 bagit
 brotli
 clamd
-django-autoslug==1.9.8
+django-autoslug
 django-csp
-django-forms-bootstrap>=3.0.0,<4.0.0
-django-prometheus>=2.0,<2.1
-django-tastypie==0.14.3
+django-forms-bootstrap
+django-prometheus>=2.2,<2.3
+django-tastypie
 elasticsearch>=6.0.0,<7.0.0
 gearman3
 gevent
@@ -22,21 +22,21 @@ lazy-paged-sequence
 lxml
 metsrw
 mysqlclient
-opf-fido==1.6.1
+opf-fido
 pip
 pip-tools
 prometheus_client
 python-dateutil
 requests~=2.27
 unidecode
-whitenoise~=5.3
+whitenoise>=5.3.0,<6.0
 
 # Required by LDAP authentication
-django-auth-ldap>=2.2,<2.3
+django-auth-ldap
 python-ldap
 
 # Required by CAS authentication
-django-cas-ng~=3.6
+django-cas-ng
 
 # Required for OpenID Connect authentication
-mozilla-django-oidc~=1.2
+mozilla-django-oidc~=2.0

requirements.txt

@@ -10,13 +10,15 @@ amclient==1.2.3
     # via -r requirements.in
 ammcpc==0.1.3
     # via -r requirements.in
+asgiref==3.4.1
+    # via django
 attrs==22.2.0
     # via jsonschema
 bagit==1.8.1
     # via -r requirements.in
 brotli==1.0.9
     # via -r requirements.in
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
 cffi==1.15.1
     # via cryptography
@@ -31,28 +33,28 @@ cryptography==40.0.2
     #   josepy
     #   mozilla-django-oidc
     #   pyopenssl
-django==1.11.29
+django==3.2.20
     # via
     #   -r requirements.in
     #   django-auth-ldap
     #   django-cas-ng
     #   django-csp
     #   mozilla-django-oidc
-django-auth-ldap==2.2.0
+django-auth-ldap==4.0.0
     # via -r requirements.in
-django-autoslug==1.9.8
+django-autoslug==1.9.9
     # via -r requirements.in
-django-cas-ng==3.6.0
+django-cas-ng==4.3.0
     # via -r requirements.in
 django-csp==3.7
     # via -r requirements.in
 django-forms-bootstrap==3.1.0
     # via -r requirements.in
-django-prometheus==2.0.0
+django-prometheus==2.2.0
     # via -r requirements.in
-django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@67d270c65c201606fb86d548493d4b3fd8cc7a76
+django-shibboleth-remoteuser @ git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git@962f6f9818683ef5f6432f091d22945e54b82592
     # via -r requirements.in
-django-tastypie==0.14.3
+django-tastypie==0.14.5
     # via -r requirements.in
 elasticsearch==6.8.2
     # via -r requirements.in
@@ -62,7 +64,7 @@ gevent==22.10.2
     # via -r requirements.in
 greenlet==2.0.2
     # via gevent
-gunicorn==21.0.1
+gunicorn==21.2.0
     # via -r requirements.in
 idna==3.4
     # via requests
@@ -91,7 +93,7 @@ lxml==4.9.3
     #   python-cas
 metsrw==0.4.0
     # via -r requirements.in
-mozilla-django-oidc==1.2.4
+mozilla-django-oidc==2.0.0
     # via -r requirements.in
 mysqlclient==1.4.6
     # via
@@ -121,7 +123,7 @@ pycparser==2.21
     # via cffi
 pyopenssl==23.2.0
     # via josepy
-pyparsing==3.1.0
+pyparsing==3.1.1
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
@@ -151,14 +153,17 @@ six==1.16.0
     # via
     #   amclient
     #   jsonschema
-    #   mozilla-django-oidc
     #   opf-fido
     #   python-cas
     #   python-dateutil
+sqlparse==0.4.4
+    # via django
 tomli==1.2.3
     # via pep517
 typing-extensions==4.1.1
-    # via importlib-metadata
+    # via
+    #   asgiref
+    #   importlib-metadata
 unidecode==1.3.6
     # via -r requirements.in
 urllib3==1.26.16

src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py

@@ -60,7 +60,7 @@ def process_xml_metadata(mets, sip_dir, sip_uuid, sip_type, xml_validation):
                 try:
                     metadata_file = models.File.objects.get(
                         sip_id=sip_uuid,
-                        currentlocation=f"%SIPDirectory%{xml_rel_path}",
+                        currentlocation=f"%SIPDirectory%{xml_rel_path}".encode(),
                     )
                 except models.File.DoesNotExist:
                     xml_metadata_errors.append(f"No uuid for file: {xml_rel_path}")

src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py

@@ -437,7 +437,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
                 if rel_path != old_mets_rel_path:
                     job.pyprint(rel_path, "not found in METS, must be new file")
                     f = models.File.objects.get(
-                        currentlocation=current_loc, sip_id=str(sip_uuid)
+                        currentlocation=current_loc.encode(), sip_id=str(sip_uuid)
                     )
                     new_files.append(f)
                     if rel_path == "objects/metadata/metadata.csv":
@@ -476,7 +476,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
 
         # Create parent directories if needed
         dirs = os.path.dirname(
-            f.currentlocation.replace("%SIPDirectory%objects/", "", 1)
+            f.currentlocation.decode().replace("%SIPDirectory%objects/", "", 1)
         ).split("/")
         parent_fsentry = objects_fsentry
         for dirname in (d for d in dirs if d):
@@ -491,7 +491,7 @@ def add_new_files(job, mets, sip_uuid, sip_dir):
             original_f = f.original_file_set.get().source_file
             derived_from = mets.get_file(file_uuid=str(original_f.uuid))
         entry = metsrw.FSEntry(
-            path=f.currentlocation.replace("%SIPDirectory%", "", 1),
+            path=f.currentlocation.decode().replace("%SIPDirectory%", "", 1),
             use=f.filegrpuse,
             type="Item",
             file_uuid=str(f.uuid),
@@ -545,7 +545,9 @@ def _get_directory_fsentry(mets, path):
 
 def update_metadata_csv(job, mets, metadata_csv, sip_uuid, sip_dir, state):
     job.pyprint("Parse new metadata.csv")
-    full_path = metadata_csv.currentlocation.replace("%SIPDirectory%", sip_dir, 1)
+    full_path = metadata_csv.currentlocation.decode().replace(
+        "%SIPDirectory%", sip_dir, 1
+    )
     csvmetadata = createmetscsv.parseMetadataCSV(job, full_path)
 
     # FIXME This doesn't support having both DC and non-DC metadata in dmdSecs
@@ -607,7 +609,6 @@ def _get_old_mets_rel_path(sip_uuid):
 
 
 def update_mets(job, sip_dir, sip_uuid, state):
-
     old_mets_path = os.path.join(sip_dir, _get_old_mets_rel_path(sip_uuid))
     job.pyprint("Looking for old METS at path", old_mets_path)
 

src/MCPClient/lib/clientScripts/archivematicaCreateMETSRightsDspaceMDRef.py

@@ -61,12 +61,14 @@ def archivematicaCreateMETSRightsDspaceMDRef(
         # Find the mets file. May find none.
         path = f"%SIPDirectory%{os.path.dirname(filePath)}/mets.xml"
         try:
-            mets = File.objects.get(currentlocation=path, transfer_id=transferUUID)
+            mets = File.objects.get(
+                currentlocation=path.encode(), transfer_id=transferUUID
+            )
         except File.DoesNotExist:
             pass
         else:
             metsFileUUID = mets.uuid
-            metsLoc = mets.currentlocation.replace("%SIPDirectory%", "", 1)
+            metsLoc = mets.currentlocation.decode().replace("%SIPDirectory%", "", 1)
             metsLocation = os.path.join(os.path.dirname(itemdirectoryPath), "mets.xml")
             LABEL = "mets.xml-%s" % (metsFileUUID)
             ret.append(createMDRefDMDSec(LABEL, metsLocation, metsLoc))
@@ -86,12 +88,14 @@ def archivematicaCreateMETSRightsDspaceMDRef(
 
             path = f"%SIPDirectory%{fullDir2}/mets.xml"
             try:
-                f = File.objects.get(currentlocation=path, transfer_id=transferUUID)
+                f = File.objects.get(
+                    currentlocation=path.encode(), transfer_id=transferUUID
+                )
             except File.DoesNotExist:
                 pass
             else:
                 metsFileUUID = f.uuid
-                metsLoc = f.currentlocation.replace("%SIPDirectory%", "", 1)
+                metsLoc = f.currentlocation.decode().replace("%SIPDirectory%", "", 1)
                 metsLocation = os.path.join(fullDir, "mets.xml")
                 job.pyprint(metsLocation)
                 LABEL = "mets.xml-" + metsFileUUID