Code refactoring: Simplify MockDeploymentBackend

dependencies/pip/dev_requirements.in

@@ -11,8 +11,8 @@ pytest
 pytest-cov
 pytest-django
 pytest-env
+pytest-xdist
 
-
-# Kobocat
+# KoboCAT
 httmock
 simplejson

dependencies/pip/dev_requirements.txt

@@ -242,6 +242,8 @@ et-xmlfile==1.1.0
     # via openpyxl
 exceptiongroup==1.2.0
     # via pytest
+execnet==2.1.1
+    # via pytest-xdist
 executing==2.0.1
     # via stack-data
 fabric==3.2.2
@@ -469,12 +471,15 @@ pytest==8.1.1
     #   pytest-cov
     #   pytest-django
     #   pytest-env
+    #   pytest-xdist
 pytest-cov==5.0.0
     # via -r dependencies/pip/dev_requirements.in
 pytest-django==4.8.0
     # via -r dependencies/pip/dev_requirements.in
 pytest-env==1.1.3
     # via -r dependencies/pip/dev_requirements.in
+pytest-xdist==3.6.1
+    # via -r dependencies/pip/dev_requirements.in
 python-crontab==3.0.0
     # via django-celery-beat
 python-dateutil==2.9.0.post0

kobo/apps/hook/tests/hook_test_case.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 import json
+import uuid
 
 import pytest
 import responses
@@ -23,16 +24,16 @@ def setUp(self):
         self.asset = self.create_asset(
             "some_asset",
             content=json.dumps({'survey': [
-                {'type': 'text', 'name': 'q1'},
-                {'type': 'begin_group', 'name': 'group1'},
-                {'type': 'text', 'name': 'q2'},
-                {'type': 'text', 'name': 'q3'},
+                {'type': 'text', 'label': 'q1', 'name': 'q1'},
+                {'type': 'begin_group', 'label': 'group1', 'name': 'group1'},
+                {'type': 'text', 'label': 'q2', 'name': 'q2'},
+                {'type': 'text', 'label': 'q3', 'name': 'q3'},
                 {'type': 'end_group'},
-                {'type': 'begin_group', 'name': 'group2'},
-                {'type': 'begin_group', 'name': 'subgroup1'},
-                {'type': 'text', 'name': 'q4'},
-                {'type': 'text', 'name': 'q5'},
-                {'type': 'text', 'name': 'q6'},
+                {'type': 'begin_group', 'label': 'group2', 'name': 'group2'},
+                {'type': 'begin_group', 'label': 'subgroup1', 'name': 'subgroup1'},
+                {'type': 'text', 'label': 'q4', 'name': 'q4'},
+                {'type': 'text', 'label': 'q5', 'name': 'q5'},
+                {'type': 'text', 'label': 'q6', 'name': 'q6'},
                 {'type': 'end_group'},
                 {'type': 'end_group'},
             ]}),
@@ -76,8 +77,9 @@ def _create_hook(self, return_response_only=False, **kwargs):
         if return_response_only:
             return response
         else:
-            self.assertEqual(response.status_code, status.HTTP_201_CREATED,
-                             msg=response.data)
+            self.assertEqual(
+                response.status_code, status.HTTP_201_CREATED, msg=response.data
+            )
             hook = self.asset.hooks.last()
             self.assertTrue(hook.active)
             return hook
@@ -151,14 +153,15 @@ def _send_and_wait_for_retry(self):
 
     def __prepare_submission(self):
         v_uid = self.asset.latest_deployed_version.uid
-        submission = {
+        self.submission = {
             '__version__': v_uid,
             'q1': '¿Qué tal?',
+            '_uuid': str(uuid.uuid4()),
             'group1/q2': '¿Cómo está en el grupo uno la primera vez?',
             'group1/q3': '¿Cómo está en el grupo uno la segunda vez?',
             'group2/subgroup1/q4': '¿Cómo está en el subgrupo uno la primera vez?',
             'group2/subgroup1/q5': '¿Cómo está en el subgrupo uno la segunda vez?',
             'group2/subgroup1/q6': '¿Cómo está en el subgrupo uno la tercera vez?',
             'group2/subgroup11/q1': '¿Cómo está en el subgrupo once?',
         }
-        self.asset.deployment.mock_submissions([submission])
+        self.asset.deployment.mock_submissions([self.submission])

kobo/apps/hook/tests/test_parser.py

@@ -16,10 +16,10 @@ def test_json_parser(self):
 
         ServiceDefinition = hook.get_service_definition()
         submissions = hook.asset.deployment.get_submissions(hook.asset.owner)
-        uuid = submissions[0]['_id']
-        service_definition = ServiceDefinition(hook, uuid)
+        submission_id = submissions[0]['_id']
+        service_definition = ServiceDefinition(hook, submission_id)
         expected_data = {
-            '_id': 1,
+            '_id': submission_id,
             'group1/q3': u'¿Cómo está en el grupo uno la segunda vez?',
             'group2/subgroup1/q4': u'¿Cómo está en el subgrupo uno la primera vez?',
             'group2/subgroup1/q5': u'¿Cómo está en el subgrupo uno la segunda vez?',
@@ -29,26 +29,25 @@ def test_json_parser(self):
 
     def test_xml_parser(self):
         self.asset = self.create_asset(
-            "some_asset_with_xml_submissions",
+            'some_asset_with_xml_submissions',
             content=json.dumps(self.asset.content),
-            format="json")
+            format='json',
+        )
         self.asset.deploy(backend='mock', active=True)
         self.asset.save()
 
-        hook = self._create_hook(subset_fields=['_id', 'subgroup1', 'q3'],
-                                 format_type=SUBMISSION_FORMAT_TYPE_XML)
+        hook = self._create_hook(
+            subset_fields=['meta', 'subgroup1', 'q3'],
+            format_type=SUBMISSION_FORMAT_TYPE_XML,
+        )
 
         ServiceDefinition = hook.get_service_definition()
-        submissions = hook.asset.deployment.get_submissions(
-            self.asset.owner, format_type=SUBMISSION_FORMAT_TYPE_XML)
-        xml_doc = etree.fromstring(submissions[0].encode())
-        tree = etree.ElementTree(xml_doc)
-        uuid = tree.find('_id').text
-
-        service_definition = ServiceDefinition(hook, uuid)
+        submissions = hook.asset.deployment.get_submissions(self.asset.owner)
+        submission_id = submissions[0]['_id']
+        submission_uuid = submissions[0]['_uuid']
+        service_definition = ServiceDefinition(hook, submission_id)
         expected_etree = etree.fromstring(
-            f'<{self.asset.uid}>'
-            f'   <_id>{uuid}</_id>'
+            f'<{self.asset.uid} id="{self.asset.uid}">'
             f'   <group1>'
             f'      <q3>¿Cómo está en el grupo uno la segunda vez?</q3>'
             f'   </group1>'
@@ -59,13 +58,23 @@ def test_xml_parser(self):
             f'          <q6>¿Cómo está en el subgrupo uno la tercera vez?</q6>'
             f'      </subgroup1>'
             f'   </group2>'
+            f'   <meta>'
+            f'      <instanceID>uuid:{submission_uuid}</instanceID>'
+            f'   </meta>'
             f'</{self.asset.uid}>'
         )
-        expected_xml = etree.tostring(expected_etree, pretty_print=True,
-                                      xml_declaration=True, encoding='utf-8')
+
+        expected_xml = etree.tostring(
+            expected_etree,
+            pretty_print=True,
+            xml_declaration=True,
+            encoding='utf-8',
+        )
 
         def remove_whitespace(str_):
             return re.sub(r'>\s+<', '><', to_str(str_))
 
-        self.assertEqual(remove_whitespace(service_definition._get_data()),
-                         remove_whitespace(expected_xml.decode()))
+        self.assertEqual(
+            remove_whitespace(service_definition._get_data()),
+            remove_whitespace(expected_xml.decode()),
+        )

kobo/apps/kobo_auth/__init__.py

@@ -6,4 +6,5 @@ class KoboAuthAppConfig(AppConfig):
     verbose_name = 'Authentication and authorization'
 
     def ready(self):
+        from . import signals
         super().ready()

kobo/apps/kobo_auth/signals.py

@@ -0,0 +1,58 @@
+from django.conf import settings
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from rest_framework.authtoken.models import Token
+
+from kobo.apps.kobo_auth.shortcuts import User
+from kobo.apps.openrosa.apps.main.models.user_profile import UserProfile
+from kpi.deployment_backends.kc_access.utils import (
+    grant_kc_model_level_perms,
+    kc_transaction_atomic,
+)
+from kpi.utils.permissions import (
+    grant_default_model_level_perms,
+    is_user_anonymous,
+)
+
+
+@receiver(post_save, sender=User)
+def create_auth_token(sender, instance=None, created=False, **kwargs):
+    if is_user_anonymous(instance):
+        return
+
+    if created:
+        Token.objects.get_or_create(user_id=instance.pk)
+
+
+@receiver(post_save, sender=User)
+def default_permissions_post_save(sender, instance, created, raw, **kwargs):
+    """
+    Users must have both model-level and object-level permissions to satisfy
+    DRF, so assign the newly-created user all available collection and asset
+    permissions at the model level
+    """
+    if raw:
+        # `raw` means we can't touch (so make sure your fixtures include
+        # all necessary permissions!)
+        return
+    if not created:
+        # We should only grant default permissions when the user is first
+        # created
+        return
+    grant_default_model_level_perms(instance)
+
+
+@receiver(post_save, sender=User)
+def save_kobocat_user(sender, instance, created, raw, **kwargs):
+    """
+    Sync auth_user table between KPI and KC, and, if the user is newly created,
+    grant all KoboCAT model-level permissions for the content types listed in
+    `settings.KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES`
+    """
+
+    if not settings.TESTING:
+        with kc_transaction_atomic():
+            instance.sync_to_openrosa_db()
+            if created:
+                grant_kc_model_level_perms(instance)
+                UserProfile.objects.get_or_create(user=instance)

kobo/apps/openrosa/apps/api/viewsets/xform_submission_api.py

@@ -26,6 +26,7 @@
     safe_create_instance,
     UnauthenticatedEditAttempt,
 )
+from kobo.apps.openrosa.libs.utils.string import dict_lists2strings
 from kpi.authentication import DigestAuthentication
 from kpi.utils.object_permission import get_database_user
 from ..utils.rest_framework.viewsets import OpenRosaGenericViewSet
@@ -37,25 +38,11 @@ def is_json(request):
     return 'application/json' in request.content_type.lower()
 
 
-def dict_lists2strings(d):
-    """Convert lists in a dict to joined strings.
-
-    :param d: The dict to convert.
-    :returns: The converted dict."""
-    for k, v in d.items():
-        if isinstance(v, list) and all([isinstance(e, str) for e in v]):
-            d[k] = ' '.join(v)
-        elif isinstance(v, dict):
-            d[k] = dict_lists2strings(v)
-
-    return d
-
-
 def create_instance_from_xml(username, request):
     xml_file_list = request.FILES.pop('xml_submission_file', [])
     xml_file = xml_file_list[0] if len(xml_file_list) else None
     media_files = request.FILES.values()
-    return safe_create_instance(username, xml_file, media_files, None, request)
+    return safe_create_instance(username, xml_file, media_files, None, request=request)
 
 
 def create_instance_from_json(username, request):
@@ -73,7 +60,7 @@ def create_instance_from_json(username, request):
     xml_string = dict2xform(submission_joined, dict_form.get('id'))
 
     xml_file = io.StringIO(xml_string)
-    return safe_create_instance(username, xml_file, [], None, request)
+    return safe_create_instance(username, xml_file, [], None, request=request)
 
 
 class XFormSubmissionApi(

kobo/apps/openrosa/apps/logger/models/instance.py

@@ -6,6 +6,7 @@
     from backports.zoneinfo import ZoneInfo
 
 import reversion
+from django.apps import apps
 from django.contrib.gis.db import models
 from django.contrib.gis.geos import GeometryCollection, Point
 from django.utils import timezone
@@ -21,8 +22,11 @@
 from kobo.apps.openrosa.apps.logger.fields import LazyDefaultBooleanField
 from kobo.apps.openrosa.apps.logger.models.survey_type import SurveyType
 from kobo.apps.openrosa.apps.logger.models.xform import XForm
-from kobo.apps.openrosa.apps.logger.xform_instance_parser import XFormInstanceParser, \
-    clean_and_parse_xml, get_uuid_from_xml
+from kobo.apps.openrosa.apps.logger.xform_instance_parser import (
+    XFormInstanceParser,
+    clean_and_parse_xml,
+    get_uuid_from_xml,
+)
 from kobo.apps.openrosa.libs.utils.common_tags import (
     ATTACHMENTS,
     GEOLOCATION,
@@ -129,12 +133,12 @@ def check_active(self, force):
             return
         if self.xform and not self.xform.downloadable:
             raise FormInactiveError()
-        try:
-            profile = self.xform.user.profile
-        except self.xform.user.profile.RelatedObjectDoesNotExist:
-            return
-        if profile.metadata.get('submissions_suspended', False):
+
+        UserProfile = apps.get_model('main', 'UserProfile')  # noqa - Avoid circular imports
+        profile, created = UserProfile.objects.get_or_create(user=self.xform.user)
+        if not created and profile.metadata.get('submissions_suspended', False):
             raise TemporarilyUnavailableError()
+        return
 
     def _set_geom(self):
         xform = self.xform

kobo/apps/openrosa/apps/logger/models/xform.py

@@ -28,6 +28,7 @@
 from kpi.deployment_backends.kc_access.storage import (
     default_kobocat_storage as default_storage,
 )
+from kpi.fields.file import ExtendedFileField
 from kpi.utils.xml import XMLFormWithDisclaimer
 
 XFORM_TITLE_LENGTH = 255
@@ -53,7 +54,7 @@ class XForm(models.Model):
     CLONED_SUFFIX = '_cloned'
     MAX_ID_LENGTH = 100
 
-    xls = models.FileField(
+    xls = ExtendedFileField(
         storage=default_storage, upload_to=upload_to, null=True
     )
     json = models.TextField(default='')

kobo/apps/openrosa/apps/logger/signals.py

@@ -185,7 +185,7 @@ def update_xform_submission_count(sender, instance, created, **kwargs):
             last_submission_time=instance.date_created,
         )
         # Hack to avoid circular imports
-        UserProfile = User.profile.related.related_model
+        UserProfile = User.profile.related.related_model  # noqa
         profile, created = UserProfile.objects.only('pk').get_or_create(
             user_id=xform.user_id
         )

kobo/apps/openrosa/apps/logger/tests/models/test_attachment.py

@@ -12,6 +12,7 @@
     default_kobocat_storage as default_storage,
 )
 
+
 class TestAttachment(TestBase):
 
     def setUp(self):

kobo/apps/openrosa/apps/logger/tests/test_publish_xls.py

@@ -13,6 +13,7 @@
 from kobo.apps.openrosa.apps.logger.models.xform import XForm
 from kobo.apps.openrosa.libs.utils.logger_tools import report_exception
 
+
 class TestPublishXLS(TestBase):
 
     def test_publish_xls(self):

kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py

@@ -120,7 +120,8 @@ def test_corrupted_submission(self):
         request = RequestFactory().post('/')
         request.user = self.user
         error, instance = safe_create_instance(
-            self.user.username, TempFileProxy(xml), None, None, request)
+            self.user.username, TempFileProxy(xml), None, None, request=request
+        )
         # No `DjangoUnicodeDecodeError` errors are raised anymore.
         # An `ExpatError` is raised instead
         text = 'Improperly formatted XML'