From 276d8e9a308803ae6847e02d73c38f6e3480a5fb Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 6 Aug 2024 09:01:04 -0400 Subject: [PATCH 01/10] Fix bug on redeploy --- kpi/deployment_backends/openrosa_backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index 83c906194a..ca0c93e3e7 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -838,11 +838,9 @@ def redeploy(self, active=None): XForm.objects.filter(pk=self.xform.id).update( downloadable=active, title=self.asset.name, - has_kpi_hooks=self.asset.has_active_hooks, ) self.xform.downloadable = active self.xform.title = self.asset.name - self.xform.has_kpi_hooks = self.asset.has_active_hooks publish_xls_form(xlsx_file, self.asset.owner, self.xform.id_string) From 726fb9a81e3d57eeccd7cc121d97f2da20b533fa Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 6 Aug 2024 10:33:11 -0400 Subject: [PATCH 02/10] Refactor MockDeploymentBackend: fix unit tests --- kobo/apps/hook/tests/hook_test_case.py | 29 +- kobo/apps/hook/tests/test_parser.py | 49 +- kobo/apps/kobo_auth/__init__.py | 1 + kobo/apps/kobo_auth/signals.py | 58 ++ .../apps/api/viewsets/xform_submission_api.py | 19 +- .../openrosa/apps/logger/models/instance.py | 23 +- .../apps/openrosa/apps/logger/models/xform.py | 3 +- kobo/apps/openrosa/apps/logger/signals.py | 2 +- .../logger/tests/test_simple_submission.py | 3 +- .../openrosa/apps/logger/utils/instance.py | 35 +- .../apps/logger/xform_instance_parser.py | 14 +- .../openrosa/apps/main/models/meta_data.py | 3 +- kobo/apps/openrosa/libs/utils/logger_tools.py | 58 +- kobo/apps/openrosa/libs/utils/string.py | 17 + .../tests/api/v2/test_api.py | 12 +- .../stripe/tests/test_organization_usage.py | 26 +- .../tests/test_submission_extras_api_post.py | 13 +- .../tests/test_submission_stream.py | 7 +- kobo/apps/subsequences/utils/__init__.py | 39 +- kobo/apps/trackers/submission_utils.py | 82 +- kobo/apps/trackers/tests/test_trackers.py | 2 +- kobo/settings/base.py | 6 - kobo/settings/testing.py | 4 +- kpi/deployment_backends/base_backend.py | 5 + kpi/deployment_backends/mock_backend.py | 858 +++--------------- kpi/deployment_backends/openrosa_backend.py | 72 +- kpi/fixtures/test_data.json | 27 + kpi/signals.py | 51 -- kpi/tests/api/v1/test_api_assets.py | 12 +- kpi/tests/api/v1/test_api_submissions.py | 42 +- kpi/tests/api/v2/test_api_asset_counts.py | 9 + kpi/tests/api/v2/test_api_asset_usage.py | 13 +- kpi/tests/api/v2/test_api_attachments.py | 154 +--- kpi/tests/api/v2/test_api_service_usage.py | 86 +- kpi/tests/api/v2/test_api_submissions.py | 575 +++++++----- kpi/tests/kpi_test_case.py | 5 +- kpi/tests/test_asset_versions.py | 23 +- kpi/tests/test_deployment_backends.py | 45 +- kpi/tests/test_mock_data.py | 606 +++++++++---- ...t_mock_data_conflicting_version_exports.py | 3 +- kpi/tests/test_mock_data_exports.py | 134 +-- kpi/tests/test_mongo_helper.py | 17 +- kpi/tests/utils/dicts.py | 35 + kpi/tests/utils/mock.py | 28 +- kpi/tests/utils/xml.py | 2 +- kpi/utils/files.py | 10 +- kpi/views/v2/attachment.py | 2 +- kpi/views/v2/data.py | 10 +- 48 files changed, 1541 insertions(+), 1788 deletions(-) create mode 100644 kobo/apps/kobo_auth/signals.py create mode 100644 kpi/tests/utils/dicts.py diff --git a/kobo/apps/hook/tests/hook_test_case.py b/kobo/apps/hook/tests/hook_test_case.py index c5c31d420a..66450390b5 100644 --- a/kobo/apps/hook/tests/hook_test_case.py +++ b/kobo/apps/hook/tests/hook_test_case.py @@ -1,5 +1,6 @@ # coding: utf-8 import json +import uuid import pytest import responses @@ -30,16 +31,16 @@ def setUp(self): self.asset = self.create_asset( "some_asset", content=json.dumps({'survey': [ - {'type': 'text', 'name': 'q1'}, - {'type': 'begin_group', 'name': 'group1'}, - {'type': 'text', 'name': 'q2'}, - {'type': 'text', 'name': 'q3'}, + {'type': 'text', 'label': 'q1', 'name': 'q1'}, + {'type': 'begin_group', 'label': 'group1', 'name': 'group1'}, + {'type': 'text', 'label': 'q2', 'name': 'q2'}, + {'type': 'text', 'label': 'q3', 'name': 'q3'}, {'type': 'end_group'}, - {'type': 'begin_group', 'name': 'group2'}, - {'type': 'begin_group', 'name': 'subgroup1'}, - {'type': 'text', 'name': 'q4'}, - {'type': 'text', 'name': 'q5'}, - {'type': 'text', 'name': 'q6'}, + {'type': 'begin_group', 'label': 'group2', 'name': 'group2'}, + {'type': 'begin_group', 'label': 'subgroup1', 'name': 'subgroup1'}, + {'type': 'text', 'label': 'q4', 'name': 'q4'}, + {'type': 'text', 'label': 'q5', 'name': 'q5'}, + {'type': 'text', 'label': 'q6', 'name': 'q6'}, {'type': 'end_group'}, {'type': 'end_group'}, ]}), @@ -83,8 +84,9 @@ def _create_hook(self, return_response_only=False, **kwargs): if return_response_only: return response else: - self.assertEqual(response.status_code, status.HTTP_201_CREATED, - msg=response.data) + self.assertEqual( + response.status_code, status.HTTP_201_CREATED, msg=response.data + ) hook = self.asset.hooks.last() self.assertTrue(hook.active) return hook @@ -158,9 +160,10 @@ def _send_and_wait_for_retry(self): def __prepare_submission(self): v_uid = self.asset.latest_deployed_version.uid - submission = { + self.submission = { '__version__': v_uid, 'q1': '¿Qué tal?', + '_uuid': str(uuid.uuid4()), 'group1/q2': '¿Cómo está en el grupo uno la primera vez?', 'group1/q3': '¿Cómo está en el grupo uno la segunda vez?', 'group2/subgroup1/q4': '¿Cómo está en el subgrupo uno la primera vez?', @@ -168,4 +171,4 @@ def __prepare_submission(self): 'group2/subgroup1/q6': '¿Cómo está en el subgrupo uno la tercera vez?', 'group2/subgroup11/q1': '¿Cómo está en el subgrupo once?', } - self.asset.deployment.mock_submissions([submission]) + self.asset.deployment.mock_submissions([self.submission]) diff --git a/kobo/apps/hook/tests/test_parser.py b/kobo/apps/hook/tests/test_parser.py index 33f406a8c1..8f890ec778 100644 --- a/kobo/apps/hook/tests/test_parser.py +++ b/kobo/apps/hook/tests/test_parser.py @@ -16,10 +16,10 @@ def test_json_parser(self): ServiceDefinition = hook.get_service_definition() submissions = hook.asset.deployment.get_submissions(hook.asset.owner) - uuid = submissions[0]['_id'] - service_definition = ServiceDefinition(hook, uuid) + submission_id = submissions[0]['_id'] + service_definition = ServiceDefinition(hook, submission_id) expected_data = { - '_id': 1, + '_id': submission_id, 'group1/q3': u'¿Cómo está en el grupo uno la segunda vez?', 'group2/subgroup1/q4': u'¿Cómo está en el subgrupo uno la primera vez?', 'group2/subgroup1/q5': u'¿Cómo está en el subgrupo uno la segunda vez?', @@ -29,26 +29,25 @@ def test_json_parser(self): def test_xml_parser(self): self.asset = self.create_asset( - "some_asset_with_xml_submissions", + 'some_asset_with_xml_submissions', content=json.dumps(self.asset.content), - format="json") + format='json', + ) self.asset.deploy(backend='mock', active=True) self.asset.save() - hook = self._create_hook(subset_fields=['_id', 'subgroup1', 'q3'], - format_type=SUBMISSION_FORMAT_TYPE_XML) + hook = self._create_hook( + subset_fields=['meta', 'subgroup1', 'q3'], + format_type=SUBMISSION_FORMAT_TYPE_XML, + ) ServiceDefinition = hook.get_service_definition() - submissions = hook.asset.deployment.get_submissions( - self.asset.owner, format_type=SUBMISSION_FORMAT_TYPE_XML) - xml_doc = etree.fromstring(submissions[0].encode()) - tree = etree.ElementTree(xml_doc) - uuid = tree.find('_id').text - - service_definition = ServiceDefinition(hook, uuid) + submissions = hook.asset.deployment.get_submissions(self.asset.owner) + submission_id = submissions[0]['_id'] + submission_uuid = submissions[0]['_uuid'] + service_definition = ServiceDefinition(hook, submission_id) expected_etree = etree.fromstring( - f'<{self.asset.uid}>' - f' <_id>{uuid}' + f'<{self.asset.uid} id="{self.asset.uid}">' f' ' f' ¿Cómo está en el grupo uno la segunda vez?' f' ' @@ -59,13 +58,23 @@ def test_xml_parser(self): f' ¿Cómo está en el subgrupo uno la tercera vez?' f' ' f' ' + f' ' + f' uuid:{submission_uuid}' + f' ' f'' ) - expected_xml = etree.tostring(expected_etree, pretty_print=True, - xml_declaration=True, encoding='utf-8') + + expected_xml = etree.tostring( + expected_etree, + pretty_print=True, + xml_declaration=True, + encoding='utf-8', + ) def remove_whitespace(str_): return re.sub(r'>\s+<', '><', to_str(str_)) - self.assertEqual(remove_whitespace(service_definition._get_data()), - remove_whitespace(expected_xml.decode())) + self.assertEqual( + remove_whitespace(service_definition._get_data()), + remove_whitespace(expected_xml.decode()), + ) diff --git a/kobo/apps/kobo_auth/__init__.py b/kobo/apps/kobo_auth/__init__.py index cbbf997004..4b7550c50b 100644 --- a/kobo/apps/kobo_auth/__init__.py +++ b/kobo/apps/kobo_auth/__init__.py @@ -6,4 +6,5 @@ class KoboAuthAppConfig(AppConfig): verbose_name = 'Authentication and authorization' def ready(self): + from . import signals super().ready() diff --git a/kobo/apps/kobo_auth/signals.py b/kobo/apps/kobo_auth/signals.py new file mode 100644 index 0000000000..bc4a80d21c --- /dev/null +++ b/kobo/apps/kobo_auth/signals.py @@ -0,0 +1,58 @@ +from django.conf import settings +from django.db.models.signals import post_save +from django.dispatch import receiver +from rest_framework.authtoken.models import Token + +from kobo.apps.kobo_auth.shortcuts import User +from kobo.apps.openrosa.apps.main.models.user_profile import UserProfile +from kpi.deployment_backends.kc_access.utils import ( + grant_kc_model_level_perms, + kc_transaction_atomic, +) +from kpi.utils.permissions import ( + grant_default_model_level_perms, + is_user_anonymous, +) + + +@receiver(post_save, sender=User) +def create_auth_token(sender, instance=None, created=False, **kwargs): + if is_user_anonymous(instance): + return + + if created: + Token.objects.get_or_create(user_id=instance.pk) + + +@receiver(post_save, sender=User) +def default_permissions_post_save(sender, instance, created, raw, **kwargs): + """ + Users must have both model-level and object-level permissions to satisfy + DRF, so assign the newly-created user all available collection and asset + permissions at the model level + """ + if raw: + # `raw` means we can't touch (so make sure your fixtures include + # all necessary permissions!) + return + if not created: + # We should only grant default permissions when the user is first + # created + return + grant_default_model_level_perms(instance) + + +@receiver(post_save, sender=User) +def save_kobocat_user(sender, instance, created, raw, **kwargs): + """ + Sync auth_user table between KPI and KC, and, if the user is newly created, + grant all KoboCAT model-level permissions for the content types listed in + `settings.KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES` + """ + + if not settings.TESTING: + with kc_transaction_atomic(): + instance.sync_to_openrosa_db() + if created: + grant_kc_model_level_perms(instance) + UserProfile.objects.get_or_create(user=instance) diff --git a/kobo/apps/openrosa/apps/api/viewsets/xform_submission_api.py b/kobo/apps/openrosa/apps/api/viewsets/xform_submission_api.py index dc8b816d21..5ca73aaa58 100644 --- a/kobo/apps/openrosa/apps/api/viewsets/xform_submission_api.py +++ b/kobo/apps/openrosa/apps/api/viewsets/xform_submission_api.py @@ -26,6 +26,7 @@ safe_create_instance, UnauthenticatedEditAttempt, ) +from kobo.apps.openrosa.libs.utils.string import dict_lists2strings from kpi.authentication import DigestAuthentication from kpi.utils.object_permission import get_database_user from ..utils.rest_framework.viewsets import OpenRosaGenericViewSet @@ -37,25 +38,11 @@ def is_json(request): return 'application/json' in request.content_type.lower() -def dict_lists2strings(d): - """Convert lists in a dict to joined strings. - - :param d: The dict to convert. - :returns: The converted dict.""" - for k, v in d.items(): - if isinstance(v, list) and all([isinstance(e, str) for e in v]): - d[k] = ' '.join(v) - elif isinstance(v, dict): - d[k] = dict_lists2strings(v) - - return d - - def create_instance_from_xml(username, request): xml_file_list = request.FILES.pop('xml_submission_file', []) xml_file = xml_file_list[0] if len(xml_file_list) else None media_files = request.FILES.values() - return safe_create_instance(username, xml_file, media_files, None, request) + return safe_create_instance(username, xml_file, media_files, None, request=request) def create_instance_from_json(username, request): @@ -73,7 +60,7 @@ def create_instance_from_json(username, request): xml_string = dict2xform(submission_joined, dict_form.get('id')) xml_file = io.StringIO(xml_string) - return safe_create_instance(username, xml_file, [], None, request) + return safe_create_instance(username, xml_file, [], None, request=request) class XFormSubmissionApi( diff --git a/kobo/apps/openrosa/apps/logger/models/instance.py b/kobo/apps/openrosa/apps/logger/models/instance.py index 7b685d261b..79a99d72b9 100644 --- a/kobo/apps/openrosa/apps/logger/models/instance.py +++ b/kobo/apps/openrosa/apps/logger/models/instance.py @@ -6,6 +6,7 @@ from backports.zoneinfo import ZoneInfo import reversion +from django.apps import apps from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point from django.utils import timezone @@ -21,8 +22,11 @@ from kobo.apps.openrosa.apps.logger.fields import LazyDefaultBooleanField from kobo.apps.openrosa.apps.logger.models.survey_type import SurveyType from kobo.apps.openrosa.apps.logger.models.xform import XForm -from kobo.apps.openrosa.apps.logger.xform_instance_parser import XFormInstanceParser, \ - clean_and_parse_xml, get_uuid_from_xml +from kobo.apps.openrosa.apps.logger.xform_instance_parser import ( + XFormInstanceParser, + clean_and_parse_xml, + get_uuid_from_xml, +) from kobo.apps.openrosa.libs.utils.common_tags import ( ATTACHMENTS, GEOLOCATION, @@ -129,12 +133,15 @@ def check_active(self, force): return if self.xform and not self.xform.downloadable: raise FormInactiveError() - try: - profile = self.xform.user.profile - except self.xform.user.profile.RelatedObjectDoesNotExist: - return - if profile.metadata.get('submissions_suspended', False): - raise TemporarilyUnavailableError() + + # FIXME Access `self.xform.user.profile` directly could raise a + # `RelatedObjectDoesNotExist` error if profile does not exist even if + # wrapped in try/except + UserProfile = apps.get_model('main', 'UserProfile') # noqa - Avoid circular imports + if profile := UserProfile.objects.filter(user=self.xform.user).first(): + if profile.metadata.get('submissions_suspended', False): + raise TemporarilyUnavailableError() + return def _set_geom(self): xform = self.xform diff --git a/kobo/apps/openrosa/apps/logger/models/xform.py b/kobo/apps/openrosa/apps/logger/models/xform.py index 9a53b64b0c..4c376d8c18 100644 --- a/kobo/apps/openrosa/apps/logger/models/xform.py +++ b/kobo/apps/openrosa/apps/logger/models/xform.py @@ -28,6 +28,7 @@ from kpi.deployment_backends.kc_access.storage import ( default_kobocat_storage as default_storage, ) +from kpi.fields.file import ExtendedFileField from kpi.utils.xml import XMLFormWithDisclaimer XFORM_TITLE_LENGTH = 255 @@ -53,7 +54,7 @@ class XForm(models.Model): CLONED_SUFFIX = '_cloned' MAX_ID_LENGTH = 100 - xls = models.FileField( + xls = ExtendedFileField( storage=default_storage, upload_to=upload_to, null=True ) json = models.TextField(default='') diff --git a/kobo/apps/openrosa/apps/logger/signals.py b/kobo/apps/openrosa/apps/logger/signals.py index 760e407205..4e72c598a6 100644 --- a/kobo/apps/openrosa/apps/logger/signals.py +++ b/kobo/apps/openrosa/apps/logger/signals.py @@ -185,7 +185,7 @@ def update_xform_submission_count(sender, instance, created, **kwargs): last_submission_time=instance.date_created, ) # Hack to avoid circular imports - UserProfile = User.profile.related.related_model + UserProfile = User.profile.related.related_model # noqa profile, created = UserProfile.objects.only('pk').get_or_create( user_id=xform.user_id ) diff --git a/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py b/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py index 1630ea1a28..a400b9c7ee 100644 --- a/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py +++ b/kobo/apps/openrosa/apps/logger/tests/test_simple_submission.py @@ -120,7 +120,8 @@ def test_corrupted_submission(self): request = RequestFactory().post('/') request.user = self.user error, instance = safe_create_instance( - self.user.username, TempFileProxy(xml), None, None, request) + self.user.username, TempFileProxy(xml), None, None, request=request + ) # No `DjangoUnicodeDecodeError` errors are raised anymore. # An `ExpatError` is raised instead text = 'Improperly formatted XML' diff --git a/kobo/apps/openrosa/apps/logger/utils/instance.py b/kobo/apps/openrosa/apps/logger/utils/instance.py index f54d87b0d4..f87c71612f 100644 --- a/kobo/apps/openrosa/apps/logger/utils/instance.py +++ b/kobo/apps/openrosa/apps/logger/utils/instance.py @@ -118,23 +118,22 @@ def set_instance_validation_statuses( xform: XForm, request_data: dict, request_username: str ) -> int: - try: - new_validation_status_uid = request_data['validation_status.uid'] - except KeyError: - raise MissingValidationStatusPayloadError - - # Create new validation_status object - new_validation_status = get_validation_status( - new_validation_status_uid, request_username - ) + try: + new_validation_status_uid = request_data['validation_status.uid'] + except KeyError: + raise MissingValidationStatusPayloadError - postgres_query, mongo_query = build_db_queries(xform, request_data) + # Create new validation_status object + new_validation_status = get_validation_status( + new_validation_status_uid, request_username + ) + postgres_query, mongo_query = build_db_queries(xform, request_data) - # Update Postgres & Mongo - updated_records_count = Instance.objects.filter( - **postgres_query - ).update(validation_status=new_validation_status) - ParsedInstance.bulk_update_validation_statuses( - mongo_query, new_validation_status - ) - return updated_records_count + # Update Postgres & Mongo + updated_records_count = Instance.objects.filter( + **postgres_query + ).update(validation_status=new_validation_status) + ParsedInstance.bulk_update_validation_statuses( + mongo_query, new_validation_status + ) + return updated_records_count diff --git a/kobo/apps/openrosa/apps/logger/xform_instance_parser.py b/kobo/apps/openrosa/apps/logger/xform_instance_parser.py index ecfeda23b0..bd1e6d0077 100644 --- a/kobo/apps/openrosa/apps/logger/xform_instance_parser.py +++ b/kobo/apps/openrosa/apps/logger/xform_instance_parser.py @@ -1,8 +1,11 @@ -# coding: utf-8 +from __future__ import annotations + import logging import re import sys +from datetime import datetime from xml.dom import Node +from typing import Optional import dateutil.parser import six @@ -71,6 +74,7 @@ def get_meta_from_xml(xml_str, meta_name): def get_uuid_from_xml(xml): + def _uuid_only(uuid, regex): matches = regex.match(uuid) if matches and len(matches.groups()) > 0: @@ -94,7 +98,7 @@ def _uuid_only(uuid, regex): return None -def get_submission_date_from_xml(xml): +def get_submission_date_from_xml(xml) -> Optional[datetime]: # check in survey_node attributes xml = clean_and_parse_xml(xml) children = xml.childNodes @@ -103,9 +107,9 @@ def get_submission_date_from_xml(xml): if children.length == 0: raise ValueError(t("XML string must have a survey element.")) survey_node = children[0] - submissionDate = survey_node.getAttribute('submissionDate') - if submissionDate != '': - return dateutil.parser.parse(submissionDate) + submission_date = survey_node.getAttribute('submissionDate') + if submission_date != '': + return dateutil.parser.parse(submission_date) return None diff --git a/kobo/apps/openrosa/apps/main/models/meta_data.py b/kobo/apps/openrosa/apps/main/models/meta_data.py index be49726e3c..aaf7545e7d 100644 --- a/kobo/apps/openrosa/apps/main/models/meta_data.py +++ b/kobo/apps/openrosa/apps/main/models/meta_data.py @@ -19,6 +19,7 @@ from kpi.deployment_backends.kc_access.storage import ( default_kobocat_storage as default_storage, ) +from kpi.fields.file import ExtendedFileField CHUNK_SIZE = 1024 @@ -136,7 +137,7 @@ class MetaData(models.Model): xform = models.ForeignKey(XForm, on_delete=models.CASCADE) data_type = models.CharField(max_length=255) data_value = models.CharField(max_length=255) - data_file = models.FileField( + data_file = ExtendedFileField( storage=default_storage, upload_to=upload_to, blank=True, diff --git a/kobo/apps/openrosa/libs/utils/logger_tools.py b/kobo/apps/openrosa/libs/utils/logger_tools.py index 0950733991..3fd880394e 100644 --- a/kobo/apps/openrosa/libs/utils/logger_tools.py +++ b/kobo/apps/openrosa/libs/utils/logger_tools.py @@ -7,7 +7,7 @@ import sys import traceback from datetime import date, datetime, timezone -from typing import Generator, Optional +from typing import Generator, Optional, Union from xml.etree import ElementTree as ET from xml.parsers.expat import ExpatError try: @@ -110,6 +110,7 @@ def check_submission_permissions( :returns: None. :raises: PermissionDenied based on the above criteria. """ + if not xform.require_auth: # Anonymous submissions are allowed! return @@ -198,8 +199,15 @@ def create_instance( existing_instance.parsed_instance.save(asynchronous=False) return existing_instance else: - instance = save_submission(request, xform, xml, media_files, new_uuid, - status, date_created_override) + instance = save_submission( + request, + xform, + xml, + media_files, + new_uuid, + status, + date_created_override, + ) return instance @@ -211,12 +219,14 @@ def disposition_ext_and_date(name, extension, show_date=True): return 'attachment; filename=%s.%s' % (name, extension) -def dict2xform(jsform, form_id): - dd = {'form_id': form_id} - xml_head = "\n<%(form_id)s id='%(form_id)s'>\n" % dd - xml_tail = "\n" % dd +def dict2xform(submission: dict, xform_id_string: str) -> str: + xml_head = ( + f'\n' + f' <{xform_id_string} id="{xform_id_string}">\n' + ) + xml_tail = f'\n\n' - return xml_head + dict2xml(jsform) + xml_tail + return xml_head + dict2xml(submission) + xml_tail def get_instance_or_404(**criteria): @@ -456,6 +466,7 @@ def publish_xls_form(xls_file, user, id_string=None): with transaction.atomic(): dd = DataDictionary.objects.create(user=user, xls=xls_file) except IntegrityError as e: + breakpoint() raise e return dd @@ -528,10 +539,11 @@ def response_with_mimetype_and_name( def safe_create_instance( - username, - xml_file, - media_files, + username: str, + xml_file: File, + media_files: Union[list, Generator[File]], uuid: Optional[str] = None, + date_created_override: Optional[datetime] = None, request: Optional['rest_framework.request.Request'] = None, ): """Create an instance and catch exceptions. @@ -543,7 +555,12 @@ def safe_create_instance( try: instance = create_instance( - username, xml_file, media_files, uuid=uuid, request=request + username, + xml_file, + media_files, + uuid=uuid, + date_created_override=date_created_override, + request=request, ) except InstanceInvalidUserError: error = OpenRosaResponseBadRequest(t("Username or ID required.")) @@ -630,7 +647,7 @@ def save_submission( request: 'rest_framework.request.Request', xform: XForm, xml: str, - media_files: Generator[File], + media_files: Union[list, Generator[File]], new_uuid: str, status: str, date_created_override: datetime, @@ -666,13 +683,15 @@ def save_submission( if not dj_timezone.is_aware(date_created_override): # default to utc? date_created_override = dj_timezone.make_aware( - date_created_override, timezone.utc) + date_created_override, timezone.utc + ) instance.date_created = date_created_override - instance.save() + instance.save(update_fields=['date_created']) if instance.xform is not None: pi, created = ParsedInstance.objects.get_or_create( - instance=instance) + instance=instance + ) if not created: pi.save(asynchronous=False) @@ -820,7 +839,12 @@ def _has_edit_xform_permission( if request.user.is_superuser: return True - return request.user.has_perm('logger.change_xform', xform) + if request.user.has_perm('logger.change_xform', xform): + return True + + # User's permissions have been already checked when calling KPI endpoint + # If `has_partial_perms` is True, user is allowed to perform the action. + return getattr(request.user, 'has_partial_perms', False) return False diff --git a/kobo/apps/openrosa/libs/utils/string.py b/kobo/apps/openrosa/libs/utils/string.py index 60813cfc06..cb5047ad7c 100644 --- a/kobo/apps/openrosa/libs/utils/string.py +++ b/kobo/apps/openrosa/libs/utils/string.py @@ -13,6 +13,23 @@ def base64_decodestring(obj): return base64.b64decode(obj).decode() +def dict_lists2strings(d: dict) -> dict: + """ + Convert lists in a dict to joined strings. + + :param d: The dict to convert. + :returns: The converted dict. + """ + + for k, v in d.items(): + if isinstance(v, list) and all([isinstance(e, str) for e in v]): + d[k] = ' '.join(v) + elif isinstance(v, dict): + d[k] = dict_lists2strings(v) + + return d + + def str2bool(v): return v.lower() in ( 'yes', 'true', 't', '1') if isinstance(v, str) else v diff --git a/kobo/apps/project_ownership/tests/api/v2/test_api.py b/kobo/apps/project_ownership/tests/api/v2/test_api.py index 92a531e470..3420843798 100644 --- a/kobo/apps/project_ownership/tests/api/v2/test_api.py +++ b/kobo/apps/project_ownership/tests/api/v2/test_api.py @@ -333,14 +333,12 @@ def __add_submissions(self): 'formhub/uuid': self.asset.uid, '_attachments': [ { - 'id': 1, 'download_url': 'http://testserver/someuser/audio_conversion_test_clip.3gp', 'filename': 'someuser/audio_conversion_test_clip.3gp', 'mimetype': 'video/3gpp', 'bytes': 5000, }, { - 'id': 2, 'download_url': 'http://testserver/someuser/audio_conversion_test_image.jpg', 'filename': 'someuser/audio_conversion_test_image.jpg', 'mimetype': 'image/jpeg', @@ -353,14 +351,6 @@ def __add_submissions(self): self.asset.deployment.mock_submissions(submissions) self.submissions = submissions - @patch( - 'kpi.serializers.v2.service_usage.ServiceUsageSerializer._get_storage_usage', - new=MockServiceUsageSerializer._get_storage_usage - ) - @patch( - 'kpi.serializers.v2.service_usage.ServiceUsageSerializer._get_submission_counters', - new=MockServiceUsageSerializer._get_submission_counters - ) @patch( 'kobo.apps.project_ownership.models.transfer.reset_kc_permissions', MagicMock() @@ -385,7 +375,7 @@ def test_account_usage_transferred_to_new_user(self): 'asr_seconds_all_time': 120, 'mt_characters_all_time': 1000, }, - 'total_storage_bytes': 15000, + 'total_storage_bytes': 191642, 'total_submission_count': { 'all_time': 1, 'current_year': 1, diff --git a/kobo/apps/stripe/tests/test_organization_usage.py b/kobo/apps/stripe/tests/test_organization_usage.py index f9764c98e5..5cc13d1e52 100644 --- a/kobo/apps/stripe/tests/test_organization_usage.py +++ b/kobo/apps/stripe/tests/test_organization_usage.py @@ -1,4 +1,5 @@ import timeit +import itertools import pytest from django.core.cache import cache @@ -10,14 +11,19 @@ from kobo.apps.kobo_auth.shortcuts import User from kobo.apps.organizations.models import Organization, OrganizationUser -from kobo.apps.stripe.tests.utils import generate_enterprise_subscription, generate_plan_subscription -from kobo.apps.trackers.submission_utils import create_mock_assets, add_mock_submissions +from kobo.apps.stripe.tests.utils import ( + generate_enterprise_subscription, + generate_plan_subscription, +) +from kobo.apps.trackers.submission_utils import ( + create_mock_assets, + add_mock_submissions, +) from kpi.tests.api.v2.test_api_service_usage import ServiceUsageAPIBase from kpi.tests.api.v2.test_api_asset_usage import AssetUsageAPITestCase from rest_framework import status - class OrganizationServiceUsageAPITestCase(ServiceUsageAPIBase): """ Test organization service usage when Stripe is enabled. @@ -26,9 +32,10 @@ class OrganizationServiceUsageAPITestCase(ServiceUsageAPIBase): when Stripe is installed. """ - user_count = 5 - assets_per_user = 5 - submissions_per_asset = 5 + names = ['alice', 'bob'] + user_count = len(names) + assets_per_user = 2 + submissions_per_asset = 2 org_id = 'orgAKWMFskafsngf' @classmethod @@ -40,7 +47,12 @@ def setUpTestData(cls): cls.organization.add_user(cls.anotheruser, is_admin=True) assets = create_mock_assets([cls.anotheruser], cls.assets_per_user) - users = baker.make(User, _quantity=cls.user_count - 1, _bulk_create=True) + users = baker.make( + User, + username=itertools.cycle(cls.names), + _quantity=cls.user_count - 1, + _bulk_create=True, + ) baker.make( OrganizationUser, user=users.__iter__(), diff --git a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py index 1e951ff389..1502fcccf5 100644 --- a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py +++ b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py @@ -1,3 +1,4 @@ +import uuid from copy import deepcopy from unittest.mock import patch @@ -31,7 +32,10 @@ class ValidateSubmissionTest(APITestCase): def setUp(self): user = User.objects.create_user(username='someuser', email='user@example.com') self.asset = Asset( - owner=user, content={'survey': [{'type': 'audio', 'name': 'q1'}]} + owner=user, + content={ + 'survey': [{'type': 'audio', 'label': 'q1', 'name': 'q1'}] + }, ) self.asset.advanced_features = {} self.asset.save() @@ -388,7 +392,7 @@ class GoogleTranscriptionSubmissionTest(APITestCase): def setUp(self): self.user = User.objects.create_user(username='someuser', email='user@example.com') self.asset = Asset( - content={'survey': [{'type': 'audio', 'label': 'q1'}]} + content={'survey': [{'type': 'audio', 'label': 'q1', 'name': 'q1'}]} ) self.asset.advanced_features = {'transcript': {'values': ['q1']}} self.asset.owner = self.user @@ -418,7 +422,6 @@ def test_google_transcript_post(self, m1, m2): '_uuid': submission_id, '_attachments': [ { - 'id': 1, 'filename': 'someuser/audio_conversion_test_clip.3gp', 'mimetype': 'video/3gpp', }, @@ -431,10 +434,10 @@ def test_google_transcript_post(self, m1, m2): 'submission': submission_id, 'q1': {GOOGLETS: {'status': 'requested', 'languageCode': ''}} } - with self.assertNumQueries(FuzzyInt(210, 215)): + with self.assertNumQueries(FuzzyInt(55, 65)): res = self.client.post(url, data, format='json') self.assertContains(res, 'complete') - with self.assertNumQueries(FuzzyInt(20, 26)): + with self.assertNumQueries(FuzzyInt(25, 35)): self.client.post(url, data, format='json') @override_settings(CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}) diff --git a/kobo/apps/subsequences/tests/test_submission_stream.py b/kobo/apps/subsequences/tests/test_submission_stream.py index 4542d603e1..dc6f770873 100644 --- a/kobo/apps/subsequences/tests/test_submission_stream.py +++ b/kobo/apps/subsequences/tests/test_submission_stream.py @@ -85,6 +85,7 @@ def _create_asset(self): }, ) self.asset.deploy(backend='mock', active=True) + self.asset.save() def _create_mock_submissions(self): self.asset.deployment.mock_submissions( @@ -95,8 +96,6 @@ def _create_mock_submissions(self): 'meta/instanceID': ( 'uuid:1c05898e-b43c-491d-814c-79595eb84e81' ), - # `MockDeploymentBackend` should probably add `_uuid`, but - # it doesn't. It's going away soon enough, though. '_uuid': '1c05898e-b43c-491d-814c-79595eb84e81', }, ] @@ -283,5 +282,5 @@ def test_stream_with_extras_handles_duplicated_submission_uuids(self): for v in qual_response['val']: assert isinstance(v['uuid'], str) - # Clear all mocked submissions to avoid duplicate submission errors - self.asset.deployment.mock_submissions([]) + ## Clear all mocked submissions to avoid duplicate submission errors + #self.asset.deployment.mock_submissions([]) diff --git a/kobo/apps/subsequences/utils/__init__.py b/kobo/apps/subsequences/utils/__init__.py index bddf773004..cb1aecd456 100644 --- a/kobo/apps/subsequences/utils/__init__.py +++ b/kobo/apps/subsequences/utils/__init__.py @@ -1,11 +1,10 @@ from collections import defaultdict from copy import deepcopy + from ..actions.automatic_transcription import AutomaticTranscriptionAction from ..actions.translation import TranslationAction from ..actions.qual import QualAction -from ..actions.unknown_action import UnknownAction - AVAILABLE_ACTIONS = ( AutomaticTranscriptionAction, @@ -42,14 +41,16 @@ # if not action.test_submission_passes_action(submission): # return action + def advanced_feature_instances(content, actions): action_instances = [] for action_id, action_params in actions.items(): action_kls = ACTIONS_BY_ID[action_id] - if action_params == True: + if action_params is True: action_params = action_kls.build_params({}, content) yield action_kls(action_params) + def populate_paths(_content): content = deepcopy(_content) group_stack = [] @@ -76,6 +77,7 @@ def populate_paths(_content): row['qpath'] = '-'.join([*group_stack, rowname]) return content + def advanced_submission_jsonschema(content, actions, url=None): actions = deepcopy(actions) action_instances = [] @@ -90,7 +92,7 @@ def advanced_submission_jsonschema(content, actions, url=None): for action_id, action_params in actions.items(): action_kls = ACTIONS_BY_ID[action_id] - if action_params == True: + if action_params is True: action_params = action_kls.build_params({}, content) if 'values' not in action_params: action_params['values'] = action_kls.get_values_for_content(content) @@ -100,26 +102,32 @@ def advanced_submission_jsonschema(content, actions, url=None): # def _empty_obj(): # return {'type': 'object', 'properties': {}, 'additionalProperties': False} + def get_jsonschema(action_instances=(), url=None): sub_props = {} if url is None: url = '/advanced_submission_post/' - schema = {'type': 'object', - '$description': FEATURE_JSONSCHEMA_DESCRIPTION, - 'url': url, - 'properties': { - 'submission': {'type': 'string', - 'description': 'the uuid of the submission'}, - }, - 'additionalProperties': False, - 'required': ['submission'], - } + schema = { + 'type': 'object', + '$description': FEATURE_JSONSCHEMA_DESCRIPTION, + 'url': url, + 'properties': { + 'submission': { + 'type': 'string', + 'description': 'the uuid of the submission', + }, + }, + 'additionalProperties': False, + 'required': ['submission'], + } for instance in action_instances: schema = instance.modify_jsonschema(schema) return schema + SUPPLEMENTAL_DETAILS_KEY = '_supplementalDetails' + def stream_with_extras(submission_stream, asset): extras = dict( asset.submission_extras.values_list('submission_uuid', 'content') @@ -145,11 +153,14 @@ def stream_with_extras(submission_stream, asset): c['uuid']: c for c in choices } qual_questions_by_uuid[qual_q['uuid']] = qual_q + for submission in submission_stream: if SUBMISSION_UUID_FIELD in submission: uuid = submission[SUBMISSION_UUID_FIELD] else: uuid = submission['_uuid'] + + all_supplemental_details = deepcopy(extras.get(uuid, {})) for qpath, supplemental_details in all_supplemental_details.items(): try: diff --git a/kobo/apps/trackers/submission_utils.py b/kobo/apps/trackers/submission_utils.py index a47118a28e..18702cf842 100644 --- a/kobo/apps/trackers/submission_utils.py +++ b/kobo/apps/trackers/submission_utils.py @@ -1,4 +1,6 @@ +import itertools import os +import time import uuid from django.conf import settings @@ -31,17 +33,29 @@ def create_mock_assets(users: list, assets_per_user: int = 1): ] } assets = [] - for user in users: + + def _get_uid(count): + uids = [] + for i in range(count): + _, random = str(time.time()).split('.') + uids.append(f'a{random}_{i}') + return uids + + for idx, user in enumerate(users): assets = assets + baker.make( Asset, content=content_source_asset, owner=user, asset_type='survey', name='test', + uid=itertools.cycle(_get_uid(assets_per_user)), _quantity=assets_per_user, ) + print([a.uid for a in assets]) + breakpoint() for asset in assets: + print('DEPLOYING ', asset.uid, flush=True) asset.deploy(backend='mock', active=True) asset.deployment.set_namespace(ROUTER_URL_NAMESPACE) asset.save() # might be redundant? @@ -60,70 +74,6 @@ def expected_file_size(submissions: int = 1): )) * submissions -def update_xform_counters( - asset: Asset, xform: XForm = None, submissions: int = 1 -): - """ - Create/update the daily submission counter and the shadow xform we use to query it - """ - today = timezone.now() - if xform: - xform.attachment_storage_bytes += ( - expected_file_size(submissions) - ) - xform.save() - else: - xform_xml = ( - f'' - f'' - f'' - f' XForm test' - f' ' - f' ' - f' <{asset.uid} id="{asset.uid}" />' - f' ' - f' ' - f'' - f'' - f'' - f'' - ) - - xform = baker.make( - 'logger.XForm', - attachment_storage_bytes=( - expected_file_size(submissions) - ), - kpi_asset_uid=asset.uid, - date_created=today, - date_modified=today, - user_id=asset.owner_id, - xml=xform_xml, - json={} - ) - xform.save() - - counter = DailyXFormSubmissionCounter.objects.filter( - date=today.date(), - user_id=asset.owner.id, - ).first() - - if counter: - counter.counter += submissions - counter.save() - else: - counter = ( - baker.make( - 'logger.DailyXFormSubmissionCounter', - date=today.date(), - counter=submissions, - xform=xform, - user_id=asset.owner_id, - ) - ) - counter.save() - - def add_mock_submissions(assets: list, submissions_per_asset: int = 1): """ Add one (default) or more submissions to an asset @@ -158,6 +108,6 @@ def add_mock_submissions(assets: list, submissions_per_asset: int = 1): asset.deployment.mock_submissions(asset_submissions, flush_db=False) all_submissions = all_submissions + asset_submissions - update_xform_counters(asset, submissions=submissions_per_asset) + # update_xform_counters(asset, submissions=submissions_per_asset) return all_submissions diff --git a/kobo/apps/trackers/tests/test_trackers.py b/kobo/apps/trackers/tests/test_trackers.py index 7209416f1d..083ec16b7c 100644 --- a/kobo/apps/trackers/tests/test_trackers.py +++ b/kobo/apps/trackers/tests/test_trackers.py @@ -23,7 +23,7 @@ def setUp(self): def _create_asset(self): asset = Asset.objects.create( - content={'survey': [{"type": "text", "name": "q1"}]}, + content={'survey': [{'type': 'text', 'label': 'q1', 'name': 'q1'}]}, owner=self.user, asset_type='survey', name='тєѕт αѕѕєт', diff --git a/kobo/settings/base.py b/kobo/settings/base.py index 30dc1cbb06..812c5de440 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -1615,12 +1615,6 @@ def dj_stripe_request_callback_method(): # Django 3.2 required settings DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -SERVICE_ACCOUNT = { - 'BACKEND': env.cache_url( - 'SERVICE_ACCOUNT_BACKEND_URL', default='redis://redis_cache:6380/6' - ), - 'WHITELISTED_HOSTS': env.list('SERVICE_ACCOUNT_WHITELISTED_HOSTS', default=[]), -} AUTH_PASSWORD_VALIDATORS = [ { diff --git a/kobo/settings/testing.py b/kobo/settings/testing.py index 4ae7e54ea8..421adcaeb6 100644 --- a/kobo/settings/testing.py +++ b/kobo/settings/testing.py @@ -47,10 +47,8 @@ 'LOADER_CLASS' ] = 'webpack_loader.loader.FakeWebpackLoader' -# Kobocat settings +# KoboCAT settings TEST_HTTP_HOST = 'testserver' TEST_USERNAME = 'bob' -SERVICE_ACCOUNT['WHITELISTED_HOSTS'] = ['testserver'] -SERVICE_ACCOUNT['NAMESPACE'] = 'kobo-service-account-test' OPENROSA_DB_ALIAS = DEFAULT_DB_ALIAS diff --git a/kpi/deployment_backends/base_backend.py b/kpi/deployment_backends/base_backend.py index f7860fdf71..b26dd84fdf 100644 --- a/kpi/deployment_backends/base_backend.py +++ b/kpi/deployment_backends/base_backend.py @@ -117,6 +117,11 @@ def bulk_update_submissions( # Reset query, because all the submission ids have been already # retrieve data['query'] = {} + + # Set `has_partial_perms` flag on `request.user` to grant them + # permissions while calling `logger_tool.py::_has_edit_xform_permission()` + if request := kwargs.get('request'): + request.user.has_partial_perms = True else: submission_ids = data['submission_ids'] diff --git a/kpi/deployment_backends/mock_backend.py b/kpi/deployment_backends/mock_backend.py index 296e8fbb25..702ce886c4 100644 --- a/kpi/deployment_backends/mock_backend.py +++ b/kpi/deployment_backends/mock_backend.py @@ -1,365 +1,31 @@ -# coding: utf-8 from __future__ import annotations -import copy +import io import os -import time -import uuid -from collections import defaultdict -from contextlib import contextmanager -from datetime import date, datetime -from typing import Optional, Union -try: - from zoneinfo import ZoneInfo -except ImportError: - from backports.zoneinfo import ZoneInfo +from typing import Optional +from uuid import uuid4 -from deepmerge import always_merger -from dict2xml import dict2xml as dict2xml_real -from django.db.models import Q from django.conf import settings -from django.core.files.base import ContentFile -from django.db.models import Sum -from django.db.models.functions import Coalesce -from django.urls import reverse -from rest_framework import status +from django.contrib.auth.models import AnonymousUser +from django.utils.dateparse import parse_datetime -from kobo.apps.openrosa.apps.logger.models import Attachment, Instance, XForm -from kobo.apps.openrosa.apps.logger.models.attachment import upload_to -from kobo.apps.openrosa.apps.main.models import UserProfile -from kobo.apps.trackers.models import NLPUsageCounter -from kpi.constants import ( - SUBMISSION_FORMAT_TYPE_JSON, - SUBMISSION_FORMAT_TYPE_XML, - PERM_CHANGE_SUBMISSIONS, - PERM_DELETE_SUBMISSIONS, - PERM_VALIDATE_SUBMISSIONS, +from kobo.apps.kobo_auth.shortcuts import User +from kobo.apps.openrosa.libs.utils.logger_tools import ( + dict2xform, + safe_create_instance, ) -from kpi.exceptions import ( - AttachmentNotFoundException, - InvalidXPathException, - SubmissionNotFoundException, - XPathNotFoundException, -) -from kpi.interfaces.sync_backend_media import SyncBackendMediaInterface -from kpi.models.asset_file import AssetFile -from kpi.utils.mongo_helper import MongoHelper, drop_mock_only -from kpi.utils.xml import fromstring_preserve_root_xmlns -from .base_backend import BaseDeploymentBackend - +from kpi.constants import PERM_ADD_SUBMISSIONS, SUBMISSION_FORMAT_TYPE_JSON +from kpi.tests.utils.dicts import nested_dict_from_keys +from .openrosa_backend import OpenRosaDeploymentBackend +from ..utils.files import ExtendedContentFile -def dict2xml(*args, **kwargs): - """To facilitate mocking in unit tests""" - return dict2xml_real(*args, **kwargs) - -class MockDeploymentBackend(BaseDeploymentBackend): - """ - Only used for unit testing and interface testing. - """ - - @property - def attachment_storage_bytes(self): - submissions = self.get_submissions(self.asset.owner) - storage_bytes = 0 - for submission in submissions: - attachments = self.get_attachment_objects_from_dict(submission) - storage_bytes += sum( - [attachment.media_file_size for attachment in attachments] - ) - return storage_bytes - - def bulk_assign_mapped_perms(self): - pass - - def calculated_submission_count( - self, user: settings.AUTH_USER_MODEL, **kwargs - ) -> int: - params = self.validate_submission_list_params( - user, validate_count=True, **kwargs - ) - return MongoHelper.get_count(self.mongo_userform_id, **params) - - def connect(self, active=False): - def generate_uuid_for_form(): - # From KoboCAT's onadata.libs.utils.model_tools - return uuid.uuid4().hex - - self.store_data( - { - 'backend': 'mock', - 'active': active, - 'backend_response': { - 'downloadable': active, - 'kpi_asset_uid': self.asset.uid, - 'uuid': generate_uuid_for_form(), - # TODO use XForm object and get its primary key - 'formid': self.asset.pk - }, - 'version': self.asset.version_id, - } - ) - - @property - def form_uuid(self): - return 'formhub-uuid' # to match existing tests - - def nlp_tracking_data(asset_ids=None): - """ - Get the NLP tracking data since a specified date - If no date is provided, get all-time data - """ - filter_args = {} - if start_date: - filter_args = {'date__gte': start_date} - try: - nlp_tracking = ( - NLPUsageCounter.objects.only( - 'total_asr_seconds', 'total_mt_characters' - ) - .filter(asset_id=self.asset.id, **filter_args) - .aggregate( - total_nlp_asr_seconds=Coalesce(Sum('total_asr_seconds'), 0), - total_nlp_mt_characters=Coalesce( - Sum('total_mt_characters'), 0 - ), - ) - ) - except NLPUsageCounter.DoesNotExist: - return { - 'total_nlp_asr_seconds': 0, - 'total_nlp_mt_characters': 0, - } - else: - return nlp_tracking - - def submission_count_since_date(self, start_date=None): - # FIXME, does not reproduce KoBoCAT behaviour. - # Deleted submissions are not taken into account but they should be - monthly_counter = len(self.get_submissions(self.asset.owner)) - return monthly_counter - - @drop_mock_only - def delete_submission( - self, submission_id: int, user: settings.AUTH_USER_MODEL - ) -> dict: - """ - Delete a submission - """ - self.validate_access_with_partial_perms( - user=user, - perm=PERM_DELETE_SUBMISSIONS, - submission_ids=[submission_id], - ) - - if not settings.MONGO_DB.instances.find_one({'_id': submission_id}): - return { - 'content_type': 'application/json', - 'status': status.HTTP_404_NOT_FOUND, - 'data': {'detail': 'Not found'}, - } - - settings.MONGO_DB.instances.delete_one({'_id': submission_id}) - - return { - 'content_type': 'application/json', - 'status': status.HTTP_204_NO_CONTENT, - } - - def delete_submissions( - self, data: dict, user: settings.AUTH_USER_MODEL, **kwargs - ) -> dict: - """ - Bulk delete provided submissions authenticated by `user`'s API token. - - `data` should contains the submission ids or the query to get the subset - of submissions to delete - Example: - {"submission_ids": [1, 2, 3]} - or - {"query": {"Question": "response"} - """ - submission_ids = self.validate_access_with_partial_perms( - user=user, - perm=PERM_DELETE_SUBMISSIONS, - submission_ids=data['submission_ids'], - query=data['query'], - ) - - if not submission_ids: - submission_ids = data['submission_ids'] - else: - data['query'] = {} - - # Retrieve the subset of submissions to delete - submissions = self.get_submissions( - user, submission_ids=submission_ids, query=data['query'] - ) - - # If no submissions have been fetched, user is not allowed to perform - # the request - if not submissions: - return { - 'content_type': 'application/json', - 'status': status.HTTP_404_NOT_FOUND, - } - - # We could use `delete_many()` but we would have to recreate the query - # with submission ids or query. - for submission in submissions: - submission_id = submission['_id'] - settings.MONGO_DB.instances.delete_one({'_id': submission_id}) - - return { - 'content_type': 'application/json', - 'status': status.HTTP_200_OK, - } - - def duplicate_submission( - self, submission_id: int, request: 'rest_framework.request.Request', - ) -> dict: - # TODO: Make this operate on XML somehow and reuse code from - # KobocatDeploymentBackend, to catch issues like #3054 - user = request.user - self.validate_access_with_partial_perms( - user=user, - perm=PERM_CHANGE_SUBMISSIONS, - submission_ids=[submission_id], - ) - - submission = self.get_submission(submission_id, user=user) - _attachments = submission.get('_attachments', []) - dup_att = [] - if _attachments: - # not exactly emulating database id incrementing but probably good - # enough for the mock tests - max_attachment_id = max(a['id'] for a in _attachments) - for i, att in enumerate(_attachments, 1): - dup_att.append({**att, 'id': max_attachment_id + i}) - - duplicated_submission = copy.deepcopy(submission) - updated_time = datetime.now(tz=ZoneInfo('UTC')).isoformat( - 'T', 'milliseconds' - ) - next_id = ( - max( - ( - sub['_id'] - for sub in self.get_submissions( - self.asset.owner, fields=['_id'] - ) - ) - ) - + 1 - ) - duplicated_submission.update( - { - '_id': next_id, - 'start': updated_time, - 'end': updated_time, - self.SUBMISSION_CURRENT_UUID_XPATH: f'uuid:{uuid.uuid4()}', - self.SUBMISSION_DEPRECATED_UUID_XPATH: submission[ - self.SUBMISSION_CURRENT_UUID_XPATH - ], - '_attachments': dup_att, - } - ) - - self.asset.deployment.mock_submissions([duplicated_submission]) - return duplicated_submission +class MockDeploymentBackend(OpenRosaDeploymentBackend): @property def enketo_id(self): return 'self' - def get_attachment( - self, - submission_id_or_uuid: Union[int, str], - user: settings.AUTH_USER_MODEL, - attachment_id: Optional[int] = None, - xpath: Optional[str] = None, - ) -> 'logger.Attachment': - submission_json = None - # First try to get the json version of the submission. - # It helps to retrieve the id if `submission_id_or_uuid` is a `UUIDv4` - try: - submission_id_or_uuid = int(submission_id_or_uuid) - except ValueError: - submissions = self.get_submissions( - user, - format_type=SUBMISSION_FORMAT_TYPE_JSON, - query={'_uuid': submission_id_or_uuid}, - ) - if submissions: - submission_json = submissions[0] - else: - submission_json = self.get_submission( - submission_id_or_uuid, - user, - format_type=SUBMISSION_FORMAT_TYPE_JSON, - ) - - if not submission_json: - raise SubmissionNotFoundException - - submission_xml = self.get_submission( - submission_json['_id'], user, format_type=SUBMISSION_FORMAT_TYPE_XML - ) - - if xpath: - submission_root = fromstring_preserve_root_xmlns(submission_xml) - try: - element = submission_root.find(xpath) - except KeyError: - raise InvalidXPathException - - try: - attachment_filename = element.text - except AttributeError: - raise XPathNotFoundException - - attachments = submission_json['_attachments'] - for attachment in attachments: - filename = os.path.basename(attachment['filename']) - - if xpath: - is_good_file = attachment_filename == filename - else: - is_good_file = int(attachment['id']) == int(attachment_id) - - if is_good_file: - return self._get_attachment_object( - attachment_id=attachment['id'], - submission_xml=submission_xml, - submission_id=submission_json['_id'], - filename=filename, - mimetype=attachment.get('mimetype'), - ) - - raise AttachmentNotFoundException - - def get_attachment_objects_from_dict(self, submission: dict) -> list: - if not submission.get('_attachments'): - return [] - attachments = submission.get('_attachments') - submission_xml = self.get_submission( - submission['_id'], self.asset.owner, format_type=SUBMISSION_FORMAT_TYPE_XML - ) - - return [ - self._get_attachment_object( - attachment_id=attachment['id'], - submission_xml=submission_xml, - submission_id=submission['_id'], - filename=os.path.basename(attachment['filename']), - mimetype=attachment.get('mimetype'), - ) - for attachment in attachments - ] - - def get_data_download_links(self): - return {} - def get_enketo_survey_links(self): return { 'offline_url': f'https://example.org/_/#{self.enketo_id}', @@ -368,161 +34,103 @@ def get_enketo_survey_links(self): 'preview_url': f'https://example.org/preview/::#{self.enketo_id}', } - def get_submission_detail_url(self, submission_id: int) -> str: - # This doesn't really need to be implemented. - # We keep it to stay close to `KobocatDeploymentBackend` - url = f'{self.submission_list_url}{submission_id}/' - return url - - def get_submission_validation_status_url(self, submission_id: int) -> str: - url = '{detail_url}validation_status/'.format( - detail_url=self.get_submission_detail_url(submission_id) - ) - return url - - def get_daily_counts( - self, user: settings.AUTH_USER_MODEL, timeframe: tuple[date, date] - ) -> dict: - submissions = self.get_submissions(user=self.asset.owner) - daily_counts = defaultdict(int) - for submission in submissions: - submission_date = datetime.strptime( - submission['_submission_time'], '%Y-%m-%dT%H:%M:%S' - ) - daily_counts[str(submission_date.date())] += 1 - - return daily_counts - def get_submissions( self, user: settings.AUTH_USER_MODEL, format_type: str = SUBMISSION_FORMAT_TYPE_JSON, - submission_ids: list = [], + submission_ids: list = None, request: Optional['rest_framework.request.Request'] = None, - **mongo_query_params, + **mongo_query_params ) -> list: + # Overload parent to cast generator to a list. Many tests are expecting + # a list + return list(super().get_submissions( + user, format_type, submission_ids, request, **mongo_query_params + )) + + def mock_submissions( + self, submissions, create_uuids: bool = True, flush_db: bool = True + ): """ - Retrieve submissions that `user` is allowed to access. - - The format `format_type` can be either: - - 'json' (See `kpi.constants.SUBMISSION_FORMAT_TYPE_JSON`) - - 'xml' (See `kpi.constants.SUBMISSION_FORMAT_TYPE_XML`) - - Results can be filtered by submission ids. Moreover MongoDB filters can - be passed through `mongo_query_params` to narrow down the results. + Simulate client (i.e.: Enketo or Collect) data submission. - If `user` has no access to these submissions or no matches are found, - an empty list is returned. - If `format_type` is 'json', a list of dictionaries is returned. - Otherwise, if `format_type` is 'xml', a list of strings is returned. + Read test data and convert it to proper XML to be saved as a real + Instance object. """ - mongo_query_params['submission_ids'] = submission_ids - params = self.validate_submission_list_params( - user, format_type=format_type, **mongo_query_params - ) - - mongo_cursor, total_count = MongoHelper.get_instances( - self.mongo_userform_id, **params - ) - - # Python-only attribute used by `kpi.views.v2.data.DataViewSet.list()` - self.current_submission_count = total_count - - submissions = [ - self._rewrite_json_attachment_urls( - MongoHelper.to_readable_dict(submission), - request, - ) - for submission in mongo_cursor - ] - - if format_type != SUBMISSION_FORMAT_TYPE_XML: - return submissions + class FakeRequest: + pass - return [ - dict2xml( - self.__prepare_xml(submission), - wrap=self.asset.uid, - newlines=False, - ) - for submission in submissions - ] + request = FakeRequest() + owner_username = self.asset.owner.username - def get_validation_status( - self, submission_id: int, user: settings.AUTH_USER_MODEL - ) -> dict: - submission = self.get_submission(submission_id, user) - return { - 'content_type': 'application/json', - 'data': submission.get('_validation_status'), - } - - @drop_mock_only - def mock_submissions(self, submissions: list, flush_db: bool = True): - """ - Insert dummy submissions into deployment data - """ - if flush_db: - settings.MONGO_DB.instances.drop() - count = settings.MONGO_DB.instances.count_documents({}) + for submission in submissions: + sub_copy = nested_dict_from_keys(submission) - for idx, submission in enumerate(submissions): - submission[MongoHelper.USERFORM_ID] = self.mongo_userform_id - # Some data already provide `_id`. Use it if it is present. - # There could be conflicts if some submissions come with an id - # or others do not. - # MockMongo will raise a DuplicateKey error - if '_id' not in submission: - submission['_id'] = count + idx + 1 - settings.MONGO_DB.instances.insert_one(submission) - # Do not add `MongoHelper.USERFORM_ID` to original `submissions` - del submission[MongoHelper.USERFORM_ID] + if create_uuids: + if 'formhub/uuid' not in submission: + sub_copy['formhub'] = {'uuid': self.xform.uuid} - @property - def mongo_userform_id(self): - return f'{self.asset.owner.username}_{self.asset.uid}' + if 'meta/instanceID' not in submission: + try: + uuid_ = submission['_uuid'] + except KeyError: + uuid_ = str(uuid4()) + else: + uuid_ = submission['meta/instanceID'].replace('uuid:', '') - def redeploy(self, active: bool = None): - """ - Replace (overwrite) the deployment, and - optionally changing whether the deployment is active - """ - if active is None: - active = self.active + sub_copy['meta'] = {'instanceID': f'uuid:{uuid_}'} + submission['_uuid'] = uuid_ - self.store_data( - { - 'active': active, - 'version': self.asset.version_id, - } - ) - - self.set_asset_uid() - - def rename_enketo_id_key(self, previous_owner_username: str): - pass + assign_perm = False + try: + submitted_by = sub_copy['_submitted_by'] + except KeyError: + request.user = self.asset.owner + submitted_by = self.asset.owner.username + else: + if not submitted_by: + request.user = AnonymousUser() + submitted_by = '' + elif owner_username != submitted_by: + request.user = User.objects.get(username=submitted_by) + else: + request.user = self.asset.owner + + if not self.asset.has_perm(request.user, PERM_ADD_SUBMISSIONS): + # We want `request.user` to be able to add submissions + # (temporarily) to match `_submitted_by` value while saving + # in DB + self.asset.assign_perm(request.user, PERM_ADD_SUBMISSIONS) + assign_perm = True + + media_files = self._get_media_files(sub_copy) + + xml_string = dict2xform(sub_copy, self.xform.id_string) + xml_file = io.StringIO(xml_string) + error, instance = safe_create_instance( + owner_username, + xml_file, + media_files, + date_created_override=parse_datetime( + submission.get('_submission_time', '') # Returns None if empty + ), + request=request, + ) + if error: + raise Exception(error) - def set_active(self, active: bool): - self.save_to_db( - { - 'active': bool(active), - } - ) + # Inject (or update) real PK in submission + # FIXME TRY TO ASSIGN Instance.PK if it already exists + submission['_id'] = instance.pk - def set_asset_uid(self, **kwargs) -> bool: - backend_response = self.backend_response - backend_response.update( - { - 'kpi_asset_uid': self.asset.uid, - } - ) - self.store_data({'backend_response': backend_response}) + # Reassign attachment PKs + if '_attachments' in submission: + for idx, attachment in enumerate(instance.attachments.all()): + submission['_attachments'][idx]['id'] = attachment.pk - def set_enketo_open_rosa_server( - self, require_auth: bool, enketo_id: str = None - ): - pass + if assign_perm: + self.asset.remove_perm(request.user, PERM_ADD_SUBMISSIONS) def set_namespace(self, namespace): self.store_data( @@ -531,225 +139,23 @@ def set_namespace(self, namespace): } ) - def set_validation_status( - self, - submission_id: int, - user: settings.AUTH_USER_MODEL, - data: dict, - method: str, - ) -> dict: - self.validate_access_with_partial_perms( - user=user, - perm=PERM_VALIDATE_SUBMISSIONS, - submission_ids=[submission_id], - ) - - validation_status = {} - status_code = status.HTTP_204_NO_CONTENT - - if method != 'DELETE': - validation_status = { - 'timestamp': int(time.time()), - 'uid': data['validation_status.uid'], - 'by_whom': user.username, - } - status_code = status.HTTP_200_OK - - settings.MONGO_DB.instances.update_one( - {'_id': submission_id}, - {'$set': {'_validation_status': validation_status}}, - ) - return { - 'content_type': 'application/json', - 'status': status_code, - 'data': validation_status, - } - - def set_validation_statuses( - self, user: settings.AUTH_USER_MODEL, data: dict - ) -> dict: - """ - Bulk update validation status for provided submissions. - - `data` should contains either the submission ids or the query to - retrieve the subset of submissions chosen by then user. - If none of them are provided, all the submissions are selected - Examples: - {"submission_ids": [1, 2, 3]} - {"query":{"_validation_status.uid":"validation_status_not_approved"} - - """ - - submission_ids = self.validate_access_with_partial_perms( - user=user, - perm=PERM_VALIDATE_SUBMISSIONS, - submission_ids=data['submission_ids'], - query=data['query'], - ) - - if not submission_ids: - submission_ids = data['submission_ids'] - else: - # Reset query because submission ids are provided from partial - # perms validation - data['query'] = {} - - submissions = self.get_submissions( - user=user, - submission_ids=submission_ids, - query=data['query'], - fields=['_id'], - ) - - submission_count = 0 - - for submission in submissions: - if not data['validation_status.uid']: - validation_status = {} - else: - validation_status = { - 'timestamp': int(time.time()), - 'uid': data['validation_status.uid'], - 'by_whom': user.username, - } - settings.MONGO_DB.instances.update_one( - {'_id': submission['_id']}, - {'$set': {'_validation_status': validation_status}}, - ) - - submission_count += 1 - - return { - 'content_type': 'application/json', - 'status': status.HTTP_200_OK, - 'data': { - 'detail': f'{submission_count} submissions have been updated' - }, - } - - def store_submission( - self, user, xml_submission, submission_uuid, attachments=None, **kwargs - ): - """ - Return a mock response without actually storing anything - """ - - return { - 'uuid': submission_uuid, - 'status_code': status.HTTP_201_CREATED, - 'message': 'Successful submission', - 'updated_submission': xml_submission, - } - @property - def submission_count(self): - return self.calculated_submission_count(self.asset.owner) + def _backend_identifier(self): + return 'mock' - @property - def submission_list_url(self): - # This doesn't really need to be implemented. - # We keep it to stay close to `KobocatDeploymentBackend` - view_name = 'submission-list' - namespace = self.get_data('namespace', None) - if namespace is not None: - view_name = '{}:{}'.format(namespace, view_name) - return reverse( - view_name, kwargs={'parent_lookup_asset': self.asset.uid} - ) + def _get_media_files(self, submission): - @property - def submission_model(self): - class MockLoggerInstance: - @classmethod - def get_app_label_and_model_name(cls): - return 'mocklogger', 'instance' - - return MockLoggerInstance - - @staticmethod - @contextmanager - def suspend_submissions(user_ids: list[int]): try: - yield - finally: - pass - - def sync_media_files(self, file_type: str = AssetFile.FORM_MEDIA): - queryset = self._get_metadata_queryset(file_type=file_type) - for obj in queryset: - assert issubclass(obj.__class__, SyncBackendMediaInterface) - - def transfer_counters_ownership(self, new_owner: 'kobo_auth.User'): - NLPUsageCounter.objects.filter( - asset=self.asset, user=self.asset.owner - ).update(user=new_owner) - - # Kobocat models are not implemented, but mocked in unit tests. - - def transfer_submissions_ownership( - self, previous_owner_username: str - ) -> bool: - - results = settings.MONGO_DB.instances.update_many( - {'_userform_id': f'{previous_owner_username}_{self.xform_id_string}'}, - { - '$set': { - '_userform_id': self.mongo_userform_id - } - }, - ) - - return ( - results.matched_count == 0 or - ( - results.matched_count > 0 - and results.matched_count == results.modified_count - ) - ) - - @property - def xform(self): - """ - Create related XForm on the fly - """ - if not ( - xform := XForm.objects.filter(id_string=self.asset.uid).first() - ): - UserProfile.objects.get_or_create(user_id=self.asset.owner_id) - xform = XForm() - xform.xml = self.asset.snapshot().xml - xform.user_id = self.asset.owner_id - xform.kpi_asset_uid = self.asset.uid - xform.save() - - return xform + attachments = submission['_attachments'] + except KeyError: + return [] - @property - def xform_id_string(self): - return self.xform.id_string + for attachment in attachments: + filename = attachment['filename'] - def _get_attachment_object( - self, - submission_xml: str, - submission_id: int, - attachment_id: Optional[int, str] = None, - filename: Optional[str] = None, - mimetype: Optional[str] = None, - ): - if not ( - attachment := Attachment.objects.filter( - Q(pk=attachment_id) | Q(media_file_basename=filename) - ).first() - ): - if not ( - instance := Instance.objects.filter(pk=submission_id).first() - ): - instance = Instance.objects.create( - pk=submission_id, xml=submission_xml, xform=self.xform - ) + if filename == 'path/to/image.png': + continue - attachment = Attachment() - attachment.instance = instance basename = os.path.basename(filename) file_ = os.path.join( settings.BASE_DIR, @@ -757,58 +163,12 @@ def _get_attachment_object( 'tests', basename ) - with open(file_, 'rb') as f: - attachment.media_file = ContentFile( - f.read(), name=upload_to(attachment, basename) + if not os.path.isfile(file_): + raise Exception( + f'File `filename` does not exist! Use `path/to/image.png` if' + f' you need a fake attachment, or ' + f'`audio_conversion_test_image.(jpg|3gp)` for real attachment' ) - if mimetype: - attachment.mimetype = mimetype - attachment.save() - - return attachment - @classmethod - def __prepare_bulk_update_data(cls, updates: dict) -> dict: - """ - Preparing the request payload for bulk updating of submissions - """ - # Sanitizing the payload of potentially destructive keys - sanitized_updates = copy.deepcopy(updates) - for key in updates: - if ( - key in cls.PROTECTED_XML_FIELDS - or '/' in key - and key.split('/')[0] in cls.PROTECTED_XML_FIELDS - ): - sanitized_updates.pop(key) - - return sanitized_updates - - @staticmethod - def prepare_bulk_update_response(kc_responses: list) -> dict: - total_update_attempts = len(kc_responses) - total_successes = total_update_attempts # all will be successful - return { - 'status': status.HTTP_200_OK, - 'data': { - 'count': total_update_attempts, - 'successes': total_successes, - 'failures': total_update_attempts - total_successes, - 'results': kc_responses, - }, - } - - @staticmethod - def __prepare_xml(submission: dict) -> dict: - submission_copy = copy.deepcopy(submission) - - for k, v in submission_copy.items(): - if '/' not in k: - continue - value = v - for key in reversed(k.strip('/').split('/')): - value = {key: value} - always_merger.merge(submission, value) - del submission[k] - - return submission + with open(file_, 'rb') as f: + yield ExtendedContentFile(f.read(), name=basename) diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index ca0c93e3e7..bf4933a52b 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -57,6 +57,7 @@ from kpi.exceptions import ( AttachmentNotFoundException, InvalidXFormException, + InvalidXPathException, SubmissionIntegrityError, SubmissionNotFoundException, XPathNotFoundException, @@ -157,7 +158,7 @@ def connect(self, active=False): self.store_data( { - 'backend': 'openrosa', + 'backend': self._backend_identifier, 'active': active, 'backend_response': { 'formid': self._xform.pk, @@ -209,7 +210,7 @@ def delete_submission( self, submission_id: int, user: settings.AUTH_USER_MODEL ) -> dict: """ - Delete a submission through KoBoCAT proxy + Delete a submission It returns a dictionary which can used as Response object arguments """ @@ -220,7 +221,13 @@ def delete_submission( submission_ids=[submission_id] ) - Instance.objects.filter(pk=submission_id).delete() + count, _ = Instance.objects.filter(pk=submission_id).delete() + if not count: + return { + 'data': {'detail': 'Submission not found'}, + 'content_type': 'application/json', + 'status': status.HTTP_404_NOT_FOUND, + } return { 'content_type': 'application/json', @@ -268,15 +275,15 @@ def duplicate_submission( self, submission_id: int, request: 'rest_framework.request.Request', ) -> dict: """ - Duplicates a single submission proxied through KoBoCAT. The submission - with the given `submission_id` is duplicated and the `start`, `end` and - `instanceID` parameters of the submission are reset before being posted - to KoBoCAT. + Duplicates a single submission. The submission with the given + `submission_id` is duplicated and the `start`, `end` and + `instanceID` parameters of the submission are reset before being + saving the instance. - Returns a dict with message response from KoBoCAT and uuid of created + Returns a dict with uuid of created submission if successful - """ + user = request.user self.validate_access_with_partial_perms( user=user, @@ -321,14 +328,16 @@ def duplicate_submission( ) safe_create_instance( - username=user.username, + username=self.asset.owner.username, xml_file=ContentFile(xml_tostring(xml_parsed)), media_files=attachments, uuid=_uuid, request=request, ) + + # Cast to list to help unit tests to pass. return self._rewrite_json_attachment_urls( - next(self.get_submissions(user, query={'_uuid': _uuid})), request + list(self.get_submissions(user, query={'_uuid': _uuid}))[0], request ) def edit_submission( @@ -477,7 +486,11 @@ def get_attachment( if xpath: submission_root = fromstring_preserve_root_xmlns(submission_xml) - element = submission_root.find(xpath) + try: + element = submission_root.find(xpath) + except KeyError: + raise InvalidXPathException + if element is None: raise XPathNotFoundException attachment_filename = element.text @@ -701,14 +714,16 @@ def get_orphan_postgres_submissions(self) -> Optional[QuerySet, bool]: return None def get_submission_detail_url(self, submission_id: int) -> str: - url = f'{self.submission_list_url}/{submission_id}' - return url + 1/0 + #url = f'{self.submission_list_url}/{submission_id}' + #return url def get_submission_validation_status_url(self, submission_id: int) -> str: - url = '{detail_url}/validation_status'.format( - detail_url=self.get_submission_detail_url(submission_id) - ) - return url + 1/0 + #url = '{detail_url}/validation_status'.format( + # detail_url=self.get_submission_detail_url(submission_id) + #) + #return url def get_submissions( self, @@ -848,7 +863,7 @@ def redeploy(self, active=None): # after calling this method in `DeployableMixin.deploy()` self.store_data( { - 'backend': 'openrosa', + 'backend': self._backend_identifier, 'active': active, 'backend_response': { 'formid': self.xform.pk, @@ -1115,7 +1130,7 @@ def set_validation_statuses( # TODO handle errors update_instances = set_instance_validation_statuses( - self.xform, data, user + self.xform, data, user.username ) return { @@ -1134,7 +1149,7 @@ def store_submission( ) return safe_create_instance( - username=user.username, + username=self.asset.owner.username, xml_file=ContentFile(xml_submission), media_files=media_files, uuid=submission_uuid, @@ -1174,11 +1189,12 @@ def submission_count_since_date(self, start_date=None): @property def submission_list_url(self): - url = '{kc_base}/api/v1/data/{formid}'.format( - kc_base=settings.KOBOCAT_INTERNAL_URL, - formid=self.backend_response['formid'] - ) - return url + 1/0 + #url = '{kc_base}/api/v1/data/{formid}'.format( + # kc_base=settings.KOBOCAT_INTERNAL_URL, + # formid=self.backend_response['formid'] + #) + #return url @property def submission_model(self): @@ -1377,6 +1393,10 @@ def transfer_counters_ownership(self, new_owner: 'kobo_auth.User'): + self.xform.attachment_storage_bytes ) + @property + def _backend_identifier(self): + return 'openrosa' + def _delete_openrosa_metadata( self, metadata_file_: dict, file_: Union[AssetFile, PairedData] = None ): diff --git a/kpi/fixtures/test_data.json b/kpi/fixtures/test_data.json index 0ea399393e..2c6fee0074 100644 --- a/kpi/fixtures/test_data.json +++ b/kpi/fixtures/test_data.json @@ -75,6 +75,33 @@ "model": "kobo_auth.user", "pk": 3 }, + { + "fields": { + "name": "Administrator", + "validated_password": true, + "user": 1 + }, + "model": "main.userprofile", + "pk": 1 + }, + { + "fields": { + "name": "Some User", + "validated_password": true, + "user": 2 + }, + "model": "main.userprofile", + "pk": 2 + }, + { + "fields": { + "name": "Another User", + "validated_password": true, + "user": 3 + }, + "model": "main.userprofile", + "pk": 3 + }, { "model": "kpi.asset", "pk": 1, diff --git a/kpi/signals.py b/kpi/signals.py index 89be23937d..2c41091ad7 100644 --- a/kpi/signals.py +++ b/kpi/signals.py @@ -4,68 +4,17 @@ from django.contrib.auth.models import AnonymousUser from django.db.models.signals import post_save, post_delete from django.dispatch import receiver -from rest_framework.authtoken.models import Token from taggit.models import Tag -from kobo.apps.kobo_auth.shortcuts import User -from kobo.apps.hook.models.hook import Hook from kpi.constants import PERM_ADD_SUBMISSIONS - -from kpi.deployment_backends.kc_access.utils import ( - grant_kc_model_level_perms, - kc_transaction_atomic, -) from kpi.exceptions import DeploymentNotFound from kpi.models import Asset, TagUid from kpi.utils.object_permission import post_assign_perm, post_remove_perm from kpi.utils.permissions import ( - grant_default_model_level_perms, is_user_anonymous, ) -@receiver(post_save, sender=User) -def create_auth_token(sender, instance=None, created=False, **kwargs): - if is_user_anonymous(instance): - return - - if created: - Token.objects.get_or_create(user_id=instance.pk) - - -@receiver(post_save, sender=User) -def default_permissions_post_save(sender, instance, created, raw, **kwargs): - """ - Users must have both model-level and object-level permissions to satisfy - DRF, so assign the newly-created user all available collection and asset - permissions at the model level - """ - if raw: - # `raw` means we can't touch (so make sure your fixtures include - # all necessary permissions!) - return - if not created: - # We should only grant default permissions when the user is first - # created - return - grant_default_model_level_perms(instance) - - -@receiver(post_save, sender=User) -def save_kobocat_user(sender, instance, created, raw, **kwargs): - """ - Sync auth_user table between KPI and KC, and, if the user is newly created, - grant all KoboCAT model-level permissions for the content types listed in - `settings.KOBOCAT_DEFAULT_PERMISSION_CONTENT_TYPES` - """ - - if not settings.TESTING: - with kc_transaction_atomic(): - instance.sync_to_openrosa_db() - if created: - grant_kc_model_level_perms(instance) - - @receiver(post_save, sender=Tag) def tag_uid_post_save(sender, instance, created, raw, **kwargs): """ Make sure we have a TagUid object for each newly-created Tag """ diff --git a/kpi/tests/api/v1/test_api_assets.py b/kpi/tests/api/v1/test_api_assets.py index 70d5a998d3..74dedbd253 100644 --- a/kpi/tests/api/v1/test_api_assets.py +++ b/kpi/tests/api/v1/test_api_assets.py @@ -256,7 +256,7 @@ def setUp(self): self.client.login(username='someuser', password='someuser') self.user = User.objects.get(username='someuser') self.asset = Asset.objects.create( - content={'survey': [{"type": "text", "name": "q1"}]}, + content={'survey': [{'type': 'text', 'label': 'q1', 'name': 'q1'}]}, owner=self.user, asset_type='survey', name='тєѕт αѕѕєт' @@ -264,11 +264,12 @@ def setUp(self): self.asset.deploy(backend='mock', active=True) self.asset.save() v_uid = self.asset.latest_deployed_version.uid - submission = { + self.submission = { '__version__': v_uid, - 'q1': '¿Qué tal?' + 'q1': '¿Qué tal?', + '_submission_time': '2024-08-07T23:42:21', } - self.asset.deployment.mock_submissions([submission]) + self.asset.deployment.mock_submissions([self.submission], ) def test_owner_can_create_export(self): post_url = reverse('exporttask-list') @@ -292,9 +293,8 @@ def test_owner_can_create_export(self): version_uid = self.asset.latest_deployed_version_uid expected_content = ''.join([ '"q1";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"\r\n', - f'"¿Qué tal?";"1";"";"";"";"";"";"";"{version_uid}";"";"1"\r\n', + f'"¿Qué tal?";"{self.submission["_id"]}";"{self.submission["_uuid"]}";"2024-08-07T23:42:21";"";"";"submitted_via_web";"someuser";"{version_uid}";"";"1"\r\n', ]) - self.assertEqual(result_content, expected_content) return detail_response diff --git a/kpi/tests/api/v1/test_api_submissions.py b/kpi/tests/api/v1/test_api_submissions.py index 2504f6a128..4c5ee14ae0 100644 --- a/kpi/tests/api/v1/test_api_submissions.py +++ b/kpi/tests/api/v1/test_api_submissions.py @@ -1,6 +1,7 @@ # coding: utf-8 import pytest from django.conf import settings +from django.urls import reverse from rest_framework import status from kpi.constants import ( @@ -27,14 +28,18 @@ def test_retrieve_submission_with_partial_permissions_as_anotheruser(self): def test_list_submissions_as_owner(self): response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, self.submissions) + expected_ids = [s['_id'] for s in self.submissions] + response_ids = [r['_id'] for r in response.data] + assert sorted(response_ids) == sorted(expected_ids) def test_list_submissions_shared_as_anotheruser(self): self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, self.submissions) + expected_ids = [s['_id'] for s in self.submissions] + response_ids = [r['_id'] for r in response.data] + assert sorted(response_ids) == sorted(expected_ids) def test_list_submissions_limit(self): limit = settings.SUBMISSION_LIST_LIMIT @@ -42,7 +47,7 @@ def test_list_submissions_limit(self): asset = Asset.objects.create( name='Lots of submissions', owner=self.asset.owner, - content={'survey': [{'name': 'q', 'type': 'integer'}]}, + content={'survey': [{'label': 'q', 'name': 'q', 'type': 'integer'}]}, ) asset.deploy(backend='mock', active=True) asset.deployment.set_namespace(self.URL_NAMESPACE) @@ -56,15 +61,21 @@ def test_list_submissions_limit(self): asset.deployment.mock_submissions(submissions) # Server-wide limit should apply if no limit specified - response = self.client.get( - asset.deployment.submission_list_url, {'format': 'json'} + url = reverse( + self._get_endpoint('submission-list'), + kwargs={'format': 'json', 'parent_lookup_asset': asset.uid}, ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data), limit) # Limit specified in query parameters should not be able to exceed # server-wide limit + url = reverse( + self._get_endpoint('submission-list'), + kwargs={'parent_lookup_asset': asset.uid, 'format': 'json'}, + ) response = self.client.get( - asset.deployment.submission_list_url, + url, {'limit': limit + excess, 'format': 'json'} ) @@ -79,7 +90,7 @@ def test_list_submissions_as_owner_with_params(self): 'limit': 5, 'sort': '{"q1": -1}', 'fields': '["q1", "_submitted_by"]', - 'query': '{"_submitted_by": {"$in": ["", "someuser", "another"]}}', + 'query': '{"_submitted_by": {"$in": ["unknown", "someuser", "another"]}}', } ) # ToDo add more assertions. E.g. test whether sort, limit, start really work @@ -88,8 +99,13 @@ def test_list_submissions_as_owner_with_params(self): def test_delete_submission_as_owner(self): submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url( - submission['_id']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) @@ -101,7 +117,13 @@ def test_delete_submission_shared_as_anotheruser(self): self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) response = self.client.get(self.submission_list_url, {'format': 'json'}) diff --git a/kpi/tests/api/v2/test_api_asset_counts.py b/kpi/tests/api/v2/test_api_asset_counts.py index 617dc6d11e..d5e3ffe8bc 100644 --- a/kpi/tests/api/v2/test_api_asset_counts.py +++ b/kpi/tests/api/v2/test_api_asset_counts.py @@ -1,4 +1,5 @@ from django.urls import reverse +from django.test import override_settings from rest_framework import status from kobo.apps.kobo_auth.shortcuts import User @@ -56,7 +57,11 @@ def setUp(self): self.asset.deployment.mock_submissions(submissions) + @override_settings(DEFAULT_SUBMISSIONS_COUNT_NUMBER_OF_DAYS=10000) def test_count_endpoint_owner(self): + # Submission submitted time is 2022-09-12. + # DEFAULT_SUBMISSIONS_COUNT_NUMBER_OF_DAYS must be big enough to include + # this date. count_url = reverse( self._get_endpoint('asset-counts-list'), kwargs={'parent_lookup_asset': self.asset.uid} @@ -102,7 +107,11 @@ def test_count_endpoint_another_user_no_perms(self): response = self.client.get(count_url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + @override_settings(DEFAULT_SUBMISSIONS_COUNT_NUMBER_OF_DAYS=10000) def test_count_endpoint_another_with_perms(self): + # Submission submitted time is 2022-09-12. + # DEFAULT_SUBMISSIONS_COUNT_NUMBER_OF_DAYS must be big enough to include + # this date. count_url = reverse( self._get_endpoint('asset-counts-list'), kwargs={'parent_lookup_asset': self.asset.uid} diff --git a/kpi/tests/api/v2/test_api_asset_usage.py b/kpi/tests/api/v2/test_api_asset_usage.py index 3a34712f8c..261db675b4 100644 --- a/kpi/tests/api/v2/test_api_asset_usage.py +++ b/kpi/tests/api/v2/test_api_asset_usage.py @@ -19,12 +19,7 @@ class AssetUsageAPITestCase(BaseAssetTestCase): URL_NAMESPACE = ROUTER_URL_NAMESPACE def setUp(self): - try: - self.anotheruser = User.objects.get(username='anotheruser') - except: - self.anotheruser = User.objects.create_user( - username='anotheruser', password='anotheruser' - ) + self.anotheruser = User.objects.get(username='anotheruser') self.client.login(username='anotheruser', password='anotheruser') def __add_nlp_trackers(self): @@ -133,7 +128,11 @@ def __create_asset(self): self.asset.save() self.asset.deployment.set_namespace(self.URL_NAMESPACE) - self.submission_list_url = self.asset.deployment.submission_list_url + self.submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, + ) + self._deployment = self.asset.deployment def __expected_file_size(self): diff --git a/kpi/tests/api/v2/test_api_attachments.py b/kpi/tests/api/v2/test_api_attachments.py index 0f3c891cf7..49f83ff3b8 100644 --- a/kpi/tests/api/v2/test_api_attachments.py +++ b/kpi/tests/api/v2/test_api_attachments.py @@ -1,22 +1,18 @@ import uuid -from django.conf import settings -from django.core.files.base import File from django.http import QueryDict from django.urls import reverse +from mock import patch from rest_framework import status from kobo.apps.kobo_auth.shortcuts import User -from kobo.apps.openrosa.apps.main.models import UserProfile -from kobo.apps.openrosa.apps.logger.models import XForm, Instance, Attachment -from kobo.apps.openrosa.apps.logger.models.attachment import upload_to -from kobo.apps.openrosa.apps.viewer.models import ParsedInstance from kpi.deployment_backends.kc_access.storage import ( default_kobocat_storage as default_storage, ) from kpi.models import Asset from kpi.tests.base_test_case import BaseAssetTestCase from kpi.urls.router_api_v2 import URL_NAMESPACE as ROUTER_URL_NAMESPACE +from kpi.tests.utils.mock import guess_type_mock class AttachmentApiTests(BaseAssetTestCase): @@ -43,8 +39,12 @@ def setUp(self) -> None: self.__add_submissions() self.asset.deployment.set_namespace(self.URL_NAMESPACE) - self.submission_list_url = self.asset.deployment.submission_list_url + self.submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={'format': 'json', 'parent_lookup_asset': self.asset.uid}, + ) self._deployment = self.asset.deployment + self.submission_id = self.submissions[0]['_id'] def __add_submissions(self): submissions = [] @@ -59,13 +59,11 @@ def __add_submissions(self): 'meta/instanceID': f'uuid:{_uuid}', '_attachments': [ { - 'id': 1, 'download_url': 'http://testserver/someuser/audio_conversion_test_clip.3gp', 'filename': 'someuser/audio_conversion_test_clip.3gp', 'mimetype': 'video/3gpp', }, { - 'id': 2, 'download_url': 'http://testserver/someuser/audio_conversion_test_image.jpg', 'filename': 'someuser/audio_conversion_test_image.jpg', 'mimetype': 'image/jpeg', @@ -74,7 +72,11 @@ def __add_submissions(self): '_submitted_by': 'someuser' } submissions.append(submission) - self.asset.deployment.mock_submissions(submissions) + + with patch('mimetypes.guess_type') as guess_mock: + guess_mock.side_effect = guess_type_mock + self.asset.deployment.mock_submissions(submissions) + self.submissions = submissions def test_convert_mp4_to_mp3(self): @@ -90,11 +92,12 @@ def test_convert_mp4_to_mp3(self): self._get_endpoint('attachment-list'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, + 'parent_lookup_data': self.submission_id, }, ), querystring=query_dict.urlencode() ) + response = self.client.get(url) assert response.status_code == status.HTTP_200_OK assert response['Content-Type'] == 'audio/mpeg' @@ -112,7 +115,7 @@ def test_reject_image_with_conversion(self): self._get_endpoint('attachment-list'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, + 'parent_lookup_data': self.submission_id, }, ), querystring=query_dict.urlencode() @@ -136,7 +139,7 @@ def test_get_mp4_without_conversion(self): self._get_endpoint('attachment-list'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, + 'parent_lookup_data': self.submission_id, }, ), querystring=query_dict.urlencode() @@ -147,12 +150,13 @@ def test_get_mp4_without_conversion(self): assert response['Content-Type'] == 'video/3gpp' def test_get_attachment_with_id(self): + attachment_id = self.submissions[0]['_attachments'][0]['id'] url = reverse( self._get_endpoint('attachment-detail'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, - 'pk': 1, + 'parent_lookup_data': self.submission_id, + 'pk': attachment_id, }, ) @@ -177,14 +181,16 @@ def test_duplicate_attachment_with_submission(self): original_file = response.data # Duplicate the submission - duplicate_url = reverse( - self._get_endpoint('submission-duplicate'), - kwargs={ - 'parent_lookup_asset': self.asset.uid, - 'pk': submission['_id'], - }, - ) - response = self.client.post(duplicate_url, {'format': 'json'}) + with patch('mimetypes.guess_type') as guess_mock: + guess_mock.side_effect = guess_type_mock + duplicate_url = reverse( + self._get_endpoint('submission-duplicate'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.post(duplicate_url, {'format': 'json'}) duplicate_submission = response.data # Increment the max attachment id of the original submission to get the @@ -205,7 +211,7 @@ def test_duplicate_attachment_with_submission(self): duplicate_file = response.data # Ensure that the files are the same - assert original_file == duplicate_file + assert original_file.read() == duplicate_file.read() def test_xpath_not_found(self): query_dict = QueryDict('', mutable=True) @@ -220,7 +226,7 @@ def test_xpath_not_found(self): self._get_endpoint('attachment-list'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, + 'parent_lookup_data': self.submission_id, }, ), querystring=query_dict.urlencode() @@ -244,7 +250,7 @@ def test_invalid_xpath_syntax(self): self._get_endpoint('attachment-list'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'parent_lookup_data': 1, + 'parent_lookup_data': self.submission_id, }, ), querystring=query_dict.urlencode() @@ -262,7 +268,7 @@ def test_get_attachment_with_submission_uuid(self): kwargs={ 'parent_lookup_asset': self.asset.uid, 'parent_lookup_data': submission['_uuid'], - 'pk': 1, + 'pk': submission['_attachments'][0]['id'], }, ) @@ -271,86 +277,17 @@ def test_get_attachment_with_submission_uuid(self): assert response['Content-Type'] == 'video/3gpp' def test_thumbnail_creation_on_demand(self): - media_file = settings.BASE_DIR + '/kpi/tests/audio_conversion_test_image.jpg' - - xform_xml = f""" - - - Project with attachments - - - <{self.asset.uid} id="{self.asset.uid}"> - - - - - <__version__/> - - - - - - - - - - - - - - - - - - """ - - instance_xml = f""" - <{self.asset.uid} xmlns:jr="http://openrosa.org/javarosa" xmlns:orx="http://openrosa.org/xforms" id="{self.asset.uid}"> - - 027e8acb31b24acebb7f6b2a74ac1ff3 - - audio_conversion_test_image.jpg - <__version__>vd3dpf3fL2C8abWG4EPJWC - - uuid:ba82fbca-9a05-45c7-afb6-295c90f838e5 - - - """ - - UserProfile.objects.get_or_create(user=self.someuser) - xform = XForm.objects.create( - user=self.someuser, - xml=xform_xml, - id_string=self.asset.uid, - kpi_asset_uid=self.asset.uid - ) - instance = Instance.objects.create(xform=xform, xml=instance_xml) - attachment = Attachment.objects.create(instance=instance) - attachment.media_file = File( - open(media_file, 'rb'), upload_to(attachment, media_file) - ) - attachment.save() - - pi = ParsedInstance.objects.create(instance=instance) - self.asset.deployment.mock_submissions( - [pi.to_dict_for_mongo()] - ) - detail_url = reverse( + submission = self.submissions[0] + url = reverse( self._get_endpoint('attachment-detail'), - args=( - self.asset.uid, - instance.pk, - attachment.pk - ), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'parent_lookup_data': submission['_id'], + 'pk': submission['_attachments'][1]['id'], + }, ) - self.client.get(detail_url) - filename = attachment.media_file.name.replace('.jpg', '') + response = self.client.get(url) + filename = response.data.name.replace('.jpg', '') thumbnail = f'{filename}-small.jpg' # Thumbs should not exist yet self.assertFalse(default_storage.exists(thumbnail)) @@ -359,14 +296,11 @@ def test_thumbnail_creation_on_demand(self): self._get_endpoint('attachment-thumb'), args=( self.asset.uid, - instance.pk, - attachment.pk, + submission['_id'], + submission['_attachments'][1]['id'], 'small' ), ) self.client.get(thumb_url) # Thumbs should exist self.assertTrue(default_storage.exists(thumbnail)) - - # Clean-up - attachment.delete() diff --git a/kpi/tests/api/v2/test_api_service_usage.py b/kpi/tests/api/v2/test_api_service_usage.py index 63826cd516..0d56a71133 100644 --- a/kpi/tests/api/v2/test_api_service_usage.py +++ b/kpi/tests/api/v2/test_api_service_usage.py @@ -9,10 +9,6 @@ from rest_framework import status from kobo.apps.kobo_auth.shortcuts import User -from kobo.apps.openrosa.apps.logger.models import ( - XForm, - DailyXFormSubmissionCounter, -) from kobo.apps.trackers.models import NLPUsageCounter from kpi.models import Asset from kpi.tests.base_test_case import BaseAssetTestCase @@ -27,7 +23,6 @@ class ServiceUsageAPIBase(BaseAssetTestCase): URL_NAMESPACE = ROUTER_URL_NAMESPACE - xform = None counter = None attachment_id = 0 @@ -69,7 +64,10 @@ def _create_asset(self, user=None): self.asset.save() self.asset.deployment.set_namespace(self.URL_NAMESPACE) - self.submission_list_url = self.asset.deployment.submission_list_url + self.submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={'format': 'json', 'parent_lookup_asset': self.asset.uid}, + ) self._deployment = self.asset.deployment def add_nlp_trackers(self): @@ -140,63 +138,9 @@ def add_submissions(self, count=2): submissions.append(submission) self.asset.deployment.mock_submissions(submissions, flush_db=False) - self.update_xform_counters(self.asset, submissions=count) - - def update_xform_counters(self, asset: Asset, submissions: int = 0): - """ - Create/update the daily submission counter and the shadow xform we use to query it - """ - today = timezone.now() - if self.xform: - self.xform.attachment_storage_bytes += ( - self.expected_file_size() * submissions - ) - self.xform.save() - else: - xform_xml = ( - f'' - f'' - f'' - f' XForm test' - f' ' - f' ' - f' <{asset.uid} id="{asset.uid}" />' - f' ' - f' ' - f'' - f'' - f'' - f'' - ) - self.xform = XForm.objects.create( - attachment_storage_bytes=( - self.expected_file_size() * submissions - ), - kpi_asset_uid=asset.uid, - date_created=today, - date_modified=today, - user_id=asset.owner_id, - xml=xform_xml, - json={} - ) - self.xform.save() - - if self.counter: - self.counter.counter += submissions - self.counter.save() - else: - self.counter = ( - DailyXFormSubmissionCounter.objects.create( - date=today.date(), - counter=submissions, - xform=self.xform, - user_id=asset.owner_id, - ) - ) - self.counter.save() - - def expected_file_size(self): + @staticmethod + def expected_file_size(): """ Calculate the expected combined file size for the test audio clip and image """ @@ -270,14 +214,16 @@ def test_multiple_forms(self): def test_service_usages_with_projects_in_trash_bin(self): self.test_multiple_forms() # Simulate trash bin - self.asset.pending_delete = True - self.asset.save( - update_fields=['pending_delete'], - create_version=False, - adjust_content=False, - ) - self.xform.pending_delete = True - self.xform.save(update_fields=['pending_delete']) + for asset in self.anotheruser.assets.all(): + asset.pending_delete = True + asset.save( + update_fields=['pending_delete'], + create_version=False, + adjust_content=False, + ) + if asset.has_deployment: + asset.deployment.xform.pending_delete = True + asset.deployment.xform.save(update_fields=['pending_delete']) # Retry endpoint url = reverse(self._get_endpoint('service-usage-list')) diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index 9a1c0409dd..9ffad5722d 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -5,7 +5,6 @@ import mock import random import string -import time import uuid from datetime import datetime try: @@ -15,13 +14,14 @@ import pytest import responses -from dict2xml import dict2xml from django.conf import settings from django.urls import reverse from django_digest.test import Client as DigestClient from rest_framework import status from kobo.apps.audit_log.models import AuditLog +from kobo.apps.openrosa.apps.main.models.user_profile import UserProfile +from kobo.apps.openrosa.libs.utils.logger_tools import dict2xform from kobo.apps.kobo_auth.shortcuts import User from kpi.constants import ( ASSET_TYPE_SURVEY, @@ -33,6 +33,7 @@ PERM_VALIDATE_SUBMISSIONS, PERM_VIEW_ASSET, PERM_VIEW_SUBMISSIONS, + SUBMISSION_FORMAT_TYPE_JSON, SUBMISSION_FORMAT_TYPE_XML, ) from kpi.models import Asset @@ -48,15 +49,9 @@ ) -def dict2xml_with_encoding_declaration(*args, **kwargs): - return '' + dict2xml( - *args, **kwargs - ) - - -def dict2xml_with_namespace(*args, **kwargs): - xml_string = dict2xml(*args, **kwargs) - xml_root = lxml.etree.fromstring(xml_string) +def dict2xform_with_namespace(submission: dict, xform_id_string: str) -> str: + xml_string = dict2xform(submission, xform_id_string) + xml_root = lxml.etree.fromstring(xml_string.encode()) xml_root.set('xmlns', 'http://opendatakit.org/submissions') return lxml.etree.tostring(xml_root).decode() @@ -73,21 +68,27 @@ class BaseSubmissionTestCase(BaseTestCase): URL_NAMESPACE = ROUTER_URL_NAMESPACE def setUp(self): - self.client.login(username="someuser", password="someuser") - self.someuser = User.objects.get(username="someuser") - self.anotheruser = User.objects.get(username="anotheruser") + self.client.login(username='someuser', password='someuser') + self.someuser = User.objects.get(username='someuser') + self.anotheruser = User.objects.get(username='anotheruser') + self.unknown_user = User.objects.create(username='unknown_user') + UserProfile.objects.create(user=self.unknown_user) + content_source_asset = Asset.objects.get(id=1) - self.asset = Asset.objects.create(content=content_source_asset.content, - owner=self.someuser, - asset_type='survey') + self.asset = Asset.objects.create( + content=content_source_asset.content, + owner=self.someuser, + asset_type='survey', + ) self.asset.deploy(backend='mock', active=True) self.asset.save() - self.__add_submissions() - self.asset.deployment.set_namespace(self.URL_NAMESPACE) - self.submission_list_url = self.asset.deployment.submission_list_url + self.submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, + ) self._deployment = self.asset.deployment def get_random_submission(self, user: settings.AUTH_USER_MODEL) -> dict: @@ -98,26 +99,20 @@ def get_random_submissions( ) -> list: """ Get random submissions within all generated submissions. - If user is the owner, we only return submissions submitted by unknown. + + If user is not the owner, we only return submissions submitted by them. It is useful to ensure restricted users fail tests with forbidden submissions. """ query = {} - if self.asset.owner == user: - query = {'_submitted_by': ''} + if self.asset.owner != user: + query = {'_submitted_by': user.username} submissions = self.asset.deployment.get_submissions(user, query=query) random.shuffle(submissions) return submissions[:limit] - def _log_in_as_another_user(self): - """ - Helper to switch user from `someuser` to `anotheruser`. - """ - self.client.logout() - self.client.login(username="anotheruser", password="anotheruser") - - def __add_submissions(self): + def _add_submissions(self, other_fields: dict = None): letters = string.ascii_letters submissions = [] v_uid = self.asset.latest_deployed_version.uid @@ -125,7 +120,7 @@ def __add_submissions(self): self.submissions_submitted_by_unknown = [] self.submissions_submitted_by_anotheruser = [] - submitted_by_choices = ['', 'someuser', 'anotheruser'] + submitted_by_choices = ['unknown_user', 'someuser', 'anotheruser'] for i in range(20): # We want to have at least one submission from each if i <= 2: @@ -139,20 +134,15 @@ def __add_submissions(self): 'q2': ''.join(random.choice(letters) for l in range(10)), 'meta/instanceID': f'uuid:{uuid_}', '_uuid': str(uuid_), - '_validation_status': { - 'by_whom': 'someuser', - 'timestamp': int(time.time()), - 'uid': 'validation_status_on_hold', - 'color': '#0000ff', - 'label': 'On Hold' - }, '_submitted_by': submitted_by } + if other_fields is not None: + submission.update(**other_fields) if submitted_by == 'someuser': self.submissions_submitted_by_someuser.append(submission) - if submitted_by == '': + if submitted_by == 'unknown_user': self.submissions_submitted_by_unknown.append(submission) if submitted_by == 'anotheruser': @@ -163,6 +153,13 @@ def __add_submissions(self): self.asset.deployment.mock_submissions(submissions) self.submissions = submissions + def _log_in_as_another_user(self): + """ + Helper to switch user from `someuser` to `anotheruser`. + """ + self.client.logout() + self.client.login(username='anotheruser', password='anotheruser') + class BulkDeleteSubmissionsApiTests(BaseSubmissionTestCase): @@ -170,6 +167,8 @@ class BulkDeleteSubmissionsApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() + + self._add_submissions() self.submission_list_url = reverse( self._get_endpoint('submission-list'), kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, @@ -187,9 +186,9 @@ def test_delete_submissions_as_owner(self): someuser can delete their own data """ data = {'payload': {'confirm': True}} - response = self.client.delete(self.submission_bulk_url, - data=data, - format='json') + response = self.client.delete( + self.submission_bulk_url, data=data, format='json' + ) self.assertEqual(response.status_code, status.HTTP_200_OK) response = self.client.get(self.submission_list_url, {'format': 'json'}) @@ -341,7 +340,7 @@ def test_delete_some_allowed_submissions_with_partial_perms_as_anotheruser(self) ) # Try first submission submitted by unknown - random_submissions = self.get_random_submissions(self.asset.owner, 3) + random_submissions = self.get_random_submissions(self.unknown_user, 3) data = { 'payload': { 'submission_ids': [rs['_id'] for rs in random_submissions] @@ -425,6 +424,10 @@ def test_cannot_delete_view_only_submissions_with_partial_perms_as_anotheruser(s class SubmissionApiTests(BaseSubmissionTestCase): + def setUp(self): + super().setUp() + self._add_submissions() + def test_cannot_create_submission(self): """ someuser is the owner of the project. @@ -459,8 +462,9 @@ def test_list_submissions_as_owner(self): """ response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('results'), self.submissions) - self.assertEqual(response.data.get('count'), len(self.submissions)) + response_ids = [r['_id'] for r in response.data.get('results')] + submissions_ids = [s['_id'] for s in self.submissions] + self.assertEqual(sorted(response_ids), sorted(submissions_ids)) def test_list_submissions_as_owner_with_params(self): """ @@ -475,7 +479,7 @@ def test_list_submissions_as_owner_with_params(self): 'limit': 5, 'sort': '{"q1": -1}', 'fields': '["q1", "_submitted_by"]', - 'query': '{"_submitted_by": {"$in": ["", "someuser", "another"]}}', + 'query': '{"_submitted_by": {"$in": ["unknown", "someuser", "another"]}}', } ) # ToDo add more assertions. E.g. test whether sort, limit, start really work @@ -492,7 +496,7 @@ def test_list_submissions_limit(self): asset = Asset.objects.create( name='Lots of submissions', owner=self.asset.owner, - content={'survey': [{'name': 'q', 'type': 'integer'}]}, + content={'survey': [{'label': 'q', 'type': 'integer'}]}, ) asset.deploy(backend='mock', active=True) asset.deployment.set_namespace(self.URL_NAMESPACE) @@ -504,17 +508,19 @@ def test_list_submissions_limit(self): } for i in range(limit + excess) ] asset.deployment.mock_submissions(submissions) + submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={'parent_lookup_asset': asset.uid, 'format': 'json'}, + ) # Server-wide limit should apply if no limit specified - response = self.client.get( - asset.deployment.submission_list_url, {'format': 'json'} - ) + response = self.client.get(submission_list_url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response.data['results']), limit) # Limit specified in query parameters should not be able to exceed # server-wide limit response = self.client.get( - asset.deployment.submission_list_url, + submission_list_url, {'limit': limit + excess, 'format': 'json'} ) @@ -541,8 +547,9 @@ def test_list_submissions_shared_as_anotheruser(self): self._log_in_as_another_user() response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data.get('results'), self.submissions) - self.assertEqual(response.data.get('count'), len(self.submissions)) + response_ids = [r['_id'] for r in response.data.get('results')] + submissions_ids = [s['_id'] for s in self.submissions] + self.assertEqual(sorted(response_ids), sorted(submissions_ids)) def test_list_submissions_with_partial_permissions_as_anotheruser(self): """ @@ -663,18 +670,23 @@ def test_list_query_elem_match(self): """ Ensure query is able to filter on an array """ - submission = self.submissions[0] + submission = copy.deepcopy(self.submissions_submitted_by_someuser[0]) + del submission['_id'] + uuid_ = str(uuid.uuid4()) + submission['meta/instanceID'] = f'uuid:{uuid_}' + submission['_uuid'] = str(uuid_) group = 'group_lx4sf58' question = 'q3' submission[group] = [ { - f'{group}/{question}': 'whap.gif', + f'{question}': 'whap.gif', }, ] - self.asset.deployment.mock_submissions(self.submissions) + self.asset.deployment.mock_submissions([submission]) + # FIXME with attachments data = { - 'query': f'{{"{group}":{{"$elemMatch":{{"{group}/{question}":{{"$exists":true}}}}}}}}', + 'query': f'{{"{group}/{question}":{{"$exists":true}}}}', 'format': 'json', } response = self.client.get(self.submission_list_url, data) @@ -691,9 +703,14 @@ def test_retrieve_submission_as_owner(self): someuser can view one of their submission. """ submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) - - response = self.client.get(url, {"format": "json"}) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, submission) @@ -703,11 +720,17 @@ def test_retrieve_submission_by_uuid(self): someuser can view one of their submission. """ submission = self.submissions[0] - url = self.asset.deployment.get_submission_detail_url(submission['_uuid']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_uuid'], + }, + ) response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, submission) + self.assertEqual(response.data['_id'], submission['_id']) def test_retrieve_submission_not_shared_as_anotheruser(self): """ @@ -716,9 +739,15 @@ def test_retrieve_submission_not_shared_as_anotheruser(self): someuser's data existence should not be revealed. """ self._log_in_as_another_user() - submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) - response = self.client.get(url, {"format": "json"}) + submission = self.get_random_submission(self.unknown_user) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) def test_retrieve_submission_shared_as_anotheruser(self): @@ -728,9 +757,15 @@ def test_retrieve_submission_shared_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() - submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) - response = self.client.get(url, {"format": "json"}) + submission = self.get_random_submission(self.unknown_user) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, submission) @@ -748,15 +783,27 @@ def test_retrieve_submission_with_partial_permissions_as_anotheruser(self): partial_perms=partial_perms) # Try first submission submitted by unknown - submission = self.get_random_submission(self.asset.owner) - url = self._deployment.get_submission_detail_url(submission['_id']) - response = self.client.get(url, {"format": "json"}) + submission = self.get_random_submission(self.unknown_user) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) # Try second submission submitted by another submission = self.submissions_submitted_by_anotheruser[0] - url = self._deployment.get_submission_detail_url(submission['_id']) - response = self.client.get(url, {"format": "json"}) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) def test_delete_submission_as_owner(self): @@ -765,7 +812,13 @@ def test_delete_submission_as_owner(self): someuser can delete their own data. """ submission = self.submissions_submitted_by_someuser[0] - url = self.asset.deployment.get_submission_detail_url(submission['_id']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) @@ -800,7 +853,13 @@ def test_delete_not_existing_submission_as_owner(self): someuser should receive a 404 if they try to delete a non-existing submission. """ - url = self.asset.deployment.get_submission_detail_url(9999) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': 9999, + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -813,7 +872,13 @@ def test_delete_submission_as_anonymous(self): """ self.client.logout() submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -826,8 +891,14 @@ def test_delete_submission_not_shared_as_anotheruser(self): someuser's data existence should not be revealed. """ self._log_in_as_another_user() - submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) + submission = self.get_random_submission(self.unknown_user) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -840,8 +911,14 @@ def test_delete_submission_shared_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() - submission = self.get_random_submission(self.asset.owner) - url = self.asset.deployment.get_submission_detail_url(submission['_id']) + submission = self.get_random_submission(self.unknown_user) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.delete(url, HTTP_ACCEPT='application/json') self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -878,19 +955,31 @@ def test_delete_submission_with_partial_perms_as_anotheruser(self): # Try first submission submitted by unknown submission = self.submissions_submitted_by_unknown[0] - url = self._deployment.get_submission_detail_url(submission['_id']) - response = self.client.delete(url, - content_type='application/json', - HTTP_ACCEPT='application/json') + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.delete( + url, content_type='application/json', HTTP_ACCEPT='application/json' + ) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try second submission submitted by anotheruser anotheruser_submission_count = len(self.submissions_submitted_by_anotheruser) submission = self.get_random_submission(self.anotheruser) - url = self._deployment.get_submission_detail_url(submission['_id']) - response = self.client.delete(url, - content_type='application/json', - HTTP_ACCEPT='application/json') + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) + response = self.client.delete( + url, content_type='application/json', HTTP_ACCEPT='application/json' + ) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) response = self.client.get(self.submission_list_url, {'format': 'json'}) self.assertEqual( @@ -953,7 +1042,6 @@ def test_attachments_rewrite(self): v_uid = asset.latest_deployed_version.uid submission = { - '_id': 1000, '__version__': v_uid, '_xform_id_string': asset.uid, 'formhub/uuid': 'formhub-uuid', @@ -1018,7 +1106,13 @@ def test_attachments_rewrite(self): asset.deployment.set_namespace(self.URL_NAMESPACE) self._log_in_as_another_user() - url = asset.deployment.get_submission_detail_url(submission['_id']) + url = reverse( + self._get_endpoint('submission-detail'), + kwargs={ + 'parent_lookup_asset': asset.uid, + 'pk': submission['_id'], + }, + ) response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -1051,7 +1145,9 @@ class SubmissionEditApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() - self.submission = self.get_random_submission(self.asset.owner) + + self._add_submissions() + self.submission = self.submissions_submitted_by_someuser[0] self.submission_url_legacy = reverse( self._get_endpoint('submission-enketo-edit'), kwargs={ @@ -1191,7 +1287,7 @@ def test_get_edit_link_with_partial_perms_as_anotheruser(self): ) # Try first submission submitted by unknown - submission = self.get_random_submission(self.asset.owner) + submission = self.get_random_submission(self.unknown_user) url = reverse( self._get_endpoint('submission-enketo-edit'), kwargs={ @@ -1408,61 +1504,65 @@ def test_edit_submission_with_different_root_name(self): @responses.activate def test_edit_submission_with_xml_encoding_declaration(self): - with mock.patch( - 'kpi.deployment_backends.mock_backend.dict2xml' - ) as mock_dict2xml: - mock_dict2xml.side_effect = dict2xml_with_encoding_declaration - submission = self.submissions[-1] - submission_xml = self.asset.deployment.get_submissions( - user=self.asset.owner, - format_type=SUBMISSION_FORMAT_TYPE_XML, - submission_ids=[submission['_id']], - )[0] - assert submission_xml.startswith( - '' - ) + submission = self.submissions[-1] + submission_xml = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_XML, + submission_ids=[submission['_id']], + )[0] + assert submission_xml.startswith( + '' + ) - # Get edit endpoint - edit_url = reverse( - self._get_endpoint('submission-enketo-edit'), - kwargs={ - 'parent_lookup_asset': self.asset.uid, - 'pk': submission['_id'], - }, - ) + # Get edit endpoint + edit_url = reverse( + self._get_endpoint('submission-enketo-edit'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, + ) - # Set up a mock Enketo response and attempt the edit request - ee_url = f'{settings.ENKETO_URL}/{settings.ENKETO_EDIT_INSTANCE_ENDPOINT}' - responses.add_callback( - responses.POST, - ee_url, - callback=enketo_edit_instance_response_with_uuid_validation, - content_type='application/json', - ) - response = self.client.get(edit_url, {'format': 'json'}) - assert response.status_code == status.HTTP_200_OK + # Set up a mock Enketo response and attempt the edit request + ee_url = f'{settings.ENKETO_URL}/{settings.ENKETO_EDIT_INSTANCE_ENDPOINT}' + responses.add_callback( + responses.POST, + ee_url, + callback=enketo_edit_instance_response_with_uuid_validation, + content_type='application/json', + ) + response = self.client.get(edit_url, {'format': 'json'}) + assert response.status_code == status.HTTP_200_OK @responses.activate def test_edit_submission_with_xml_missing_uuids(self): # Make a new submission without UUIDs submission = copy.deepcopy(self.submissions[-1]) submission['_id'] += 1 + del submission['meta/instanceID'] + del submission['_uuid'] + submission['find_this'] = 'hello!' # The form UUID is already omitted by these tests, but fail if that # changes in the future assert 'formhub/uuid' not in submission.keys() - self.asset.deployment.mock_submissions([submission]) + self.asset.deployment.mock_submissions([submission], create_uuids=False) # Find and verify the new submission submission_xml = self.asset.deployment.get_submissions( user=self.asset.owner, format_type=SUBMISSION_FORMAT_TYPE_XML, - find_this='hello!', + query={"find_this": "hello!"}, + )[0] + submission_json = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_JSON, + query={"find_this": "hello!"}, )[0] - submission_xml_root = lxml.etree.fromstring(submission_xml) - submission_id = int(submission_xml_root.find('./_id').text) - assert submission_id == submission['_id'] + + submission_xml_root = lxml.etree.fromstring(submission_xml.encode()) + assert submission_json['_id'] == submission['_id'] assert submission_xml_root.find('./find_this').text == 'hello!' assert submission_xml_root.find('./meta/instanceID') is None assert submission_xml_root.find('./formhub/uuid') is None @@ -1472,7 +1572,7 @@ def test_edit_submission_with_xml_missing_uuids(self): self._get_endpoint('submission-enketo-edit'), kwargs={ 'parent_lookup_asset': self.asset.uid, - 'pk': submission_id, + 'pk': submission_json['_id'], }, ) @@ -1515,9 +1615,10 @@ def test_get_edit_link_submission_with_latest_asset_deployment(self): { 'type': 'note', 'name': 'n', - 'label': 'A new note', + 'label': ['A new note'], } ) + self.asset.save() assert self.asset.asset_versions.count() == original_versions_count + 1 assert ( @@ -1558,7 +1659,8 @@ class SubmissionViewApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() - self.submission = self.get_random_submission(self.asset.owner) + self._add_submissions() + self.submission = self.submissions_submitted_by_someuser[0] self.submission_view_link_url = reverse( self._get_endpoint('submission-enketo-view'), kwargs={ @@ -1699,22 +1801,16 @@ def test_get_view_link_with_partial_perms_as_anotheruser(self): self.assertEqual(response.data, expected_response) -class SubmissionDuplicateApiTests(BaseSubmissionTestCase): +class SubmissionDuplicateBaseApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() current_time = datetime.now(tz=ZoneInfo('UTC')).isoformat('T', 'milliseconds') - # TODO: also test a submission that's missing `start` or `end`; see - # #3054. Right now that would be useless, though, because the - # MockDeploymentBackend doesn't use XML at all and won't fail if an - # expected field is missing - for submission in self.submissions: - submission['start'] = current_time - submission['end'] = current_time - - self.asset.deployment.mock_submissions(self.submissions) + self._add_submissions( + other_fields={'start': current_time, 'end': current_time} + ) - self.submission = self.get_random_submission(self.asset.owner) + self.submission = self.submissions_submitted_by_someuser[0] self.submission_url = reverse( self._get_endpoint('submission-duplicate'), kwargs={ @@ -1735,11 +1831,43 @@ def _check_duplicate(self, response, submission: dict = None): expected_next_id = max((sub['_id'] for sub in self.submissions)) + 1 assert submission['_id'] != duplicate_submission['_id'] assert duplicate_submission['_id'] == expected_next_id + assert submission['meta/instanceID'] != duplicate_submission['meta/instanceID'] - assert submission['meta/instanceID'] == duplicate_submission['meta/deprecatedID'] assert submission['start'] != duplicate_submission['start'] assert submission['end'] != duplicate_submission['end'] + +class SubmissionDuplicateWithXMLNamespaceApiTests( + SubmissionDuplicateBaseApiTests +): + + def setUp(self): + with mock.patch( + 'kpi.deployment_backends.mock_backend.dict2xform' + ) as mock_dict2xform: + mock_dict2xform.side_effect = dict2xform_with_namespace + super().setUp() + + def test_duplicate_submission_with_xml_namespace(self): + + submission_xml = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_XML, + submission_ids=[self.submission['_id']], + )[0] + assert ( + 'xmlns="http://opendatakit.org/submissions"' in submission_xml + ) + response = self.client.post(self.submission_url, {'format': 'json'}) + assert response.status_code == status.HTTP_201_CREATED + self._check_duplicate(response) + + +class SubmissionDuplicateApiTests(SubmissionDuplicateBaseApiTests): + + def setUp(self): + super().setUp() + def test_duplicate_submission_as_owner_allowed(self): """ someuser is the owner of the project. @@ -1750,34 +1878,15 @@ def test_duplicate_submission_as_owner_allowed(self): self._check_duplicate(response) def test_duplicate_submission_with_xml_encoding(self): - with mock.patch( - 'kpi.deployment_backends.mock_backend.dict2xml' - ) as mock_dict2xml: - mock_dict2xml.side_effect = dict2xml_with_encoding_declaration - submission_xml = self.asset.deployment.get_submissions( - user=self.asset.owner, - format_type=SUBMISSION_FORMAT_TYPE_XML, - submission_ids=[self.submission['_id']], - )[0] - assert submission_xml.startswith( - '' - ) - self.test_duplicate_submission_as_owner_allowed() - - def test_duplicate_submission_with_xml_namespace(self): - with mock.patch( - 'kpi.deployment_backends.mock_backend.dict2xml' - ) as mock_dict2xml: - mock_dict2xml.side_effect = dict2xml_with_namespace - submission_xml = self.asset.deployment.get_submissions( - user=self.asset.owner, - format_type=SUBMISSION_FORMAT_TYPE_XML, - submission_ids=[self.submission['_id']], - )[0] - assert ( - 'xmlns="http://opendatakit.org/submissions"' in submission_xml - ) - self.test_duplicate_submission_as_owner_allowed() + submission_xml = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_XML, + submission_ids=[self.submission['_id']], + )[0] + assert submission_xml.startswith( + '' + ) + self.test_duplicate_submission_as_owner_allowed() def test_duplicate_submission_as_anotheruser_not_allowed(self): """ @@ -1861,7 +1970,7 @@ def test_duplicate_submission_as_anotheruser_with_partial_perms(self): ) # Try first submission submitted by unknown - submission = self.get_random_submission(self.asset.owner) + submission = self.get_random_submission(self.unknown_user) url = reverse( self._get_endpoint('submission-duplicate'), kwargs={ @@ -1890,6 +1999,7 @@ class BulkUpdateSubmissionsApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() + self._add_submissions() self.submission_url = reverse( self._get_endpoint('submission-bulk'), kwargs={ @@ -1933,22 +2043,18 @@ def test_bulk_update_submissions_allowed_as_owner(self): ) ) def test_bulk_update_submissions_with_xml_encoding(self): - with mock.patch( - 'kpi.deployment_backends.mock_backend.dict2xml' - ) as mock_dict2xml: - mock_dict2xml.side_effect = dict2xml_with_encoding_declaration - submission = self.submissions[ - self.updated_submission_data['submission_ids'][-1] - ] - submission_xml = self.asset.deployment.get_submissions( - user=self.asset.owner, - format_type=SUBMISSION_FORMAT_TYPE_XML, - submission_ids=[submission['_id']], - )[0] - assert submission_xml.startswith( - '' - ) - self.test_bulk_update_submissions_allowed_as_owner() + submission = self.submissions[ + self.updated_submission_data['submission_ids'][-1] + ] + submission_xml = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_XML, + submission_ids=[submission['_id']], + )[0] + assert submission_xml.startswith( + '' + ) + self.test_bulk_update_submissions_allowed_as_owner() @pytest.mark.skip( reason=( @@ -1958,9 +2064,9 @@ def test_bulk_update_submissions_with_xml_encoding(self): ) def test_bulk_update_submissions_with_xml_namespace(self): with mock.patch( - 'kpi.deployment_backends.mock_backend.dict2xml' - ) as mock_dict2xml: - mock_dict2xml.side_effect = dict2xml_with_namespace + 'kpi.deployment_backends.mock_backend.dict2xform' + ) as mock_dict2xform: + mock_dict2xform.side_effect = dict2xform_with_namespace submission = self.submissions[ self.updated_submission_data['submission_ids'][-1] ] @@ -2070,11 +2176,14 @@ class SubmissionValidationStatusApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() - self.submission = self.get_random_submission(self.asset.owner) - self.validation_status_url = ( - self._deployment.get_submission_validation_status_url( - self.submission['_id'] - ) + self._add_submissions() + self.submission = self.submissions_submitted_by_someuser[0] + self.validation_status_url = reverse( + self._get_endpoint('submission-validation-status'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': self.submission['_id'], + }, ) def test_retrieve_status_as_owner(self): @@ -2084,7 +2193,7 @@ def test_retrieve_status_as_owner(self): """ response = self.client.get(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, self.submission.get("_validation_status")) + self.assertEqual(response.data, {}) def test_cannot_retrieve_status_of_not_shared_submission_as_anotheruser(self): """ @@ -2108,7 +2217,7 @@ def test_retrieve_status_of_shared_submission_as_anotheruser(self): self._log_in_as_another_user() response = self.client.get(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, self.submission.get("_validation_status")) + self.assertEqual(response.data, {}) def test_cannot_retrieve_status_of_shared_submission_as_anonymous(self): """ @@ -2247,28 +2356,35 @@ def test_edit_status_with_partial_perms_as_anotheruser(self): PERM_VALIDATE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } # Allow anotheruser to validate someuser's data - self.asset.assign_perm(self.anotheruser, PERM_PARTIAL_SUBMISSIONS, - partial_perms=partial_perms) + self.asset.assign_perm( + self.anotheruser, + PERM_PARTIAL_SUBMISSIONS, + partial_perms=partial_perms, + ) data = { 'validation_status.uid': 'validation_status_not_approved' } # Try first submission submitted by unknown submission = self.submissions_submitted_by_unknown[0] - url = ( - self._deployment.get_submission_validation_status_url( - submission['_id'] - ) + url = reverse( + self._get_endpoint('submission-validation-status'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, ) response = self.client.patch(url, data=data) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try second submission submitted by anotheruser submission = self.submissions_submitted_by_anotheruser[0] - url = ( - self._deployment.get_submission_validation_status_url( - submission['_id'] - ) + url = reverse( + self._get_endpoint('submission-validation-status'), + kwargs={ + 'parent_lookup_asset': self.asset.uid, + 'pk': submission['_id'], + }, ) response = self.client.patch(url, data=data) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2284,9 +2400,7 @@ class SubmissionValidationStatusesApiTests(BaseSubmissionTestCase): def setUp(self): super().setUp() - for submission in self.submissions: - submission['_validation_status']['uid'] = 'validation_status_not_approved' - self.asset.deployment.mock_submissions(self.submissions) + self._add_submissions() self.validation_statuses_url = reverse( self._get_endpoint('submission-validation-statuses'), kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, @@ -2296,6 +2410,18 @@ def setUp(self): kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, ) + # Make the owner change validation status of all submissions + data = { + 'payload': { + 'validation_status.uid': 'validation_status_not_approved', + 'confirm': True, + } + } + response = self.client.patch( + self.validation_statuses_url, data=data, format='json' + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + def test_delete_all_status_as_owner(self): """ someuser is the owner of the project. @@ -2597,8 +2723,11 @@ def test_edit_all_submission_validation_statuses_with_partial_perms_as_anotherus {'_submitted_by': 'anotheruser'}] } # Allow anotheruser to validate their own data - self.asset.assign_perm(self.anotheruser, PERM_PARTIAL_SUBMISSIONS, - partial_perms=partial_perms) + self.asset.assign_perm( + self.anotheruser, + PERM_PARTIAL_SUBMISSIONS, + partial_perms=partial_perms, + ) data = { 'payload': { 'validation_status.uid': 'validation_status_approved', @@ -2607,20 +2736,19 @@ def test_edit_all_submission_validation_statuses_with_partial_perms_as_anotherus } # Update all submissions anotheruser is allowed to edit - response = self.client.patch(self.validation_statuses_url, - data=data, - format='json') + response = self.client.patch( + self.validation_statuses_url, data=data, format='json' + ) self.assertEqual(response.status_code, status.HTTP_200_OK) - count = self._deployment.calculated_submission_count( - self.anotheruser) + count = self._deployment.calculated_submission_count(self.anotheruser) expected_response = {'detail': f'{count} submissions have been updated'} self.assertEqual(response.data, expected_response) # Get all submissions and ensure only the ones that anotheruser is # allowed to edit have been modified self.client.logout() - self.client.login(username="someuser", password="someuser") + self.client.login(username='someuser', password='someuser') response = self.client.get(self.submission_list_url) for submission in response.data['results']: validation_status = submission['_validation_status'] @@ -2751,7 +2879,12 @@ def setUp(self): ] a.deployment.mock_submissions(self.submissions) a.deployment.set_namespace(self.URL_NAMESPACE) - self.submission_list_url = a.deployment.submission_list_url + self.submission_list_url = reverse( + self._get_endpoint('submission-list'), + kwargs={ + 'parent_lookup_asset': a.uid, + }, + ) def test_list_submissions_geojson_defaults(self): response = self.client.get( diff --git a/kpi/tests/kpi_test_case.py b/kpi/tests/kpi_test_case.py index 5c90b2b79f..2b005d6c54 100644 --- a/kpi/tests/kpi_test_case.py +++ b/kpi/tests/kpi_test_case.py @@ -105,8 +105,9 @@ def create_collection(self, name, owner=None, owner_password=None, collection = self.url_to_obj(response.data['url']) return collection - def create_asset(self, name, content=None, owner=None, - owner_password=None, **kwargs): + def create_asset( + self, name, content=None, owner=None, owner_password=None, **kwargs + ): if owner and owner_password: if isinstance(owner, str): self.login(owner, owner_password) diff --git a/kpi/tests/test_asset_versions.py b/kpi/tests/test_asset_versions.py index 905d9ed5bb..a1ddb8d26a 100644 --- a/kpi/tests/test_asset_versions.py +++ b/kpi/tests/test_asset_versions.py @@ -9,8 +9,9 @@ from django.test import TestCase from django.utils import timezone - from formpack.utils.expand_content import SCHEMA_VERSION + +from kobo.apps.kobo_auth.shortcuts import User from kpi.exceptions import BadAssetTypeException from kpi.utils.hash import calculate_hash from ..models import Asset @@ -57,23 +58,27 @@ def test_init_asset_version(self): self.assertEqual(av_count + 2, AssetVersion.objects.count()) def test_asset_deployment(self): - self.asset = Asset.objects.create(asset_type='survey', content={ - 'survey': [{'type': 'note', 'label': 'Read me', 'name': 'n1'}] - }) + bob = User.objects.create(username='bob') + self.asset = Asset.objects.create( + asset_type='survey', + content={ + 'survey': [{'type': 'note', 'label': ['Read me'], 'name': 'n1'}] + }, + owner=bob + ) self.assertEqual(self.asset.asset_versions.count(), 1) self.assertEqual(self.asset.latest_version.deployed, False) - self.asset.content['survey'].append({'type': 'note', - 'label': 'Read me 2', - 'name': 'n2'}) + self.asset.content['survey'].append( + {'type': 'note', 'label': ['Read me 2'], 'name': 'n2'} + ) self.asset.save() self.assertEqual(self.asset.asset_versions.count(), 2) v2 = self.asset.latest_version self.assertEqual(self.asset.latest_version.deployed, False) self.asset.deploy(backend='mock', active=True) - self.asset.save(create_version=False, - adjust_content=False) + self.asset.save(create_version=False, adjust_content=False) # version did not increment self.assertEqual(self.asset.asset_versions.count(), 2) diff --git a/kpi/tests/test_deployment_backends.py b/kpi/tests/test_deployment_backends.py index 9763691edc..2a8f63126d 100644 --- a/kpi/tests/test_deployment_backends.py +++ b/kpi/tests/test_deployment_backends.py @@ -2,6 +2,7 @@ import pytest from django.test import TestCase +from kobo.apps.kobo_auth.shortcuts import User from kpi.exceptions import DeploymentDataException from kpi.models.asset import Asset from kpi.models.asset_version import AssetVersion @@ -9,12 +10,11 @@ class CreateDeployment(TestCase): def setUp(self): - self.asset = Asset(content={ - 'survey': [ - {'type':'text', 'name': 'q1', - 'label': 'Q1.',} - ] - }) + someuser = User.objects.create(username='someuser') + self.asset = Asset( + content={'survey': [{'type': 'text', 'name': 'q1', 'label': 'Q1.'}]}, + owner=someuser, + ) def test_invalid_backend_fails(self): self.asset.save() @@ -33,15 +33,20 @@ def test_mock_deployment_inits(self): @pytest.mark.django_db def test_initial_kuids(): initial_kuid = 'aaaa1111' - asset = Asset.objects.create(content={ - 'survey': [ - {'type': 'text', - 'name': 'q1', - 'label': 'Q1.', - '$kuid': initial_kuid, - } + someuser = User.objects.create(username='someuser') + asset = Asset.objects.create( + content={ + 'survey': [ + { + 'type': 'text', + 'name': 'q1', + 'label': 'Q1.', + '$kuid': initial_kuid, + } ] - }) + }, + owner=someuser, + ) assert asset.content['survey'][0]['$kuid'] == initial_kuid asset.deploy(backend='mock', active=False) @@ -53,13 +58,11 @@ def test_initial_kuids(): class MockDeployment(TestCase): def setUp(self): - self.asset = Asset.objects.create(content={ - 'survey': [ - {'type': 'text', 'name': 'q1', - 'label': 'Q1.' - } - ] - }) + someuser = User.objects.create(username='someuser') + self.asset = Asset.objects.create( + content={'survey': [{'type': 'text', 'name': 'q1', 'label': 'Q1.'}]}, + owner=someuser, + ) self.asset.deploy(backend='mock', active=False) self.asset.save() diff --git a/kpi/tests/test_mock_data.py b/kpi/tests/test_mock_data.py index 47e4bcb05f..a7de62bc7d 100644 --- a/kpi/tests/test_mock_data.py +++ b/kpi/tests/test_mock_data.py @@ -9,113 +9,250 @@ from kobo.apps.reports import report_data from kpi.models import Asset -F1 = {'survey': [{'$kuid': 'Uf89NP4VX', 'type': 'start', 'name': 'start'}, - {'$kuid': 'ZtZBY7XHX', 'type': 'end', 'name': 'end'}, - {'name': 'Select_one', 'select_from_list_name': 'choice_list_1', 'required': 'true', - 'label': ['Select one', 'Seleccione uno', - '\u0627\u062e\u062a\u0631 \u0648\u0627\u062d\u062f\u0627'], '$kuid': 'WXOeQ4Nc0', - 'type': 'select_one'}, - {'name': 'Select_Many', 'select_from_list_name': 'choice_list_2', 'required': 'true', - 'label': ['Select Many', 'Muchos seleccione', - '\u0627\u062e\u062a\u0631 \u0627\u0644\u0639\u062f\u064a\u062f'], '$kuid': 'BC6BNP91R', - 'type': 'select_multiple'}, - {'$kuid': '0e7sTrQzo', 'required': 'true', 'type': 'text', 'name': 'Text', - 'label': ['Text', 'Texto', '\u0646\u0635']}, - {'$kuid': 'ZzKb8DeQu', 'required': 'true', 'type': 'integer', 'name': 'Number', - 'label': ['Number', 'N\xfamero', '\u0639\u062f\u062f']}, - {'$kuid': 'gLEDxsNZo', 'required': 'true', 'type': 'decimal', 'name': 'Decimal', - 'label': ['Decimal', 'Decimal', '\u0639\u062f\u062f \u0639\u0634\u0631\u064a']}, - {'$kuid': 'pt2w8z3Xk', 'required': 'true', 'type': 'date', 'name': 'Date', - 'label': ['Date', 'Fecha', '\u062a\u0627\u0631\u064a\u062e']}, - {'$kuid': '3xn0tP9AI', 'required': 'true', 'type': 'time', 'name': 'Time', - 'label': ['Time', 'Hora', '\u0645\u0631\u0629']}, - {'$kuid': 'w0nYPBtT0', 'required': 'true', 'type': 'datetime', 'name': 'Date_and_time', - 'label': ['Date and time', 'Fecha y hora', - '\u0627\u0644\u062a\u0627\u0631\u064a\u062e \u0648 \u0627\u0644\u0648\u0642\u062a']}, - {'$kuid': '0dovjhXG6', 'required': 'false', 'type': 'geopoint', 'name': 'GPS', - 'label': ['GPS', 'GPS', - '\u0646\u0638\u0627\u0645 \u062a\u062d\u062f\u064a\u062f \u0627\u0644\u0645\u0648\u0627\u0642\u0639']}, - {'$kuid': 'NI2fsrYZI', 'required': 'true', 'type': 'image', 'name': 'Photo', - 'label': ['Photo', 'Foto', - '\u0635\u0648\u0631\u0629 \u0641\u0648\u062a\u0648\u063a\u0631\u0627\u0641\u064a\u0629']}, - {'$kuid': 'FlfOVztW3', 'required': 'true', 'type': 'audio', 'name': 'Audio', - 'label': ['Audio', 'Audio', '\u0633\u0645\u0639\u064a']}, - {'$kuid': 'GdNV76Ily', 'required': 'true', 'type': 'video', 'name': 'Video', - 'label': ['Video', 'V\xeddeo', '\u0641\u064a\u062f\u064a\u0648']}, - {'$kuid': 'EDuWkTREB', 'required': 'false', 'type': 'note', - 'name': 'Note_Should_not_be_displayed', - 'label': ['Note (Should not be displayed!)', 'Nota (no se represente!)', - '\u0645\u0644\u0627\u062d\u0638\u0629 (\u064a\u062c\u0628 \u0623\u0646 \u0644\u0627 \u064a\u062a\u0645 \u0639\u0631\u0636!)']}, - {'$kuid': 'hwik7tNXF', 'required': 'true', 'type': 'barcode', 'name': 'Barcode', - 'label': ['Barcode', 'C\xf3digo de barras', '\u0627\u0644\u0628\u0627\u0631\u0643\u0648\u062f']}, - {'$kuid': 'NTBElbRcj', 'required': 'true', 'type': 'acknowledge', 'name': 'Acknowledge', - 'label': ['Acknowledge', 'Reconocer', '\u0627\u0639\u062a\u0631\u0641']}, - {'calculation': '1', '$kuid': 'x6zr1MtmP', 'required': 'false', 'type': 'calculate', - 'name': 'calculation'}], 'translations': [None, 'Espa\xf1ol', 'Arabic'], 'choices': [ - {'$kuid': 'xm4h0m4kK', 'list_name': 'choice_list_1', 'name': 'option_1', - 'label': ['First option', 'Primera opci\xf3n', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644']}, - {'$kuid': 'slcf0IezR', 'list_name': 'choice_list_1', 'name': 'option_2', - 'label': ['Second option', 'Segunda opci\xf3n', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a']}, - {'$kuid': 'G7myzY2qX', 'list_name': 'choice_list_2', 'name': 'option_1', - 'label': ['First option', 'Primera opci\xf3n', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644']}, - {'$kuid': 'xUd28PPBs', 'list_name': 'choice_list_2', 'name': 'option_2', - 'label': ['Second option', 'Segunda opci\xf3n', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a']}]} +F1 = { + 'survey': [ + {'$kuid': 'Uf89NP4VX', 'type': 'start', 'name': 'start'}, + {'$kuid': 'ZtZBY7XHX', 'type': 'end', 'name': 'end'}, + { + 'name': 'Select_one', + 'select_from_list_name': 'choice_list_1', + 'required': 'true', + 'label': [ + 'Select one', + 'Seleccione uno', + '\u0627\u062e\u062a\u0631 \u0648\u0627\u062d\u062f\u0627', + ], + '$kuid': 'WXOeQ4Nc0', + 'type': 'select_one', + }, + { + 'name': 'Select_Many', + 'select_from_list_name': 'choice_list_2', + 'required': 'true', + 'label': [ + 'Select Many', + 'Muchos seleccione', + '\u0627\u062e\u062a\u0631 \u0627\u0644\u0639\u062f\u064a\u062f', + ], + '$kuid': 'BC6BNP91R', + 'type': 'select_multiple', + }, + { + '$kuid': '0e7sTrQzo', + 'required': 'true', + 'type': 'text', + 'name': 'Text', + 'label': ['Text', 'Texto', '\u0646\u0635'], + }, + { + '$kuid': 'ZzKb8DeQu', + 'required': 'true', + 'type': 'integer', + 'name': 'Number', + 'label': ['Number', 'N\xfamero', '\u0639\u062f\u062f'], + }, + { + '$kuid': 'gLEDxsNZo', + 'required': 'true', + 'type': 'decimal', + 'name': 'Decimal', + 'label': [ + 'Decimal', + 'Decimal', + '\u0639\u062f\u062f \u0639\u0634\u0631\u064a', + ], + }, + { + '$kuid': 'pt2w8z3Xk', + 'required': 'true', + 'type': 'date', + 'name': 'Date', + 'label': ['Date', 'Fecha', '\u062a\u0627\u0631\u064a\u062e'], + }, + { + '$kuid': '3xn0tP9AI', + 'required': 'true', + 'type': 'time', + 'name': 'Time', + 'label': ['Time', 'Hora', '\u0645\u0631\u0629'], + }, + { + '$kuid': 'w0nYPBtT0', + 'required': 'true', + 'type': 'datetime', + 'name': 'Date_and_time', + 'label': [ + 'Date and time', + 'Fecha y hora', + '\u0627\u0644\u062a\u0627\u0631\u064a\u062e \u0648 \u0627\u0644\u0648\u0642\u062a', + ], + }, + { + '$kuid': '0dovjhXG6', + 'required': 'false', + 'type': 'geopoint', + 'name': 'GPS', + 'label': [ + 'GPS', + 'GPS', + '\u0646\u0638\u0627\u0645 \u062a\u062d\u062f\u064a\u062f \u0627\u0644\u0645\u0648\u0627\u0642\u0639', + ], + }, + { + '$kuid': 'NI2fsrYZI', + 'required': 'true', + 'type': 'image', + 'name': 'Photo', + 'label': [ + 'Photo', + 'Foto', + '\u0635\u0648\u0631\u0629 \u0641\u0648\u062a\u0648\u063a\u0631\u0627\u0641\u064a\u0629', + ], + }, + { + '$kuid': 'FlfOVztW3', + 'required': 'true', + 'type': 'audio', + 'name': 'Audio', + 'label': ['Audio', 'Audio', '\u0633\u0645\u0639\u064a'], + }, + { + '$kuid': 'GdNV76Ily', + 'required': 'true', + 'type': 'video', + 'name': 'Video', + 'label': ['Video', 'V\xeddeo', '\u0641\u064a\u062f\u064a\u0648'], + }, + { + '$kuid': 'EDuWkTREB', + 'required': 'false', + 'type': 'note', + 'name': 'Note_Should_not_be_displayed', + 'label': [ + 'Note (Should not be displayed!)', + 'Nota (no se represente!)', + '\u0645\u0644\u0627\u062d\u0638\u0629 (\u064a\u062c\u0628 \u0623\u0646 \u0644\u0627 \u064a\u062a\u0645 \u0639\u0631\u0636!)', + ], + }, + { + '$kuid': 'hwik7tNXF', + 'required': 'true', + 'type': 'barcode', + 'name': 'Barcode', + 'label': [ + 'Barcode', + 'C\xf3digo de barras', + '\u0627\u0644\u0628\u0627\u0631\u0643\u0648\u062f', + ], + }, + { + '$kuid': 'NTBElbRcj', + 'required': 'true', + 'type': 'acknowledge', + 'name': 'Acknowledge', + 'label': [ + 'Acknowledge', + 'Reconocer', + '\u0627\u0639\u062a\u0631\u0641', + ], + }, + { + 'calculation': '1', + '$kuid': 'x6zr1MtmP', + 'required': 'false', + 'type': 'calculate', + 'name': 'calculation', + }, + ], + 'translations': [None, 'Espa\xf1ol', 'Arabic'], + 'choices': [ + { + '$kuid': 'xm4h0m4kK', + 'list_name': 'choice_list_1', + 'name': 'option_1', + 'label': [ + 'First option', + 'Primera opci\xf3n', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', + ], + }, + { + '$kuid': 'slcf0IezR', + 'list_name': 'choice_list_1', + 'name': 'option_2', + 'label': [ + 'Second option', + 'Segunda opci\xf3n', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a', + ], + }, + { + '$kuid': 'G7myzY2qX', + 'list_name': 'choice_list_2', + 'name': 'option_1', + 'label': [ + 'First option', + 'Primera opci\xf3n', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', + ], + }, + { + '$kuid': 'xUd28PPBs', + 'list_name': 'choice_list_2', + 'name': 'option_2', + 'label': [ + 'Second option', + 'Segunda opci\xf3n', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a', + ], + }, + ], +} -SUBMISSION_DATA = OrderedDict([ - ("start", - ["2016-06-0%dT12:00:00.000-04:00" % n for n in [1, 2, 3, 4]]), - ("end", - ["2016-06-0%dT11:0%d:00.000-04:00" % (n, n) for n in [1, 2, 3, 4]]), - ("Select_one", - ["option_1", "option_1", "option_2", "option_1"]), - ("Select_Many", - ["option_1", "option_2", "option_1 option_2", ""]), - ("Text", - ["a", "b", "c", "a"]), - ("Number", - [1, 2, 3, 2]), - ("Decimal", - [1.5, 2.5, 3.5, 3.5]), - ("Date", - ["2016-06-0%d" % n for n in [1, 2, 3, 5]]), - ("Time", - ["%d:00:00" % n for n in [1, 2, 3, 5]]), - ("Date_and_time", - ["2016-06-0%dT12:00:00.000-04:00" % n for n in [1, 2, 3, 5]]), - ("GPS", - ["1%d.43 -2%d.54 1 0" % (n, n) for n in [5, 7, 8, 5]]), - ("Photo", - ["photo_%d.jpg" % (n) for n in [1, 2, 3, 4]]), - ("Audio", - ["audio_%d.jpg" % (n) for n in [4, 3, 2, 1]]), - ("Video", - ["video_%d.jpg" % (n) for n in [6, 7, 8, 9]]), - ("Note_Should_not_be_displayed", - [None, None, None, None]), - ("Barcode", - ["barcode%d" % (n) for n in [9, 7, 7, 6]]), - ("Acknowledge", - [None, None, None, None]), - ("calculation", - ["1", "1", "1", "1"]), -]) +SUBMISSION_DATA = OrderedDict( + [ + ("start", ["2016-06-0%dT12:00:00.000-04:00" % n for n in [1, 2, 3, 4]]), + ( + "end", + ["2016-06-0%dT11:0%d:00.000-04:00" % (n, n) for n in [1, 2, 3, 4]], + ), + ("Select_one", ["option_1", "option_1", "option_2", "option_1"]), + ("Select_Many", ["option_1", "option_2", "option_1 option_2", ""]), + ("Text", ["a", "b", "c", "a"]), + ("Number", [1, 2, 3, 2]), + ("Decimal", [1.5, 2.5, 3.5, 3.5]), + ("Date", ["2016-06-0%d" % n for n in [1, 2, 3, 5]]), + ("Time", ["%d:00:00" % n for n in [1, 2, 3, 5]]), + ( + "Date_and_time", + ["2016-06-0%dT12:00:00.000-04:00" % n for n in [1, 2, 3, 5]], + ), + ("GPS", ["1%d.43 -2%d.54 1 0" % (n, n) for n in [5, 7, 8, 5]]), + ("Photo", ["photo_%d.jpg" % (n) for n in [1, 2, 3, 4]]), + ("Audio", ["audio_%d.jpg" % (n) for n in [4, 3, 2, 1]]), + ("Video", ["video_%d.jpg" % (n) for n in [6, 7, 8, 9]]), + ("Note_Should_not_be_displayed", [None, None, None, None]), + ("Barcode", ["barcode%d" % (n) for n in [9, 7, 7, 6]]), + ("Acknowledge", [None, None, None, None]), + ("calculation", ["1", "1", "1", "1"]), + ] +) -def _get_stats_object(pack, version_ids, submissions=None, lang=None, split_by=None): +def _get_stats_object( + pack, version_ids, submissions=None, lang=None, split_by=None +): if submissions == None: raise ValueError('submissions must be provided') report = pack.autoreport(versions=version_ids) field_names = [field.name for field in pack.get_fields_for_versions(-1)] stats = [] - for (field, name_or_label, data) in report.get_stats(submissions, - field_names, - lang=lang, - split_by=split_by, - ).stats: + for field, name_or_label, data in report.get_stats( + submissions, + field_names, + lang=lang, + split_by=split_by, + ).stats: stats.append((field.name, data)) return stats @@ -132,17 +269,20 @@ def setUp(self): submissions = [] for i in range(0, num_submissions): - submissions.append(OrderedDict([ - (key, SUBMISSION_DATA[key][i]) for key in SUBMISSION_DATA.keys() - ])) + submissions.append( + OrderedDict( + [ + (key, SUBMISSION_DATA[key][i]) + for key in SUBMISSION_DATA.keys() + ] + ) + ) self.asset.deploy(backend='mock', active=True) self.asset.save() v_uid = self.asset.latest_deployed_version.uid for submission in submissions: - submission.update({ - '__version__': v_uid - }) + submission.update({'__version__': v_uid}) self.asset.deployment.mock_submissions(submissions) schemas = [v.to_formpack_schema() for v in self.asset.deployed_versions] self.fp = FormPack(versions=schemas, id_string=self.asset.uid) @@ -150,73 +290,125 @@ def setUp(self): self.submissions = self.asset.deployment.get_submissions(self.user) def test_kobo_apps_reports_report_data(self): - values = report_data.data_by_identifiers(self.asset, - submission_stream=self.submissions) - expected_names = ["start", "end", "Select_one", "Select_Many", "Text", - "Number", "Decimal", "Date", "Time", "Date_and_time", - "GPS", "Photo", "Audio", "Video", "Barcode", - "Acknowledge", "calculation"] + values = report_data.data_by_identifiers( + self.asset, submission_stream=self.submissions + ) + expected_names = [ + "start", + "end", + "Select_one", + "Select_Many", + "Text", + "Number", + "Decimal", + "Date", + "Time", + "Date_and_time", + "GPS", + "Photo", + "Audio", + "Video", + "Barcode", + "Acknowledge", + "calculation", + ] self.assertEqual([v['name'] for v in values], expected_names) self.assertEqual(len(values), 17) def test_kobo_apps_reports_report_data_split_by(self): - values = report_data.data_by_identifiers(self.asset, - split_by="Select_one", - field_names=["Date"], - submission_stream=self.submissions) - self.assertEqual(values[0]['data']['values'], [ - ('2016-06-01', - {'responses': ('First option', 'Second option'), - 'frequencies': (1, 0), - 'percentages': (25.0, 0.0)}), - ('2016-06-02', - {'responses': ('First option', 'Second option'), - 'frequencies': (1, 0), - 'percentages': (25.0, 0.0)}), - ('2016-06-03', - {'responses': ('First option', 'Second option'), - 'frequencies': (0, 1), - 'percentages': (0.0, 25.0)}), - ('2016-06-05', - {'responses': ('First option', 'Second option'), - 'frequencies': (1, 0), - 'percentages': (25.0, 0.0)}), - ]) + values = report_data.data_by_identifiers( + self.asset, + split_by="Select_one", + field_names=["Date"], + submission_stream=self.submissions, + ) + self.assertEqual( + values[0]['data']['values'], + [ + ( + '2016-06-01', + { + 'responses': ('First option', 'Second option'), + 'frequencies': (1, 0), + 'percentages': (25.0, 0.0), + }, + ), + ( + '2016-06-02', + { + 'responses': ('First option', 'Second option'), + 'frequencies': (1, 0), + 'percentages': (25.0, 0.0), + }, + ), + ( + '2016-06-03', + { + 'responses': ('First option', 'Second option'), + 'frequencies': (0, 1), + 'percentages': (0.0, 25.0), + }, + ), + ( + '2016-06-05', + { + 'responses': ('First option', 'Second option'), + 'frequencies': (1, 0), + 'percentages': (25.0, 0.0), + }, + ), + ], + ) def test_kobo_apps_reports_report_data_split_by_translated(self): - values = report_data.data_by_identifiers(self.asset, - split_by="Select_one", - lang="Arabic", - field_names=["Date"], - submission_stream=self.submissions) + values = report_data.data_by_identifiers( + self.asset, + split_by="Select_one", + lang="Arabic", + field_names=["Date"], + submission_stream=self.submissions, + ) responses = set() for rv in OrderedDict(values[0]['data']['values']).values(): responses.update(rv.get('responses')) - expected = set(['\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a']) + expected = set( + [ + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a', + ] + ) self.assertEqual(responses, expected) def test_kobo_apps_reports_report_data_subset(self): - values = report_data.data_by_identifiers(self.asset, - field_names=('Select_one',), - submission_stream=self.submissions) + values = report_data.data_by_identifiers( + self.asset, + field_names=('Select_one',), + submission_stream=self.submissions, + ) self.assertEqual(values[0]['data']['frequencies'], (3, 1)) self.assertEqual(values[0]['row']['type'], 'select_one') self.assertEqual(values[0]['data']['percentages'], (75, 25)) - self.assertEqual(values[0]['data']['responses'], ('First option', 'Second option')) + self.assertEqual( + values[0]['data']['responses'], ('First option', 'Second option') + ) def test_kobo_apps_reports_report_data_translation(self): - values = report_data.data_by_identifiers(self.asset, - lang='Arabic', - field_names=('Select_one',), - submission_stream=self.submissions) - self.assertEqual(values[0]['data']['responses'], - ( # response 1 in Arabic - '\u0627\u0644\u062e\u064a\u0627\u0631 ' - '\u0627\u0644\u0623\u0648\u0644', - # response 2 in Arabic - '\u0627\u0644\u062e\u064a\u0627\u0631 ' - '\u0627\u0644\u062b\u0627\u0646\u064a')) + values = report_data.data_by_identifiers( + self.asset, + lang='Arabic', + field_names=('Select_one',), + submission_stream=self.submissions, + ) + self.assertEqual( + values[0]['data']['responses'], + ( # response 1 in Arabic + '\u0627\u0644\u062e\u064a\u0627\u0631 ' + '\u0627\u0644\u0623\u0648\u0644', + # response 2 in Arabic + '\u0627\u0644\u062e\u064a\u0627\u0631 ' + '\u0627\u0644\u062b\u0627\u0646\u064a', + ), + ) def test_export_works_if_no_version_value_provided_in_submission(self): submissions = self.asset.deployment.get_submissions(self.asset.owner) @@ -224,30 +416,43 @@ def test_export_works_if_no_version_value_provided_in_submission(self): for submission in submissions: del submission['__version__'] - values = report_data.data_by_identifiers(self.asset, - field_names=['Date', 'Decimal'], - submission_stream=submissions) + values = report_data.data_by_identifiers( + self.asset, + field_names=['Date', 'Decimal'], + submission_stream=submissions, + ) (date_stats, decimal_stats) = values - self.assertEqual(date_stats['data'], { - 'provided': 4, - 'total_count': 4, - 'stdev': 0.9574271077563381, - 'median': 3.0, - 'show_graph': False, - 'mode': 3.5, - 'not_provided': 0, - 'mean': 2.75, - }) - self.assertEqual(decimal_stats['data'], { - 'provided': 4, - 'frequencies': (1, 1, 1, 1), - 'show_graph': True, - 'not_provided': 0, - 'total_count': 4, - 'responses': ('2016-06-01', '2016-06-02', '2016-06-03', '2016-06-05'), - 'percentages': (25.0, 25.0, 25.0, 25.0), - }) + self.assertEqual( + date_stats['data'], + { + 'provided': 4, + 'total_count': 4, + 'stdev': 0.9574271077563381, + 'median': 3.0, + 'show_graph': False, + 'mode': 3.5, + 'not_provided': 0, + 'mean': 2.75, + }, + ) + self.assertEqual( + decimal_stats['data'], + { + 'provided': 4, + 'frequencies': (1, 1, 1, 1), + 'show_graph': True, + 'not_provided': 0, + 'total_count': 4, + 'responses': ( + '2016-06-01', + '2016-06-02', + '2016-06-03', + '2016-06-05', + ), + 'percentages': (25.0, 25.0, 25.0, 25.0), + }, + ) def test_has_report_styles(self): self.assertTrue(self.asset.report_styles is not None) @@ -256,32 +461,45 @@ def test_formpack_results(self): submissions = self.asset.deployment.get_submissions(self.asset.owner) def _get_autoreport_values(qname, key, lang=None, index=False): - stats = OrderedDict(_get_stats_object(self.fp, - self.vs, - submissions=submissions, - lang=lang)) + stats = OrderedDict( + _get_stats_object( + self.fp, self.vs, submissions=submissions, lang=lang + ) + ) if index is False: return stats[qname][key] else: return [s[index] for s in stats[qname][key]] - self.assertEqual(_get_autoreport_values('Select_one', 'frequency', None, 0), - ['First option', 'Second option']) - self.assertEqual(_get_autoreport_values('Select_one', 'frequency', 'Espa\xf1ol', 0), - ['Primera opci\xf3n', 'Segunda opci\xf3n']) - self.assertEqual(_get_autoreport_values('Select_one', 'frequency', 'Arabic', 0), - ['\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', - '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a']) + self.assertEqual( + _get_autoreport_values('Select_one', 'frequency', None, 0), + ['First option', 'Second option'], + ) + self.assertEqual( + _get_autoreport_values('Select_one', 'frequency', 'Espa\xf1ol', 0), + ['Primera opci\xf3n', 'Segunda opci\xf3n'], + ) + self.assertEqual( + _get_autoreport_values('Select_one', 'frequency', 'Arabic', 0), + [ + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u0623\u0648\u0644', + '\u0627\u0644\u062e\u064a\u0627\u0631 \u0627\u0644\u062b\u0627\u0646\u064a', + ], + ) self.assertEqual(_get_autoreport_values('Decimal', 'median', None), 3.0) - self.assertEqual(_get_autoreport_values('Date', 'percentage', None), [ - ("2016-06-01", 25.0), - ("2016-06-02", 25.0), - ("2016-06-03", 25.0), - ("2016-06-05", 25.0) - ]) + self.assertEqual( + _get_autoreport_values('Date', 'percentage', None), + [ + ("2016-06-01", 25.0), + ("2016-06-02", 25.0), + ("2016-06-03", 25.0), + ("2016-06-05", 25.0), + ], + ) def test_has_version_and_submissions(self): self.assertEqual(self.asset.asset_versions.count(), 2) self.assertTrue(self.asset.has_deployment) + self.asset.deployment.xform.refresh_from_db() self.assertEqual(self.asset.deployment.submission_count, 4) diff --git a/kpi/tests/test_mock_data_conflicting_version_exports.py b/kpi/tests/test_mock_data_conflicting_version_exports.py index 90fee4b17a..38fff9b67c 100644 --- a/kpi/tests/test_mock_data_conflicting_version_exports.py +++ b/kpi/tests/test_mock_data_conflicting_version_exports.py @@ -30,7 +30,8 @@ def setUp(self): # To avoid cluttering the fixture, assign permissions here self.asset.assign_perm(self.user, PERM_VIEW_SUBMISSIONS) self.submissions = self.asset.deployment.get_submissions( - self.asset.owner) + self.asset.owner + ) self.submission_id_field = '_id' self.formpack, self.submission_stream = report_data.build_formpack( self.asset, diff --git a/kpi/tests/test_mock_data_exports.py b/kpi/tests/test_mock_data_exports.py index d0bd787a1a..816f3ab0c2 100644 --- a/kpi/tests/test_mock_data_exports.py +++ b/kpi/tests/test_mock_data_exports.py @@ -35,7 +35,6 @@ class MockDataExportsBase(TestCase): fixtures = ['test_data'] - forms = { 'Identificación de animales': { 'content': { @@ -187,7 +186,6 @@ class MockDataExportsBase(TestCase): '_attachments': [], '_bamboo_dataset_id': '', '_geolocation': [None, None], - '_id': 61, '_notes': [], '_status': 'submitted_via_web', '_submission_time': '2017-10-23T09:41:19', @@ -208,7 +206,6 @@ class MockDataExportsBase(TestCase): '_attachments': [], '_bamboo_dataset_id': '', '_geolocation': [None, None], - '_id': 62, '_notes': [], '_status': 'submitted_via_web', '_submission_time': '2017-10-23T09:41:38', @@ -229,7 +226,6 @@ class MockDataExportsBase(TestCase): '_attachments': [], '_bamboo_dataset_id': '', '_geolocation': [None, None], - '_id': 63, '_notes': [], '_status': 'submitted_via_web', '_submission_time': '2017-10-23T09:42:11', @@ -277,7 +273,6 @@ class MockDataExportsBase(TestCase): }, 'submissions': [ { - '_id': 9999, 'formhub/uuid': 'cfb562511e8e44d1998de69002b492d9', 'people/person': [ { @@ -320,19 +315,18 @@ class MockDataExportsBase(TestCase): }, 'submissions': [ { - '_id': 99999, 'formhub/uuid': 'cfb562511e8e44d1998de69002b49299', - 'an_image': 'image.png', + 'an_image': 'audio_conversion_test_image.jpg', '__version__': 'vbKavWWCpgBCZms6hQX4FB', 'meta/instanceID': 'uuid:f80be949-89b5-4af1-a42d-7d292b2bc0cd', '_xform_id_string': 'aaURCfR8mYe8pzc5h3YiZz', '_uuid': 'f80be949-89b5-4af1-a42d-7d292b2bc0cd', '_attachments': [ { - 'download_url': 'http://testserver/image.png', - 'filename': 'path/to/image.png', - } - ], + 'download_url': 'http://testserver/audio_conversion_test_image.jpg', + 'filename': 'path/to/audio_conversion_test_image.jpg', + } + ], '_status': 'submitted_via_web', '_geolocation': [None, None], '_submission_time': '2021-06-30T22:12:56', @@ -378,6 +372,7 @@ def _create_asset_with_submissions(user, content, name, submissions): submission.update({ '__version__': v_uid }) + asset.deployment.set_namespace('api_v2') asset.deployment.mock_submissions(submissions, flush_db=False) return asset @@ -487,26 +482,29 @@ def run_xls_export_test( assert result_row == expected_row def test_csv_export_default_options(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version.uid expected_lines = [ '"start";"end";"What kind of symmetry do you have?";"What kind of symmetry do you have?/Spherical";"What kind of symmetry do you have?/Radial";"What kind of symmetry do you have?/Bilateral";"How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', '"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines) def test_csv_export_default_options_partial_submissions(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid expected_lines = [ '"start";"end";"What kind of symmetry do you have?";"What kind of symmetry do you have?/Spherical";"What kind of symmetry do you have?/Radial";"What kind of symmetry do you have?/Bilateral";"How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', f'"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"1"', ] self.run_csv_export_test(expected_lines, user=self.anotheruser) def test_csv_export_english_labels(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = { 'lang': 'English', @@ -514,13 +512,14 @@ def test_csv_export_english_labels(self): expected_lines = [ '"start";"end";"What kind of symmetry do you have?";"What kind of symmetry do you have?/Spherical";"What kind of symmetry do you have?/Radial";"What kind of symmetry do you have?/Bilateral";"How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', f'"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines, export_options) def test_csv_export_spanish_labels(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = { 'lang': 'Spanish', @@ -528,13 +527,14 @@ def test_csv_export_spanish_labels(self): expected_lines = [ '"start";"end";"¿Qué tipo de simetría tiene?";"¿Qué tipo de simetría tiene?/Esférico";"¿Qué tipo de simetría tiene?/Radial";"¿Qué tipo de simetría tiene?/Bilateral";"¿Cuántos segmentos tiene tu cuerpo?";"¿Tienes fluidos corporales que ocupan espacio intracelular?";"¿Desciende de un organismo unicelular ancestral?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', '"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Esférico Radial Bilateral";"1";"1";"1";"6";"Sí, y algún espacio extracelular";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Sí";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Inseguro";"Sí";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Esférico Radial Bilateral";"1";"1";"1";"6";"Sí, y algún espacio extracelular";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Sí";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Inseguro";"Sí";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines, export_options) def test_csv_export_english_labels_no_hxl(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = { 'lang': 'English', @@ -542,13 +542,14 @@ def test_csv_export_english_labels_no_hxl(self): } expected_lines = [ '"start";"end";"What kind of symmetry do you have?";"What kind of symmetry do you have?/Spherical";"What kind of symmetry do you have?/Radial";"What kind of symmetry do you have?/Bilateral";"How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines, export_options) def test_csv_export_english_labels_group_sep(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid # Check `group_sep` by looking at the `select_multiple` question export_options = { @@ -558,21 +559,22 @@ def test_csv_export_english_labels_group_sep(self): expected_lines = [ '"start";"end";"What kind of symmetry do you have?";"What kind of symmetry do you have?%Spherical";"What kind of symmetry do you have?%Radial";"What kind of symmetry do you have?%Bilateral";"How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', '"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines, export_options) def test_csv_export_hierarchy_in_labels(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'hierarchy_in_labels': 'true'} expected_lines = [ '"start";"end";"External Characteristics/What kind of symmetry do you have?";"External Characteristics/What kind of symmetry do you have?/Spherical";"External Characteristics/What kind of symmetry do you have?/Radial";"External Characteristics/What kind of symmetry do you have?/Bilateral";"External Characteristics/How many segments does your body have?";"Do you have body fluids that occupy intracellular space?";"Do you descend from an ancestral unicellular organism?";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', '"";"";"#symmetry";"";"";"";"#segments";"#fluids";"";"";"";"";"";"";"";"";"";"";""', - f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', + f'"2017-10-23T05:40:39.000-04:00";"2017-10-23T05:41:13.000-04:00";"Spherical Radial Bilateral";"1";"1";"1";"6";"Yes, and some extracellular space";"No";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"2017-10-23T05:41:14.000-04:00";"2017-10-23T05:41:32.000-04:00";"Radial";"0";"1";"0";"3";"Yes";"No";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"2017-10-23T05:41:32.000-04:00";"2017-10-23T05:42:05.000-04:00";"Bilateral";"0";"0";"1";"2";"No / Unsure";"Yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"', ] self.run_csv_export_test(expected_lines, export_options) @@ -587,62 +589,67 @@ def test_csv_export_filter_fields(self): self.run_csv_export_test(expected_lines, export_options) def test_xls_export_english_labels(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'lang': 'English'} expected_data = {self.asset.name: [ ['start', 'end', 'What kind of symmetry do you have?', 'What kind of symmetry do you have?/Spherical', 'What kind of symmetry do you have?/Radial', 'What kind of symmetry do you have?/Bilateral', 'How many segments does your body have?', 'Do you have body fluids that occupy intracellular space?', 'Do you descend from an ancestral unicellular organism?', '_id','_uuid','_submission_time','_validation_status','_notes', '_status', '_submitted_by', '__version__', '_tags', '_index'], ['', '', '#symmetry', '', '', '', '#segments', '#fluids', '', '', '', '', '', '', '', '', '', '', ''], - ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', 61.0, '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], - ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '0', '1', '0', '3', 'Yes', 'No', 62.0, '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], - ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', 63.0, '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] + ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', submissions[0]['_id'], '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], + ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '0', '1', '0', '3', 'Yes', 'No', submissions[1]['_id'], '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], + ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', submissions[2]['_id'], '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] ]} self.run_xls_export_test(expected_data, export_options) def test_xls_export_english_labels_partial_submissions(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'lang': 'English'} expected_data = {self.asset.name: [ ['start', 'end', 'What kind of symmetry do you have?', 'What kind of symmetry do you have?/Spherical', 'What kind of symmetry do you have?/Radial', 'What kind of symmetry do you have?/Bilateral', 'How many segments does your body have?', 'Do you have body fluids that occupy intracellular space?', 'Do you descend from an ancestral unicellular organism?', '_id','_uuid','_submission_time','_validation_status','_notes', '_status', '_submitted_by', '__version__', '_tags', '_index'], ['', '', '#symmetry', '', '', '', '#segments', '#fluids', '', '', '', '', '', '', '', '', '', '', ''], - ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', 63.0, '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 1.0] + ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', submissions[2]['_id'], '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 1.0] ]} self.run_xls_export_test( expected_data, export_options, user=self.anotheruser ) def test_xls_export_multiple_select_both(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'lang': 'English', 'multiple_select': 'both'} expected_data = {self.asset.name: [ ['start', 'end', 'What kind of symmetry do you have?', 'What kind of symmetry do you have?/Spherical', 'What kind of symmetry do you have?/Radial', 'What kind of symmetry do you have?/Bilateral', 'How many segments does your body have?', 'Do you have body fluids that occupy intracellular space?', 'Do you descend from an ancestral unicellular organism?', '_id','_uuid','_submission_time','_validation_status','_notes', '_status', '_submitted_by', '__version__', '_tags', '_index'], ['', '', '#symmetry', '', '', '', '#segments', '#fluids', '', '', '', '', '', '', '', '', '', '', ''], - ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', 61.0, '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], - ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '0', '1', '0', '3', 'Yes', 'No', 62.0, '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], - ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', 63.0, '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] + ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', submissions[0]['_id'], '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], + ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '0', '1', '0', '3', 'Yes', 'No', submissions[1]['_id'], '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], + ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '0', '0', '1', '2', 'No / Unsure', 'Yes', submissions[2]['_id'], '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] ]} self.run_xls_export_test(expected_data, export_options) def test_xls_export_multiple_select_summary(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'lang': 'English', 'multiple_select': 'summary'} expected_data = {self.asset.name: [ ['start', 'end', 'What kind of symmetry do you have?', 'How many segments does your body have?', 'Do you have body fluids that occupy intracellular space?', 'Do you descend from an ancestral unicellular organism?', '_id', '_uuid', '_submission_time', '_validation_status', '_notes', '_status', '_submitted_by', '__version__', '_tags', '_index'], ['', '', '#symmetry', '#segments', '#fluids', '', '', '', '', '', '', '', '', '', '', ''], - ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '6', 'Yes, and some extracellular space', 'No', 61.0, '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], - ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '3', 'Yes', 'No', 62.0, '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], - ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '2', 'No / Unsure', 'Yes', 63.0, '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] + ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', 'Spherical Radial Bilateral', '6', 'Yes, and some extracellular space', 'No', submissions[0]['_id'], '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], + ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', 'Radial', '3', 'Yes', 'No', submissions[1]['_id'], '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], + ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', 'Bilateral', '2', 'No / Unsure', 'Yes', submissions[2]['_id'], '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] ]} self.run_xls_export_test(expected_data, export_options) def test_xls_export_multiple_select_details(self): + submissions = self.forms[self.form_names[0]]['submissions'] version_uid = self.asset.latest_deployed_version_uid export_options = {'lang': 'English', 'multiple_select': 'details'} expected_data = {self.asset.name: [ ['start', 'end', 'What kind of symmetry do you have?/Spherical', 'What kind of symmetry do you have?/Radial', 'What kind of symmetry do you have?/Bilateral', 'How many segments does your body have?', 'Do you have body fluids that occupy intracellular space?', 'Do you descend from an ancestral unicellular organism?', '_id', '_uuid', '_submission_time', '_validation_status', '_notes', '_status', '_submitted_by', '__version__', '_tags', '_index'], ['', '', '#symmetry', '', '', '#segments', '#fluids', '', '', '', '', '', '', '', '', '', '', ''], - ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', 61.0, '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], - ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', '0', '1', '0', '3', 'Yes', 'No', 62.0, '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], - ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', '0', '0', '1', '2', 'No / Unsure', 'Yes', 63.0, '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] + ['2017-10-23T05:40:39.000-04:00', '2017-10-23T05:41:13.000-04:00', '1', '1', '1', '6', 'Yes, and some extracellular space', 'No', submissions[0]['_id'], '48583952-1892-4931-8d9c-869e7b49bafb', '2017-10-23T09:41:19', '', '', 'submitted_via_web', '', version_uid, '', 1.0], + ['2017-10-23T05:41:14.000-04:00', '2017-10-23T05:41:32.000-04:00', '0', '1', '0', '3', 'Yes', 'No', submissions[1]['_id'], '317ba7b7-bea4-4a8c-8620-a483c3079c4b', '2017-10-23T09:41:38', '', '', 'submitted_via_web', '', version_uid, '', 2.0], + ['2017-10-23T05:41:32.000-04:00', '2017-10-23T05:42:05.000-04:00', '0', '0', '1', '2', 'No / Unsure', 'Yes', submissions[2]['_id'], '3f15cdfe-3eab-4678-8352-7806febf158d', '2017-10-23T09:42:11', '', '', 'submitted_via_web', 'anotheruser', version_uid, '', 3.0] ]} self.run_xls_export_test(expected_data, export_options) @@ -730,12 +737,18 @@ def test_xls_export_filter_fields_without_index(self): def test_xls_export_filter_fields_with_media_url(self): asset_name = 'Simple media' export_options = {'fields': ['an_image'], 'include_media_url': True} + asset = self.assets[asset_name] + submissions = self.forms[asset_name]['submissions'] + submission = asset.deployment.get_submission( + submissions[0]['_id'], asset.owner + ) + media_url = submission['_attachments'][0]['download_url'] expected_data = { asset_name: [ ['Submit an image', 'Submit an image_URL', '_uuid'], [ - 'image.png', - 'http://testserver/image.png', + 'audio_conversion_test_image.jpg', + media_url, 'f80be949-89b5-4af1-a42d-7d292b2bc0cd', ], ] @@ -824,6 +837,7 @@ def test_xls_export_filter_fields_repeat_groups(self): def test_xls_export_repeat_groups(self): asset = self.assets['Simple repeat group'] + submissions = self.forms['Simple repeat group']['submissions'] version_uid = asset.latest_deployed_version_uid expected_data = { asset.name: [ @@ -840,7 +854,7 @@ def test_xls_export_repeat_groups(self): '_index', ], [ - 9999.0, + submissions[0]['_id'], 'f80be949-89b5-4af1-a29d-7d292b2bc0cd', '2021-06-30T22:12:56', '', @@ -875,7 +889,7 @@ def test_xls_export_repeat_groups(self): 1.0, 'Simple repeat group', 1.0, - 9999.0, + submissions[0]['_id'], 'f80be949-89b5-4af1-a29d-7d292b2bc0cd', '2021-06-30T22:12:56', '', @@ -891,7 +905,7 @@ def test_xls_export_repeat_groups(self): 2.0, 'Simple repeat group', 1.0, - 9999.0, + submissions[0]['_id'], 'f80be949-89b5-4af1-a29d-7d292b2bc0cd', '2021-06-30T22:12:56', '', @@ -1106,6 +1120,7 @@ def test_export_long_form_title(self): ) def test_export_latest_version_only(self): + submissions = self.forms[self.form_names[0]]['submissions'] new_survey_content = [{ 'label': ['Do you descend... new label', '\xbfDesciende de... etiqueta nueva'], @@ -1124,9 +1139,9 @@ def test_export_latest_version_only(self): self.asset.deploy(backend='mock', active=True) expected_lines = [ '"Do you descend... new label";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', - f'"no";"61";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', - f'"no";"62";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', - f'"yes";"63";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"' + f'"no";"{submissions[0]["_id"]}";"48583952-1892-4931-8d9c-869e7b49bafb";"2017-10-23T09:41:19";"";"";"submitted_via_web";"";"{version_uid}";"";"1"', + f'"no";"{submissions[1]["_id"]}";"317ba7b7-bea4-4a8c-8620-a483c3079c4b";"2017-10-23T09:41:38";"";"";"submitted_via_web";"";"{version_uid}";"";"2"', + f'"yes";"{submissions[2]["_id"]}";"3f15cdfe-3eab-4678-8352-7806febf158d";"2017-10-23T09:42:11";"";"";"submitted_via_web";"anotheruser";"{version_uid}";"";"3"' ] self.run_csv_export_test( expected_lines, {'fields_from_all_versions': 'false'}) @@ -1141,7 +1156,7 @@ def test_export_exceeding_api_submission_limit(self): asset = Asset.objects.create( name='Lots of submissions', owner=self.user, - content={'survey': [{'name': 'q', 'type': 'integer'}]}, + content={'survey': [{'label': 'q', 'name': 'q', 'type': 'integer'}]}, ) asset.deploy(backend='mock', active=True) submissions = [ @@ -1167,18 +1182,23 @@ def test_export_with_disabled_questions(self): name='Form with undocumented `disabled` column', owner=self.user, content={'survey': [ - {'name': 'q', 'type': 'integer'}, + {'label': 'q', 'name': 'q', 'type': 'integer'}, {'name': 'ignore', 'type': 'select_one nope', 'disabled': True}, ]}, ) asset.deploy(backend='mock', active=True) - asset.deployment.mock_submissions( - [{'__version__': asset.latest_deployed_version.uid, 'q': 123,}] - ) + submissions = [ + { + '__version__': asset.latest_deployed_version.uid, + 'q': 123, + '_submission_time': '2017-10-23T09:41:19', + } + ] + asset.deployment.mock_submissions(submissions) # observe that `ignore` does not appear! expected_lines = [ '"q";"_id";"_uuid";"_submission_time";"_validation_status";"_notes";"_status";"_submitted_by";"__version__";"_tags";"_index"', - f'"123";"1";"";"";"";"";"";"";"{asset.latest_deployed_version.uid}";"";"1"', + f'"123";"{submissions[0]["_id"]}";"{submissions[0]["_uuid"]}";"2017-10-23T09:41:19";"";"";"submitted_via_web";"someuser";"{asset.latest_deployed_version.uid}";"";"1"', ] # fails with `KeyError` prior to fix for kobotoolbox/formpack#219 self.run_csv_export_test(expected_lines, asset=asset) diff --git a/kpi/tests/test_mongo_helper.py b/kpi/tests/test_mongo_helper.py index f931e1bf14..e34d079581 100644 --- a/kpi/tests/test_mongo_helper.py +++ b/kpi/tests/test_mongo_helper.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import itertools from django.conf import settings from django.test import TestCase @@ -25,10 +26,15 @@ def assert_instances_count(self, instances: tuple, expected_count: int): assert instances[1] == expected_count def test_get_instances(self): - users = baker.make(settings.AUTH_USER_MODEL, _quantity=2) + names = ('bob', 'alice') + users = baker.make( + settings.AUTH_USER_MODEL, + username=itertools.cycle(names), + _quantity=2, + ) assets = [] - for user in users: - asset = baker.make('kpi.Asset', owner=user) + for idx, user in enumerate(users): + asset = baker.make('kpi.Asset', owner=user, uid=f'assetUid{idx}') asset.deploy(backend='mock', active=True) assets.append(asset) (asset1, asset2) = assets @@ -62,8 +68,9 @@ def test_get_instances(self): ) def test_get_instances_permission_filters(self): - user = baker.make(settings.AUTH_USER_MODEL) - asset = baker.make('kpi.Asset', owner=user) + bob = baker.make(settings.AUTH_USER_MODEL, username='bob') + alice = baker.make(settings.AUTH_USER_MODEL, username='alice') + asset = baker.make('kpi.Asset', owner=bob, uid='assetUid') asset.deploy(backend='mock', active=True) userform_id = asset.deployment.mongo_userform_id submissions = [ diff --git a/kpi/tests/utils/dicts.py b/kpi/tests/utils/dicts.py new file mode 100644 index 0000000000..3e25ddfe3c --- /dev/null +++ b/kpi/tests/utils/dicts.py @@ -0,0 +1,35 @@ +from __future__ import annotations + + +def nested_dict_from_keys(dict_: dict) -> dict: + """ + Transforms a dictionary with keys containing slashes into a nested + dictionary structure. + """ + + result = {} + + for key, value in dict_.items(): + keys = key.split('/') + sub_dict = result + for sub_key in keys[:-1]: + if sub_key not in sub_dict: + sub_dict[sub_key] = {} + sub_dict = sub_dict[sub_key] + + if isinstance(value, list): + sub_dict[keys[-1]] = [ + { + sub_key.split('/')[-1]: sub_val + for sub_key, sub_val in item.items() + } + for item in value if item + ] + else: + sub_dict[keys[-1]] = ( + nested_dict_from_keys(value) + if isinstance(value, dict) + else value + ) + + return result diff --git a/kpi/tests/utils/mock.py b/kpi/tests/utils/mock.py index 1400228e93..f579eb1e6c 100644 --- a/kpi/tests/utils/mock.py +++ b/kpi/tests/utils/mock.py @@ -1,31 +1,12 @@ # coding: utf-8 import json import lxml -import os from mimetypes import guess_type -from tempfile import NamedTemporaryFile -from typing import Optional from urllib.parse import parse_qs, unquote from django.conf import settings -from django.core.files import File - -from django.core.files.storage import default_storage from rest_framework import status -from kobo.apps.openrosa.apps.logger.models.attachment import ( - Attachment, - upload_to, -) -from kobo.apps.openrosa.libs.utils.image_tools import ( - get_optimized_image_path, - resize, -) -from kpi.deployment_backends.kc_access.storage import ( - default_kobocat_storage, - KobocatFileSystemStorage, -) -from kpi.mixins.audio_transcoding import AudioTranscodingMixin from kpi.models.asset_snapshot import AssetSnapshot from kpi.tests.utils.xml import get_form_and_submission_tag_names @@ -112,3 +93,12 @@ def enketo_view_instance_response(request): } headers = {} return status.HTTP_201_CREATED, headers, json.dumps(resp_body) + + +def guess_type_mock(url, strict=True): + """ + In the container, `*.3gp` returns "audio/3gpp" instead of "video/3gpp". + """ + if url.endswith('.3gp'): + return 'video/3gpp', None + return guess_type(url, strict) diff --git a/kpi/tests/utils/xml.py b/kpi/tests/utils/xml.py index 4e0e4d2758..57edce20fc 100644 --- a/kpi/tests/utils/xml.py +++ b/kpi/tests/utils/xml.py @@ -4,7 +4,7 @@ def get_form_and_submission_tag_names(form: str, submission: str) -> tuple[str, str]: - submission_root_name = etree.fromstring(submission).tag + submission_root_name = etree.fromstring(submission.encode()).tag tree = etree.ElementTree(etree.fromstring(form)) root = tree.getroot() # We cannot use `root.nsmap` directly because the default namespace key is diff --git a/kpi/utils/files.py b/kpi/utils/files.py index adb911ee1e..f732dd83d6 100644 --- a/kpi/utils/files.py +++ b/kpi/utils/files.py @@ -1,12 +1,18 @@ import os -from mimetypes import guess_type +import mimetypes +# from mimetypes import guess_type from django.core.files.base import ContentFile class ExtendedContentFile(ContentFile): + def __init__(self, content, name=None, *args, **kwargs): + super().__init__(content, name) + self._mimetype = kwargs.get('mimetype') + @property def content_type(self): - mimetype, _ = guess_type(os.path.basename(self.name)) + if not (mimetype := self._mimetype): + mimetype, _ = mimetypes.guess_type(os.path.basename(self.name)) return mimetype diff --git a/kpi/views/v2/attachment.py b/kpi/views/v2/attachment.py index b35c1dd4d0..fb900bbb18 100644 --- a/kpi/views/v2/attachment.py +++ b/kpi/views/v2/attachment.py @@ -140,7 +140,7 @@ def _get_response( # the content to the Response object if settings.TESTING: # setting the content type to `None` here allows the renderer to - # specify the content type for the response + # specify the content type for the response. content_type = ( attachment.mimetype if request.accepted_renderer.format != MP3ConversionRenderer.format diff --git a/kpi/views/v2/data.py b/kpi/views/v2/data.py index febab500ed..6ab9300769 100644 --- a/kpi/views/v2/data.py +++ b/kpi/views/v2/data.py @@ -396,7 +396,6 @@ def destroy(self, request, pk, *args, **kwargs): json_response = deployment.delete_submission( submission_id, user=request.user ) - if json_response['status'] == status.HTTP_204_NO_CONTENT: AuditLog.objects.create( app_label='logger', @@ -531,16 +530,11 @@ def retrieve(self, request, pk, *args, **kwargs): # Join all parameters to be passed to `deployment.get_submissions()` params.update(filters) - # The `get_submissions()` is a generator in KobocatDeploymentBackend - # class but a list in MockDeploymentBackend. We cast the result as a list - # no matter what is the deployment back-end class to make it work with - # both. Since the number of submissions is very small, it should not - # have a big impact on memory (i.e. list vs generator) - submissions = list(deployment.get_submissions(**params)) + submissions = deployment.get_submissions(**params) if not submissions: raise Http404 - submission = submissions[0] + submission = list(submissions)[0] return Response(submission) @action(detail=True, methods=['POST'], From e6139cac60865132a70017e81d2c79cab615b662 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Aug 2024 11:51:39 -0400 Subject: [PATCH 03/10] Fix last tests --- kobo/apps/openrosa/libs/utils/logger_tools.py | 1 - kobo/apps/trackers/submission_utils.py | 9 +++------ kobo/settings/testing.py | 7 +++++++ kpi/deployment_backends/mock_backend.py | 14 +++++++------- kpi/fixtures/attachments/IMG_4266-11_38_22.jpg | Bin 0 -> 1130 bytes ...creenshot_2024-02-14_at_18.31.39-11_38_35.jpg | Bin 0 -> 1130 bytes .../attachments}/audio_conversion_test_clip.3gp | Bin .../attachments}/audio_conversion_test_image.jpg | Bin ...\330\261\330\247\331\212\330\271-10_7_41.jpg" | Bin 0 -> 1130 bytes kpi/management/commands/cypress_testserver.py | 2 +- kpi/tests/api/v2/test_api_asset_usage.py | 10 +++++++--- kpi/tests/api/v2/test_api_service_usage.py | 8 +++++--- kpi/tests/api/v2/test_api_submissions.py | 4 ++-- kpi/tests/test_mock_data_exports.py | 2 +- kpi/tests/test_mongo_helper.py | 4 +--- 15 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 kpi/fixtures/attachments/IMG_4266-11_38_22.jpg create mode 100644 kpi/fixtures/attachments/Screenshot_2024-02-14_at_18.31.39-11_38_35.jpg rename kpi/{tests => fixtures/attachments}/audio_conversion_test_clip.3gp (100%) rename kpi/{tests => fixtures/attachments}/audio_conversion_test_image.jpg (100%) create mode 100644 "kpi/fixtures/attachments/\331\203\331\210\330\250\331\210-\330\261\330\247\331\212\330\271-10_7_41.jpg" diff --git a/kobo/apps/openrosa/libs/utils/logger_tools.py b/kobo/apps/openrosa/libs/utils/logger_tools.py index 3fd880394e..81decfc462 100644 --- a/kobo/apps/openrosa/libs/utils/logger_tools.py +++ b/kobo/apps/openrosa/libs/utils/logger_tools.py @@ -466,7 +466,6 @@ def publish_xls_form(xls_file, user, id_string=None): with transaction.atomic(): dd = DataDictionary.objects.create(user=user, xls=xls_file) except IntegrityError as e: - breakpoint() raise e return dd diff --git a/kobo/apps/trackers/submission_utils.py b/kobo/apps/trackers/submission_utils.py index 18702cf842..0294318d96 100644 --- a/kobo/apps/trackers/submission_utils.py +++ b/kobo/apps/trackers/submission_utils.py @@ -52,10 +52,7 @@ def _get_uid(count): _quantity=assets_per_user, ) - print([a.uid for a in assets]) - breakpoint() for asset in assets: - print('DEPLOYING ', asset.uid, flush=True) asset.deploy(backend='mock', active=True) asset.deployment.set_namespace(ROUTER_URL_NAMESPACE) asset.save() # might be redundant? @@ -68,9 +65,9 @@ def expected_file_size(submissions: int = 1): Calculate the expected combined file size for the test audio clip and image """ return (os.path.getsize( - settings.BASE_DIR + '/kpi/tests/audio_conversion_test_clip.3gp' + settings.BASE_DIR + '/kpi/fixtures/attachments/audio_conversion_test_clip.3gp' ) + os.path.getsize( - settings.BASE_DIR + '/kpi/tests/audio_conversion_test_image.jpg' + settings.BASE_DIR + '/kpi/fixtures/attachments/audio_conversion_test_image.jpg' )) * submissions @@ -106,7 +103,7 @@ def add_mock_submissions(assets: list, submissions_per_asset: int = 1): } asset_submissions.append(submission) - asset.deployment.mock_submissions(asset_submissions, flush_db=False) + asset.deployment.mock_submissions(asset_submissions) all_submissions = all_submissions + asset_submissions # update_xform_counters(asset, submissions=submissions_per_asset) diff --git a/kobo/settings/testing.py b/kobo/settings/testing.py index 421adcaeb6..fcd4a2e749 100644 --- a/kobo/settings/testing.py +++ b/kobo/settings/testing.py @@ -52,3 +52,10 @@ TEST_USERNAME = 'bob' OPENROSA_DB_ALIAS = DEFAULT_DB_ALIAS + +# STORAGES['default']['BACKEND'] = 'django.core.files.storage.FileSystemStorage' +# KOBOCAT_DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +# KOBOCAT_MEDIA_ROOT = os.environ.get( +# 'KOBOCAT_MEDIA_ROOT', MEDIA_ROOT.replace('kpi', 'kobocat') +# ) +# PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher'] diff --git a/kpi/deployment_backends/mock_backend.py b/kpi/deployment_backends/mock_backend.py index 702ce886c4..4dfa0b8e28 100644 --- a/kpi/deployment_backends/mock_backend.py +++ b/kpi/deployment_backends/mock_backend.py @@ -49,7 +49,7 @@ def get_submissions( )) def mock_submissions( - self, submissions, create_uuids: bool = True, flush_db: bool = True + self, submissions, create_uuids: bool = True ): """ Simulate client (i.e.: Enketo or Collect) data submission. @@ -120,11 +120,10 @@ class FakeRequest: if error: raise Exception(error) - # Inject (or update) real PK in submission - # FIXME TRY TO ASSIGN Instance.PK if it already exists + # Inject (or update) real PKs in submission… submission['_id'] = instance.pk - # Reassign attachment PKs + # … and attachments if '_attachments' in submission: for idx, attachment in enumerate(instance.attachments.all()): submission['_attachments'][idx]['id'] = attachment.pk @@ -160,14 +159,15 @@ def _get_media_files(self, submission): file_ = os.path.join( settings.BASE_DIR, 'kpi', - 'tests', + 'fixtures', + 'attachments', basename ) if not os.path.isfile(file_): raise Exception( f'File `filename` does not exist! Use `path/to/image.png` if' - f' you need a fake attachment, or ' - f'`audio_conversion_test_image.(jpg|3gp)` for real attachment' + f' you need a fake attachment, or use one of file names ' + f'inside `kpi/fixtures/attachments for real attachment' ) with open(file_, 'rb') as f: diff --git a/kpi/fixtures/attachments/IMG_4266-11_38_22.jpg b/kpi/fixtures/attachments/IMG_4266-11_38_22.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f60baa8ea58f5de22b57f6c2705630c797a80bee GIT binary patch literal 1130 zcma)5O;6iE5M4JRq`^R}fP?_4D|2b9#%nty!4@t`Fi}&<6@p0Z)z}NMg1wgQG=^*Y zuX-ql{)m2@xL55@aMx}UXdzWcvb6g=@6F8a+vIa{4b~1~=M;c$_bDiXn@GL`P9Iu> z3&4Opg?9l+u5)rfihLCzFBD9hoM6EsE{aVbNrH&L?q2Mh)&NKF1fMyc#{c~Ak%x}0 z@vrNKX!!fM@4UDS@bTrVo^?5}6r10B4tHZUc6}E|CXC&4FH~cV&%{-_PGQ8uEF~If zd@HRA-x!B*p9B~xf>gCcQHD)LFdJs0rZjfUXHXX9Z6wx^*r-a9T9;L^4#&aMf`NUi z9_gKNIl9yM@m!!xRr$%fGpzUlW~;jnk%V|0W=G7KzC)3;KxukHuf zj7ZQU6WSI>Pm7WgQ33TWog%;=*if z)tTK(bEe*yYtQawXC@cXW>9*$w=R1EQ3s{Vse9>W>VND}(+AYu$q%rUc^24L^c6A5 zH}D=5^7;H+zA!gecvzs{La|UNF0iH2LaD@-*(}QJ(sFruiLF#tRw|Xvjg5`XN7G>D z=jRvM#kF#IZGDwnU7!B=-$il-xFUVnOpf8e1CGgYO!5i5q{yF9Wd24#`3}O}L6Wb4 Q%`sFs$ALZ2dzXCq1x>LgzyJUM literal 0 HcmV?d00001 diff --git a/kpi/fixtures/attachments/Screenshot_2024-02-14_at_18.31.39-11_38_35.jpg b/kpi/fixtures/attachments/Screenshot_2024-02-14_at_18.31.39-11_38_35.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f60baa8ea58f5de22b57f6c2705630c797a80bee GIT binary patch literal 1130 zcma)5O;6iE5M4JRq`^R}fP?_4D|2b9#%nty!4@t`Fi}&<6@p0Z)z}NMg1wgQG=^*Y zuX-ql{)m2@xL55@aMx}UXdzWcvb6g=@6F8a+vIa{4b~1~=M;c$_bDiXn@GL`P9Iu> z3&4Opg?9l+u5)rfihLCzFBD9hoM6EsE{aVbNrH&L?q2Mh)&NKF1fMyc#{c~Ak%x}0 z@vrNKX!!fM@4UDS@bTrVo^?5}6r10B4tHZUc6}E|CXC&4FH~cV&%{-_PGQ8uEF~If zd@HRA-x!B*p9B~xf>gCcQHD)LFdJs0rZjfUXHXX9Z6wx^*r-a9T9;L^4#&aMf`NUi z9_gKNIl9yM@m!!xRr$%fGpzUlW~;jnk%V|0W=G7KzC)3;KxukHuf zj7ZQU6WSI>Pm7WgQ33TWog%;=*if z)tTK(bEe*yYtQawXC@cXW>9*$w=R1EQ3s{Vse9>W>VND}(+AYu$q%rUc^24L^c6A5 zH}D=5^7;H+zA!gecvzs{La|UNF0iH2LaD@-*(}QJ(sFruiLF#tRw|Xvjg5`XN7G>D z=jRvM#kF#IZGDwnU7!B=-$il-xFUVnOpf8e1CGgYO!5i5q{yF9Wd24#`3}O}L6Wb4 Q%`sFs$ALZ2dzXCq1x>LgzyJUM literal 0 HcmV?d00001 diff --git a/kpi/tests/audio_conversion_test_clip.3gp b/kpi/fixtures/attachments/audio_conversion_test_clip.3gp similarity index 100% rename from kpi/tests/audio_conversion_test_clip.3gp rename to kpi/fixtures/attachments/audio_conversion_test_clip.3gp diff --git a/kpi/tests/audio_conversion_test_image.jpg b/kpi/fixtures/attachments/audio_conversion_test_image.jpg similarity index 100% rename from kpi/tests/audio_conversion_test_image.jpg rename to kpi/fixtures/attachments/audio_conversion_test_image.jpg diff --git "a/kpi/fixtures/attachments/\331\203\331\210\330\250\331\210-\330\261\330\247\331\212\330\271-10_7_41.jpg" "b/kpi/fixtures/attachments/\331\203\331\210\330\250\331\210-\330\261\330\247\331\212\330\271-10_7_41.jpg" new file mode 100644 index 0000000000000000000000000000000000000000..f60baa8ea58f5de22b57f6c2705630c797a80bee GIT binary patch literal 1130 zcma)5O;6iE5M4JRq`^R}fP?_4D|2b9#%nty!4@t`Fi}&<6@p0Z)z}NMg1wgQG=^*Y zuX-ql{)m2@xL55@aMx}UXdzWcvb6g=@6F8a+vIa{4b~1~=M;c$_bDiXn@GL`P9Iu> z3&4Opg?9l+u5)rfihLCzFBD9hoM6EsE{aVbNrH&L?q2Mh)&NKF1fMyc#{c~Ak%x}0 z@vrNKX!!fM@4UDS@bTrVo^?5}6r10B4tHZUc6}E|CXC&4FH~cV&%{-_PGQ8uEF~If zd@HRA-x!B*p9B~xf>gCcQHD)LFdJs0rZjfUXHXX9Z6wx^*r-a9T9;L^4#&aMf`NUi z9_gKNIl9yM@m!!xRr$%fGpzUlW~;jnk%V|0W=G7KzC)3;KxukHuf zj7ZQU6WSI>Pm7WgQ33TWog%;=*if z)tTK(bEe*yYtQawXC@cXW>9*$w=R1EQ3s{Vse9>W>VND}(+AYu$q%rUc^24L^c6A5 zH}D=5^7;H+zA!gecvzs{La|UNF0iH2LaD@-*(}QJ(sFruiLF#tRw|Xvjg5`XN7G>D z=jRvM#kF#IZGDwnU7!B=-$il-xFUVnOpf8e1CGgYO!5i5q{yF9Wd24#`3}O}L6Wb4 Q%`sFs$ALZ2dzXCq1x>LgzyJUM literal 0 HcmV?d00001 diff --git a/kpi/management/commands/cypress_testserver.py b/kpi/management/commands/cypress_testserver.py index 42e70fdcb8..3d66c22970 100644 --- a/kpi/management/commands/cypress_testserver.py +++ b/kpi/management/commands/cypress_testserver.py @@ -381,4 +381,4 @@ def set_version(submission): submission['__version__'] = latest_version_uuid return submission submission_generator = (set_version(s) for s in submissions) - asset.deployment.mock_submissions(submission_generator, flush_db=False) + asset.deployment.mock_submissions(submission_generator) diff --git a/kpi/tests/api/v2/test_api_asset_usage.py b/kpi/tests/api/v2/test_api_asset_usage.py index 261db675b4..4f597c0e03 100644 --- a/kpi/tests/api/v2/test_api_asset_usage.py +++ b/kpi/tests/api/v2/test_api_asset_usage.py @@ -109,7 +109,7 @@ def __add_submissions(self): submissions.append(submission1) submissions.append(submission2) - self.asset.deployment.mock_submissions(submissions, flush_db=False) + self.asset.deployment.mock_submissions(submissions) def __create_asset(self): content_source_asset = { @@ -140,8 +140,12 @@ def __expected_file_size(self): Calculate the expected combined file size for the test audio clip and image """ return os.path.getsize( - settings.BASE_DIR + '/kpi/tests/audio_conversion_test_clip.3gp' - ) + os.path.getsize(settings.BASE_DIR + '/kpi/tests/audio_conversion_test_image.jpg') + settings.BASE_DIR + + '/kpi/fixtures/attachments/audio_conversion_test_clip.3gp' + ) + os.path.getsize( + settings.BASE_DIR + + '/kpi/fixtures/attachments/audio_conversion_test_image.jpg' + ) def test_anonymous_user(self): """ diff --git a/kpi/tests/api/v2/test_api_service_usage.py b/kpi/tests/api/v2/test_api_service_usage.py index 0d56a71133..9d927d93c8 100644 --- a/kpi/tests/api/v2/test_api_service_usage.py +++ b/kpi/tests/api/v2/test_api_service_usage.py @@ -137,7 +137,7 @@ def add_submissions(self, count=2): self.attachment_id = self.attachment_id + 2 submissions.append(submission) - self.asset.deployment.mock_submissions(submissions, flush_db=False) + self.asset.deployment.mock_submissions(submissions) @staticmethod def expected_file_size(): @@ -145,9 +145,11 @@ def expected_file_size(): Calculate the expected combined file size for the test audio clip and image """ return os.path.getsize( - settings.BASE_DIR + '/kpi/tests/audio_conversion_test_clip.3gp' + settings.BASE_DIR + + '/kpi/fixtures/attachments/audio_conversion_test_clip.3gp' ) + os.path.getsize( - settings.BASE_DIR + '/kpi/tests/audio_conversion_test_image.jpg' + settings.BASE_DIR + + '/kpi/fixtures/attachments/audio_conversion_test_image.jpg' ) diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index 9ffad5722d..98c4abdaf7 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -1060,7 +1060,7 @@ def test_attachments_rewrite(self): { 'group_ec9yq67/group_dq8as25/group_xt0za80': [ { - 'group_ec9yq67/group_dq8as25/group_xt0za80/my_attachment': 'Screenshot 2024-02-14 at 18.31.39-11_38_35.png' + 'group_ec9yq67/group_dq8as25/group_xt0za80/my_attachment': 'Screenshot 2024-02-14 at 18.31.39-11_38_35.jpg' } ] }, @@ -1094,7 +1094,7 @@ def test_attachments_rewrite(self): 'download_medium_url': 'http://kc.testserver/3.jpg', 'download_small_url': 'http://kc.testserver/3.jpg', 'mimetype': 'image/jpeg', - 'filename': 'anotheruser/attachments/formhub-uuid/submission-uuid/Screenshot_2024-02-14_at_18.31.39-11_38_35.png', + 'filename': 'anotheruser/attachments/formhub-uuid/submission-uuid/Screenshot_2024-02-14_at_18.31.39-11_38_35.jpg', 'instance': 1, 'xform': 1, 'id': 3, diff --git a/kpi/tests/test_mock_data_exports.py b/kpi/tests/test_mock_data_exports.py index 816f3ab0c2..fdc9986c74 100644 --- a/kpi/tests/test_mock_data_exports.py +++ b/kpi/tests/test_mock_data_exports.py @@ -373,7 +373,7 @@ def _create_asset_with_submissions(user, content, name, submissions): '__version__': v_uid }) asset.deployment.set_namespace('api_v2') - asset.deployment.mock_submissions(submissions, flush_db=False) + asset.deployment.mock_submissions(submissions) return asset diff --git a/kpi/tests/test_mongo_helper.py b/kpi/tests/test_mongo_helper.py index e34d079581..48a1192b9a 100644 --- a/kpi/tests/test_mongo_helper.py +++ b/kpi/tests/test_mongo_helper.py @@ -18,9 +18,7 @@ def setUp(self): def add_submissions(self, asset, submissions: list[dict]): for submission in submissions: submission['__version__'] = asset.latest_deployed_version.uid - asset.deployment.mock_submissions( - copy.deepcopy(submissions), flush_db=False - ) + asset.deployment.mock_submissions(copy.deepcopy(submissions)) def assert_instances_count(self, instances: tuple, expected_count: int): assert instances[1] == expected_count From 244c38ea45ec110ab483497f30f8331580f0a704 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Aug 2024 12:17:57 -0400 Subject: [PATCH 04/10] Add pytest-xdist pip dependency --- dependencies/pip/dev_requirements.in | 4 ++-- dependencies/pip/dev_requirements.txt | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dependencies/pip/dev_requirements.in b/dependencies/pip/dev_requirements.in index ce4a55e5c0..2ef94dd41d 100644 --- a/dependencies/pip/dev_requirements.in +++ b/dependencies/pip/dev_requirements.in @@ -11,8 +11,8 @@ pytest pytest-cov pytest-django pytest-env +pytest-xdist - -# Kobocat +# KoboCAT httmock simplejson diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 4030660476..69da04e3fd 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -242,6 +242,8 @@ et-xmlfile==1.1.0 # via openpyxl exceptiongroup==1.2.0 # via pytest +execnet==2.1.1 + # via pytest-xdist executing==2.0.1 # via stack-data fabric==3.2.2 @@ -469,12 +471,15 @@ pytest==8.1.1 # pytest-cov # pytest-django # pytest-env + # pytest-xdist pytest-cov==5.0.0 # via -r dependencies/pip/dev_requirements.in pytest-django==4.8.0 # via -r dependencies/pip/dev_requirements.in pytest-env==1.1.3 # via -r dependencies/pip/dev_requirements.in +pytest-xdist==3.6.1 + # via -r dependencies/pip/dev_requirements.in python-crontab==3.0.0 # via django-celery-beat python-dateutil==2.9.0.post0 From be7505e764fe0d750182e685b5b4e489370b502d Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Aug 2024 16:15:27 -0400 Subject: [PATCH 05/10] Fix unit tests with FileSystemStorage --- .../logger/tests/models/test_attachment.py | 1 + kobo/apps/openrosa/libs/utils/image_tools.py | 3 +- kobo/settings/testing.py | 7 - kpi/tests/api/v1/test_api_submissions.py | 6 +- kpi/tests/api/v2/test_api_attachments.py | 4 +- kpi/tests/api/v2/test_api_submissions.py | 168 ++++++++---------- 6 files changed, 80 insertions(+), 109 deletions(-) diff --git a/kobo/apps/openrosa/apps/logger/tests/models/test_attachment.py b/kobo/apps/openrosa/apps/logger/tests/models/test_attachment.py index 693e49e4e8..3ae291cc20 100644 --- a/kobo/apps/openrosa/apps/logger/tests/models/test_attachment.py +++ b/kobo/apps/openrosa/apps/logger/tests/models/test_attachment.py @@ -12,6 +12,7 @@ default_kobocat_storage as default_storage, ) + class TestAttachment(TestBase): def setUp(self): diff --git a/kobo/apps/openrosa/libs/utils/image_tools.py b/kobo/apps/openrosa/libs/utils/image_tools.py index 61e8684e60..39f83bc35c 100644 --- a/kobo/apps/openrosa/libs/utils/image_tools.py +++ b/kobo/apps/openrosa/libs/utils/image_tools.py @@ -72,11 +72,10 @@ def _save_thumbnails(image, original_path, size, suffix): def resize(filename): image = None - if isinstance(default_storage, FileSystemStorage): path = default_storage.path(filename) image = Image.open(path) - original_path = path + original_path = filename else: path = default_storage.url(filename) original_path = filename diff --git a/kobo/settings/testing.py b/kobo/settings/testing.py index fcd4a2e749..421adcaeb6 100644 --- a/kobo/settings/testing.py +++ b/kobo/settings/testing.py @@ -52,10 +52,3 @@ TEST_USERNAME = 'bob' OPENROSA_DB_ALIAS = DEFAULT_DB_ALIAS - -# STORAGES['default']['BACKEND'] = 'django.core.files.storage.FileSystemStorage' -# KOBOCAT_DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -# KOBOCAT_MEDIA_ROOT = os.environ.get( -# 'KOBOCAT_MEDIA_ROOT', MEDIA_ROOT.replace('kpi', 'kobocat') -# ) -# PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher'] diff --git a/kpi/tests/api/v1/test_api_submissions.py b/kpi/tests/api/v1/test_api_submissions.py index 4c5ee14ae0..509ed0365b 100644 --- a/kpi/tests/api/v1/test_api_submissions.py +++ b/kpi/tests/api/v1/test_api_submissions.py @@ -90,7 +90,7 @@ def test_list_submissions_as_owner_with_params(self): 'limit': 5, 'sort': '{"q1": -1}', 'fields': '["q1", "_submitted_by"]', - 'query': '{"_submitted_by": {"$in": ["unknown", "someuser", "another"]}}', + 'query': '{"_submitted_by": {"$in": ["unknownuser", "someuser", "anotheruser"]}}', } ) # ToDo add more assertions. E.g. test whether sort, limit, start really work @@ -98,7 +98,7 @@ def test_list_submissions_as_owner_with_params(self): self.assertEqual(response.status_code, status.HTTP_200_OK) def test_delete_submission_as_owner(self): - submission = self.get_random_submission(self.asset.owner) + submission = self.submissions_submitted_by_someuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -116,7 +116,7 @@ def test_delete_submission_as_owner(self): def test_delete_submission_shared_as_anotheruser(self): self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() - submission = self.get_random_submission(self.asset.owner) + submission = self.submissions_submitted_by_someuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ diff --git a/kpi/tests/api/v2/test_api_attachments.py b/kpi/tests/api/v2/test_api_attachments.py index 49f83ff3b8..667de9a1ce 100644 --- a/kpi/tests/api/v2/test_api_attachments.py +++ b/kpi/tests/api/v2/test_api_attachments.py @@ -211,7 +211,9 @@ def test_duplicate_attachment_with_submission(self): duplicate_file = response.data # Ensure that the files are the same - assert original_file.read() == duplicate_file.read() + with default_storage.open(str(original_file), 'rb') as of: + with default_storage.open(str(duplicate_file), 'rb') as df: + assert of.read() == df.read() def test_xpath_not_found(self): query_dict = QueryDict('', mutable=True) diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index 98c4abdaf7..c4eb694d2b 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -71,8 +71,8 @@ def setUp(self): self.client.login(username='someuser', password='someuser') self.someuser = User.objects.get(username='someuser') self.anotheruser = User.objects.get(username='anotheruser') - self.unknown_user = User.objects.create(username='unknown_user') - UserProfile.objects.create(user=self.unknown_user) + self.unknownuser = User.objects.create(username='unknownuser') + UserProfile.objects.create(user=self.unknownuser) content_source_asset = Asset.objects.get(id=1) self.asset = Asset.objects.create( @@ -91,66 +91,36 @@ def setUp(self): ) self._deployment = self.asset.deployment - def get_random_submission(self, user: settings.AUTH_USER_MODEL) -> dict: - return self.get_random_submissions(user, 1)[0] - - def get_random_submissions( - self, user: settings.AUTH_USER_MODEL, limit: int = 1 - ) -> list: - """ - Get random submissions within all generated submissions. - - If user is not the owner, we only return submissions submitted by them. - It is useful to ensure restricted users fail tests with forbidden - submissions. - """ - query = {} - if self.asset.owner != user: - query = {'_submitted_by': user.username} - - submissions = self.asset.deployment.get_submissions(user, query=query) - random.shuffle(submissions) - return submissions[:limit] - def _add_submissions(self, other_fields: dict = None): letters = string.ascii_letters submissions = [] v_uid = self.asset.latest_deployed_version.uid self.submissions_submitted_by_someuser = [] - self.submissions_submitted_by_unknown = [] + self.submissions_submitted_by_unknownuser = [] self.submissions_submitted_by_anotheruser = [] - submitted_by_choices = ['unknown_user', 'someuser', 'anotheruser'] - for i in range(20): - # We want to have at least one submission from each - if i <= 2: - submitted_by = submitted_by_choices[i] - else: - submitted_by = random.choice(submitted_by_choices) - uuid_ = uuid.uuid4() - submission = { - '__version__': v_uid, - 'q1': ''.join(random.choice(letters) for l in range(10)), - 'q2': ''.join(random.choice(letters) for l in range(10)), - 'meta/instanceID': f'uuid:{uuid_}', - '_uuid': str(uuid_), - '_submitted_by': submitted_by - } - if other_fields is not None: - submission.update(**other_fields) - - if submitted_by == 'someuser': - self.submissions_submitted_by_someuser.append(submission) - - if submitted_by == 'unknown_user': - self.submissions_submitted_by_unknown.append(submission) - - if submitted_by == 'anotheruser': - self.submissions_submitted_by_anotheruser.append(submission) - - submissions.append(submission) + submitted_by_choices = ['unknownuser', 'someuser', 'anotheruser'] + for submitted_by in submitted_by_choices: + for i in range(2): + uuid_ = uuid.uuid4() + submission = { + '__version__': v_uid, + 'q1': ''.join(random.choice(letters) for letter in range(10)), + 'q2': ''.join(random.choice(letters) for letter in range(10)), + 'meta/instanceID': f'uuid:{uuid_}', + '_uuid': str(uuid_), + '_submitted_by': submitted_by + } + if other_fields is not None: + submission.update(**other_fields) + + submissions.append(submission) self.asset.deployment.mock_submissions(submissions) + + self.submissions_submitted_by_unknownuser = submissions[0:2] + self.submissions_submitted_by_someuser = submissions[2:4] + self.submissions_submitted_by_anotheruser = submissions[4:6] self.submissions = submissions def _log_in_as_another_user(self): @@ -304,7 +274,7 @@ def test_delete_all_allowed_submissions_with_partial_perms_as_anotheruser(self): response = self.client.get(self.submission_list_url, {'format': 'json'}) unknown_submission_ids = [ - sub['_id'] for sub in self.submissions_submitted_by_unknown + sub['_id'] for sub in self.submissions_submitted_by_unknownuser ] someuser_submission_ids = [ sub['_id'] for sub in self.submissions_submitted_by_someuser @@ -340,31 +310,32 @@ def test_delete_some_allowed_submissions_with_partial_perms_as_anotheruser(self) ) # Try first submission submitted by unknown - random_submissions = self.get_random_submissions(self.unknown_user, 3) + submissions = self.submissions_submitted_by_unknownuser data = { 'payload': { - 'submission_ids': [rs['_id'] for rs in random_submissions] + 'submission_ids': [submissions[0]['_id']] } } - response = self.client.delete(self.submission_bulk_url, - data=data, - format='json') + response = self.client.delete( + self.submission_bulk_url, data=data, format='json' + ) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try second submission submitted by anotheruser count = self._deployment.calculated_submission_count(self.anotheruser) - random_submissions = self.get_random_submissions(self.anotheruser, 3) + assert count == 2 + submissions = self.submissions_submitted_by_anotheruser data = { 'payload': { - 'submission_ids': [rs['_id'] for rs in random_submissions], + 'submission_ids': [submissions[0]['_id']], } } - response = self.client.delete(self.submission_bulk_url, - data=data, - format='json') + response = self.client.delete( + self.submission_bulk_url, data=data, format='json' + ) self.assertEqual(response.status_code, status.HTTP_200_OK) response = self.client.get(self.submission_list_url, {'format': 'json'}) - self.assertEqual(response.data['count'], count - len(random_submissions)) + self.assertEqual(response.data['count'], count - 1) def test_cannot_delete_view_only_submissions_with_partial_perms_as_anotheruser(self): """ @@ -473,14 +444,15 @@ def test_list_submissions_as_owner_with_params(self): params """ response = self.client.get( - self.submission_list_url, { + self.submission_list_url, + { 'format': 'json', 'start': 1, 'limit': 5, 'sort': '{"q1": -1}', 'fields': '["q1", "_submitted_by"]', - 'query': '{"_submitted_by": {"$in": ["unknown", "someuser", "another"]}}', - } + 'query': '{"_submitted_by": {"$in": ["unknownuser", "someuser", "anotheruser"]}}', + }, ) # ToDo add more assertions. E.g. test whether sort, limit, start really work self.assertEqual(len(response.data['results']), 5) @@ -702,7 +674,7 @@ def test_retrieve_submission_as_owner(self): someuser is the owner of the project. someuser can view one of their submission. """ - submission = self.get_random_submission(self.asset.owner) + submission = self.submissions_submitted_by_someuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -712,7 +684,7 @@ def test_retrieve_submission_as_owner(self): ) response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, submission) + self.assertEqual(response.data['_id'], submission['_id']) def test_retrieve_submission_by_uuid(self): """ @@ -739,7 +711,7 @@ def test_retrieve_submission_not_shared_as_anotheruser(self): someuser's data existence should not be revealed. """ self._log_in_as_another_user() - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -757,7 +729,7 @@ def test_retrieve_submission_shared_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -767,7 +739,7 @@ def test_retrieve_submission_shared_as_anotheruser(self): ) response = self.client.get(url, {'format': 'json'}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data, submission) + self.assertEqual(response.data['_id'], submission['_id']) def test_retrieve_submission_with_partial_permissions_as_anotheruser(self): """ @@ -779,11 +751,14 @@ def test_retrieve_submission_with_partial_permissions_as_anotheruser(self): partial_perms = { PERM_VIEW_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } - self.asset.assign_perm(self.anotheruser, PERM_PARTIAL_SUBMISSIONS, - partial_perms=partial_perms) + self.asset.assign_perm( + self.anotheruser, + PERM_PARTIAL_SUBMISSIONS, + partial_perms=partial_perms, + ) # Try first submission submitted by unknown - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -871,7 +846,8 @@ def test_delete_submission_as_anonymous(self): someuser's data existence should not be revealed. """ self.client.logout() - submission = self.get_random_submission(self.asset.owner) + submission = self.submissions_submitted_by_someuser[0] + url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -891,7 +867,7 @@ def test_delete_submission_not_shared_as_anotheruser(self): someuser's data existence should not be revealed. """ self._log_in_as_another_user() - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -911,7 +887,7 @@ def test_delete_submission_shared_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) self._log_in_as_another_user() - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -954,7 +930,7 @@ def test_delete_submission_with_partial_perms_as_anotheruser(self): ) # Try first submission submitted by unknown - submission = self.submissions_submitted_by_unknown[0] + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -969,7 +945,7 @@ def test_delete_submission_with_partial_perms_as_anotheruser(self): # Try second submission submitted by anotheruser anotheruser_submission_count = len(self.submissions_submitted_by_anotheruser) - submission = self.get_random_submission(self.anotheruser) + submission = self.submissions_submitted_by_anotheruser[0] url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -1287,7 +1263,7 @@ def test_get_edit_link_with_partial_perms_as_anotheruser(self): ) # Try first submission submitted by unknown - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-enketo-edit'), kwargs={ @@ -1299,7 +1275,7 @@ def test_get_edit_link_with_partial_perms_as_anotheruser(self): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try second submission submitted by anotheruser - submission = self.get_random_submission(self.anotheruser) + submission = self.submissions_submitted_by_anotheruser[0] url = reverse( self._get_endpoint('submission-enketo-edit'), kwargs={ @@ -1418,7 +1394,7 @@ def test_get_multiple_edit_links_and_attempt_submit_edits(self): # for POSTing to later submission_urls = [] for _ in range(2): - submission = self.get_random_submission(self.asset.owner) + submission = self.submissions_submitted_by_someuser[0] edit_url = reverse( self._get_endpoint('submission-enketo-edit'), kwargs={ @@ -1760,7 +1736,7 @@ def test_get_view_link_with_partial_perms_as_anotheruser(self): ) # Try first submission submitted by unknown - submission = self.submissions_submitted_by_unknown[0] + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-enketo-view'), kwargs={ @@ -1970,7 +1946,7 @@ def test_duplicate_submission_as_anotheruser_with_partial_perms(self): ) # Try first submission submitted by unknown - submission = self.get_random_submission(self.unknown_user) + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-duplicate'), kwargs={ @@ -1982,7 +1958,7 @@ def test_duplicate_submission_as_anotheruser_with_partial_perms(self): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try second submission submitted by anotheruser - submission = self.get_random_submission(self.anotheruser) + submission = self.submissions_submitted_by_anotheruser[0] url = reverse( self._get_endpoint('submission-duplicate'), kwargs={ @@ -2007,9 +1983,9 @@ def setUp(self): }, ) - random_submissions = self.get_random_submissions(self.asset.owner, 3) + submissions = self.submissions_submitted_by_someuser self.updated_submission_data = { - 'submission_ids': [rs['_id'] for rs in random_submissions], + 'submission_ids': [rs['_id'] for rs in submissions], 'data': { 'q1': 'Updated value', 'q_new': 'A new question and value' @@ -2161,9 +2137,9 @@ def test_bulk_update_submissions_as_anotheruser_with_partial_perms(self): assert response.status_code == status.HTTP_403_FORBIDDEN # Update some of another's submissions - random_submissions = self.get_random_submissions(self.anotheruser, 3) + submissions = self.submissions_submitted_by_anotheruser self.updated_submission_data['submission_ids'] = [ - rs['_id'] for rs in random_submissions + rs['_id'] for rs in submissions ] response = self.client.patch( self.submission_url, data=self.submitted_payload, format='json' @@ -2366,7 +2342,7 @@ def test_edit_status_with_partial_perms_as_anotheruser(self): } # Try first submission submitted by unknown - submission = self.submissions_submitted_by_unknown[0] + submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-validation-status'), kwargs={ @@ -2780,12 +2756,12 @@ def test_edit_some_submission_validation_statuses_with_partial_perms_as_anotheru self.asset.assign_perm(self.anotheruser, PERM_PARTIAL_SUBMISSIONS, partial_perms=partial_perms) - random_submissions = self.get_random_submissions(self.asset.owner, 3) + submissions = self.submissions_submitted_by_someuser data = { 'payload': { 'validation_status.uid': 'validation_status_approved', 'submission_ids': [ - rs['_id'] for rs in random_submissions + rs['_id'] for rs in submissions ] } } @@ -2797,9 +2773,9 @@ def test_edit_some_submission_validation_statuses_with_partial_perms_as_anotheru self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) # Try 2nd submission submitted by anotheruser - random_submissions = self.get_random_submissions(self.anotheruser, 3) + submissions = self.submissions_submitted_by_anotheruser data['payload']['submission_ids'] = [ - rs['_id'] for rs in random_submissions + rs['_id'] for rs in submissions ] response = self.client.patch(self.validation_statuses_url, data=data, From 9c9a300e3acf9a1f63d9ebaf7352a9b80017ec72 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Aug 2024 17:36:39 -0400 Subject: [PATCH 06/10] Remove deprecated methods --- kpi/deployment_backends/base_backend.py | 15 --------------- kpi/deployment_backends/openrosa_backend.py | 21 --------------------- 2 files changed, 36 deletions(-) diff --git a/kpi/deployment_backends/base_backend.py b/kpi/deployment_backends/base_backend.py index b26dd84fdf..c848803e7b 100644 --- a/kpi/deployment_backends/base_backend.py +++ b/kpi/deployment_backends/base_backend.py @@ -342,16 +342,6 @@ def get_submission( pass return None - @abc.abstractmethod - def get_submission_detail_url(self, submission_id: int) -> str: - pass - - def get_submission_validation_status_url(self, submission_id: int) -> str: - url = '{detail_url}validation_status/'.format( - detail_url=self.get_submission_detail_url(submission_id) - ) - return url - @abc.abstractmethod def get_submissions( self, @@ -501,11 +491,6 @@ def submission_count_since_date( ): pass - @property - @abc.abstractmethod - def submission_list_url(self): - pass - @property @abc.abstractmethod def submission_model(self): diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index bf4933a52b..53579d5a4b 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -713,18 +713,6 @@ def get_orphan_postgres_submissions(self) -> Optional[QuerySet, bool]: except InvalidXFormException: return None - def get_submission_detail_url(self, submission_id: int) -> str: - 1/0 - #url = f'{self.submission_list_url}/{submission_id}' - #return url - - def get_submission_validation_status_url(self, submission_id: int) -> str: - 1/0 - #url = '{detail_url}/validation_status'.format( - # detail_url=self.get_submission_detail_url(submission_id) - #) - #return url - def get_submissions( self, user: settings.AUTH_USER_MODEL, @@ -1187,15 +1175,6 @@ def submission_count_since_date(self, start_date=None): else: return total_submissions['count_sum'] - @property - def submission_list_url(self): - 1/0 - #url = '{kc_base}/api/v1/data/{formid}'.format( - # kc_base=settings.KOBOCAT_INTERNAL_URL, - # formid=self.backend_response['formid'] - #) - #return url - @property def submission_model(self): return Instance From 2dabaec8ad73a4438c02af18ebd6a5e6153b6478 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 4 Sep 2024 15:56:26 -0400 Subject: [PATCH 07/10] Fix wrong attribute --- kpi/deployment_backends/openrosa_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index 53579d5a4b..7816df6709 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -1385,7 +1385,7 @@ def _delete_openrosa_metadata( """ # Delete MetaData object and its related file (on storage) try: - metadata = MetaData.objects.get(pk=metadata_file_['id']) + metadata = MetaData.objects.get(pk=metadata_file_['pk']) except MetaData.DoesNotExist: pass else: From d7bd068b1b2d050d9cbec147d805fd8ca50f9bcd Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 5 Sep 2024 14:51:06 -0400 Subject: [PATCH 08/10] Fix tests --- kpi/deployment_backends/openrosa_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index b82f452724..59faeb2662 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -339,7 +339,7 @@ def duplicate_submission( # Cast to list to help unit tests to pass. return self._rewrite_json_attachment_urls( - next(self.get_submissions(user, submission_id=instance.pk)), request + self.get_submission(user=user, submission_id=instance.pk), request ) def edit_submission( From 65d81b6ce6d7431909da9c66490fd47d618fa4ab Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 3 Oct 2024 10:55:57 -0400 Subject: [PATCH 09/10] Apply requested changes --- .../openrosa/apps/logger/models/instance.py | 9 +- .../stripe/tests/test_organization_usage.py | 3 +- .../tests/test_submission_stream.py | 3 - kobo/apps/trackers/submission_utils.py | 8 +- kobo/settings/base.py | 1 + kpi/deployment_backends/mock_backend.py | 18 +- kpi/deployment_backends/openrosa_backend.py | 14 +- kpi/tests/api/v1/test_api_submissions.py | 4 +- .../api/v2/test_api_asset_export_settings.py | 15 +- kpi/tests/api/v2/test_api_assets.py | 13 +- kpi/tests/api/v2/test_api_submissions.py | 164 +++++++++++------- kpi/tests/test_mongo_helper.py | 3 +- kpi/tests/test_utils.py | 141 ++++++++++++++- kpi/tests/utils/dicts.py | 92 +++++++--- kpi/views/v2/data.py | 2 +- 15 files changed, 362 insertions(+), 128 deletions(-) diff --git a/kobo/apps/openrosa/apps/logger/models/instance.py b/kobo/apps/openrosa/apps/logger/models/instance.py index 79a99d72b9..31a889856f 100644 --- a/kobo/apps/openrosa/apps/logger/models/instance.py +++ b/kobo/apps/openrosa/apps/logger/models/instance.py @@ -134,13 +134,10 @@ def check_active(self, force): if self.xform and not self.xform.downloadable: raise FormInactiveError() - # FIXME Access `self.xform.user.profile` directly could raise a - # `RelatedObjectDoesNotExist` error if profile does not exist even if - # wrapped in try/except UserProfile = apps.get_model('main', 'UserProfile') # noqa - Avoid circular imports - if profile := UserProfile.objects.filter(user=self.xform.user).first(): - if profile.metadata.get('submissions_suspended', False): - raise TemporarilyUnavailableError() + profile, created = UserProfile.objects.get_or_create(user=self.xform.user) + if not created and profile.metadata.get('submissions_suspended', False): + raise TemporarilyUnavailableError() return def _set_geom(self): diff --git a/kobo/apps/stripe/tests/test_organization_usage.py b/kobo/apps/stripe/tests/test_organization_usage.py index 5cc13d1e52..9ee9326048 100644 --- a/kobo/apps/stripe/tests/test_organization_usage.py +++ b/kobo/apps/stripe/tests/test_organization_usage.py @@ -1,5 +1,4 @@ import timeit -import itertools import pytest from django.core.cache import cache @@ -49,7 +48,7 @@ def setUpTestData(cls): users = baker.make( User, - username=itertools.cycle(cls.names), + username=iter(cls.names), _quantity=cls.user_count - 1, _bulk_create=True, ) diff --git a/kobo/apps/subsequences/tests/test_submission_stream.py b/kobo/apps/subsequences/tests/test_submission_stream.py index dc6f770873..e915f79e72 100644 --- a/kobo/apps/subsequences/tests/test_submission_stream.py +++ b/kobo/apps/subsequences/tests/test_submission_stream.py @@ -281,6 +281,3 @@ def test_stream_with_extras_handles_duplicated_submission_uuids(self): for v in qual_response['val']: assert isinstance(v['uuid'], str) - - ## Clear all mocked submissions to avoid duplicate submission errors - #self.asset.deployment.mock_submissions([]) diff --git a/kobo/apps/trackers/submission_utils.py b/kobo/apps/trackers/submission_utils.py index 0294318d96..b388deebe1 100644 --- a/kobo/apps/trackers/submission_utils.py +++ b/kobo/apps/trackers/submission_utils.py @@ -4,13 +4,8 @@ import uuid from django.conf import settings -from django.utils import timezone from model_bakery import baker -from kobo.apps.openrosa.apps.logger.models import ( - DailyXFormSubmissionCounter, - XForm, -) from kpi.models import Asset from kpi.urls.router_api_v2 import URL_NAMESPACE as ROUTER_URL_NAMESPACE @@ -48,7 +43,7 @@ def _get_uid(count): owner=user, asset_type='survey', name='test', - uid=itertools.cycle(_get_uid(assets_per_user)), + uid=iter(_get_uid(assets_per_user)), _quantity=assets_per_user, ) @@ -105,6 +100,5 @@ def add_mock_submissions(assets: list, submissions_per_asset: int = 1): asset.deployment.mock_submissions(asset_submissions) all_submissions = all_submissions + asset_submissions - # update_xform_counters(asset, submissions=submissions_per_asset) return all_submissions diff --git a/kobo/settings/base.py b/kobo/settings/base.py index 812c5de440..8683ffaad5 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -1722,6 +1722,7 @@ def dj_stripe_request_callback_method(): 'video/webm', 'audio/aac', 'audio/aacp', + 'audio/3gpp', 'audio/flac', 'audio/mp3', 'audio/mp4', diff --git a/kpi/deployment_backends/mock_backend.py b/kpi/deployment_backends/mock_backend.py index 4dfa0b8e28..139d11ca57 100644 --- a/kpi/deployment_backends/mock_backend.py +++ b/kpi/deployment_backends/mock_backend.py @@ -15,7 +15,7 @@ safe_create_instance, ) from kpi.constants import PERM_ADD_SUBMISSIONS, SUBMISSION_FORMAT_TYPE_JSON -from kpi.tests.utils.dicts import nested_dict_from_keys +from kpi.tests.utils.dicts import convert_hierarchical_keys_to_nested_dict from .openrosa_backend import OpenRosaDeploymentBackend from ..utils.files import ExtendedContentFile @@ -56,6 +56,20 @@ def mock_submissions( Read test data and convert it to proper XML to be saved as a real Instance object. + + 1. Each item in the iterable submissions must be a dictionary following + the format of the JSON returned by the data API. + 2. The submissions are mutated to include submission and attachments + PKs (relatively `_id`, and `_attachments[index]['id']`) after being + saved in the database. + 3. If `_submitted_by` is present in a submission, the submission is made + by the user identified there, even if that user must (temporarily) be + granted permission to submit to `self.asset`. + 4. `meta/instanceID` is added to any submission where it's missing if + `create_uuids` is `True`. + 5. If `_submission_time` is present in the submission, it is preserved by + overriding the normal logic that populates this field with the current + timestamp at the moment of submission. """ class FakeRequest: @@ -65,7 +79,7 @@ class FakeRequest: owner_username = self.asset.owner.username for submission in submissions: - sub_copy = nested_dict_from_keys(submission) + sub_copy = convert_hierarchical_keys_to_nested_dict(submission) if create_uuids: if 'formhub/uuid' not in submission: diff --git a/kpi/deployment_backends/openrosa_backend.py b/kpi/deployment_backends/openrosa_backend.py index 59faeb2662..11d16852f0 100644 --- a/kpi/deployment_backends/openrosa_backend.py +++ b/kpi/deployment_backends/openrosa_backend.py @@ -277,12 +277,11 @@ def duplicate_submission( ) -> dict: """ Duplicates a single submission. The submission with the given - `submission_id` is duplicated and the `start`, `end` and + `submission_id` is duplicated, and the `start`, `end` and `instanceID` parameters of the submission are reset before being - saving the instance. + saved to the instance. - Returns a dict with uuid of created - submission if successful + Returns the duplicated submission (if successful) """ user = request.user @@ -329,6 +328,8 @@ def duplicate_submission( ) # TODO Handle errors returned by safe_create_instance + # (safe_)create_instance uses `username` argument to identify the XForm object + # (when nothing else worked). `_submitted_by` is populated by `request.user` error, instance = safe_create_instance( username=self.asset.owner.username, xml_file=ContentFile(xml_tostring(xml_parsed)), @@ -337,7 +338,6 @@ def duplicate_submission( request=request, ) - # Cast to list to help unit tests to pass. return self._rewrite_json_attachment_urls( self.get_submission(user=user, submission_id=instance.pk), request ) @@ -408,6 +408,8 @@ def edit_submission( ) # TODO Handle errors returned by safe_create_instance + # (safe_)create_instance uses `username` argument to identify the XForm object + # (when nothing else worked). `_submitted_by` is populated by `request.user` safe_create_instance( username=user.username, xml_file=xml_submission_file, @@ -1140,6 +1142,8 @@ def store_submission( media_file for media_file in attachments.values() ) + # (safe_)create_instance uses `username` argument to identify the XForm object + # (when nothing else worked). `_submitted_by` is populated by `request.user` return safe_create_instance( username=self.asset.owner.username, xml_file=ContentFile(xml_submission), diff --git a/kpi/tests/api/v1/test_api_submissions.py b/kpi/tests/api/v1/test_api_submissions.py index 509ed0365b..875d827981 100644 --- a/kpi/tests/api/v1/test_api_submissions.py +++ b/kpi/tests/api/v1/test_api_submissions.py @@ -34,7 +34,7 @@ def test_list_submissions_as_owner(self): def test_list_submissions_shared_as_anotheruser(self): self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) expected_ids = [s['_id'] for s in self.submissions] @@ -115,7 +115,7 @@ def test_delete_submission_as_owner(self): def test_delete_submission_shared_as_anotheruser(self): self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) submission = self.submissions_submitted_by_someuser[0] url = reverse( self._get_endpoint('submission-detail'), diff --git a/kpi/tests/api/v2/test_api_asset_export_settings.py b/kpi/tests/api/v2/test_api_asset_export_settings.py index 54a32fb20e..d673d925cd 100644 --- a/kpi/tests/api/v2/test_api_asset_export_settings.py +++ b/kpi/tests/api/v2/test_api_asset_export_settings.py @@ -53,13 +53,6 @@ def setUp(self): 'type': 'csv', } - def _log_in_as_another_user(self): - """ - Helper to switch user from `someuser` to `anotheruser`. - """ - self.client.logout() - self.client.login(username='anotheruser', password='anotheruser') - def _create_foo_export_settings(self, name=None): if name is None: name = self.name @@ -225,14 +218,14 @@ def test_api_list_asset_export_settings_without_perms(self): # assign `view_asset` to anotheruser self.asset.assign_perm(self.anotheruser, PERM_VIEW_ASSET) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.export_settings_list_url) assert response.status_code == status.HTTP_404_NOT_FOUND def test_api_list_asset_export_settings_with_perms(self): self._create_foo_export_settings() - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.export_settings_list_url) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -247,14 +240,14 @@ def test_api_detail_asset_export_settings_without_perms(self): export_settings = self._create_foo_export_settings() url = self._get_detail_url(export_settings.uid) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(url) assert response.status_code == status.HTTP_404_NOT_FOUND def test_api_detail_asset_export_settings_shared_with_manage_asset_perms(self): export_settings = self._create_foo_export_settings() url = self._get_detail_url(export_settings.uid) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) # assign `view_asset` to anotheruser so that they can see the asset but # not the export settings diff --git a/kpi/tests/api/v2/test_api_assets.py b/kpi/tests/api/v2/test_api_assets.py index 230d354fa9..5e3302b9de 100644 --- a/kpi/tests/api/v2/test_api_assets.py +++ b/kpi/tests/api/v2/test_api_assets.py @@ -300,6 +300,7 @@ class AssetProjectViewListApiTests(BaseAssetTestCase): def setUp(self): self.client.login(username='someuser', password='someuser') + self.anotheruser = User.objects.get(username='anotheruser') self.asset_list_url = reverse(self._get_endpoint('asset-list')) self.region_views_url = reverse(self._get_endpoint('projectview-list')) asset_country_settings = [ @@ -379,7 +380,7 @@ def test_regional_views_list(self): ['Overview', 'Test view 1'] ) - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) res = self.client.get(self.region_views_url) data = res.json() # anotheruser should only see view 1 and 2 @@ -413,7 +414,7 @@ def test_project_views_for_someuser(self): assert asset_countries & region_for_view def test_project_views_anotheruser_submission_count(self): - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) for asset in Asset.objects.all(): if asset.has_deployment: submissions = [ @@ -443,7 +444,7 @@ def test_project_views_anotheruser_submission_count(self): assert asset_detail_response.data['deployment__submission_count'] == 1 def test_project_views_for_anotheruser(self): - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) res = self.client.get(self.region_views_url) data = res.json() results = data['results'] @@ -485,7 +486,7 @@ def test_project_views_for_someuser_can_view_submissions(self): assert data_res.status_code == status.HTTP_200_OK def test_project_views_for_anotheruser_can_view_asset_detail(self): - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) user = User.objects.get(username='anotheruser') res = self.client.get(self.region_views_url) data = res.json() @@ -511,7 +512,7 @@ def test_project_views_for_anotheruser_can_view_all_asset_permission_assignments self, ): # get the first asset from the first project view - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) anotheruser = User.objects.get(username='anotheruser') proj_view_list = self.client.get(self.region_views_url).data['results'] first_proj_view = proj_view_list[0] @@ -593,7 +594,7 @@ def test_project_views_for_anotheruser_can_preview_form(self): assert snap_response.status_code == status.HTTP_200_OK def test_project_views_for_anotheruser_can_change_metadata(self): - self._login_as_anotheruser() + self.client.force_login(self.anotheruser) res = self.client.get(self.region_views_url) data = res.json() results = data['results'] diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index c4eb694d2b..7101c92ac3 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -20,6 +20,7 @@ from rest_framework import status from kobo.apps.audit_log.models import AuditLog +from kobo.apps.openrosa.apps.logger.models.instance import Instance from kobo.apps.openrosa.apps.main.models.user_profile import UserProfile from kobo.apps.openrosa.libs.utils.logger_tools import dict2xform from kobo.apps.kobo_auth.shortcuts import User @@ -123,13 +124,6 @@ def _add_submissions(self, other_fields: dict = None): self.submissions_submitted_by_anotheruser = submissions[4:6] self.submissions = submissions - def _log_in_as_another_user(self): - """ - Helper to switch user from `someuser` to `anotheruser`. - """ - self.client.logout() - self.client.login(username='anotheruser', password='anotheruser') - class BulkDeleteSubmissionsApiTests(BaseSubmissionTestCase): @@ -211,7 +205,7 @@ def test_delete_not_shared_submissions_as_anotheruser(self): anotheruser cannot view someuser's data, therefore cannot delete it. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = {'payload': {'confirm': True}} response = self.client.delete(self.submission_bulk_url, data=data, @@ -226,7 +220,7 @@ def test_delete_shared_submissions_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_DELETE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = {'payload': {'confirm': True}} response = self.client.delete(self.submission_bulk_url, @@ -245,7 +239,7 @@ def test_delete_all_allowed_submissions_with_partial_perms_as_anotheruser(self): Test that anotheruser can delete all their data at once and if they do, only delete their data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_DELETE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -297,7 +291,7 @@ def test_delete_some_allowed_submissions_with_partial_perms_as_anotheruser(self) Test that anotheruser can delete part of their data """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_DELETE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -344,7 +338,7 @@ def test_cannot_delete_view_only_submissions_with_partial_perms_as_anotheruser(s Test that anotheruser cannot delete someuser's data """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VIEW_SUBMISSIONS: [{'_submitted_by': 'someuser'}], PERM_DELETE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] # view_submission is implied @@ -417,7 +411,7 @@ def test_cannot_create_submission(self): # Shared self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.post(self.submission_list_url, data=submission) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -506,7 +500,7 @@ def test_list_submissions_not_shared_as_anotheruser(self): anotheruser cannot view someuser's data. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -516,7 +510,7 @@ def test_list_submissions_shared_as_anotheruser(self): anotheruser has view access on someuser's data. They can view all """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_list_url, {"format": "json"}) self.assertEqual(response.status_code, status.HTTP_200_OK) response_ids = [r['_id'] for r in response.data.get('results')] @@ -529,7 +523,7 @@ def test_list_submissions_with_partial_permissions_as_anotheruser(self): anotheruser has partial view access on someuser's project. They can view only the data they submitted to someuser's project. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VIEW_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -582,7 +576,7 @@ def test_list_submissions_asset_publicly_shared_as_authenticated_user(self): """ anonymous_user = get_anonymous_user() - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) # Give the user who will access the public data--without any explicit # permission assignment--their own asset. This is needed to expose a @@ -604,7 +598,7 @@ def test_list_submissions_asset_publicly_shared_and_shared_with_user_as_anotheru unable to view submission data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) anonymous_user = get_anonymous_user() assert not self.asset.has_perm(self.anotheruser, PERM_VIEW_ASSET) @@ -651,14 +645,17 @@ def test_list_query_elem_match(self): question = 'q3' submission[group] = [ { - f'{question}': 'whap.gif', + f'{group}/{question}': 'whap.gif', }, + { + f'{group}/{question}': 'whop.gif', + } ] + self.asset.deployment.mock_submissions([submission]) - # FIXME with attachments data = { - 'query': f'{{"{group}/{question}":{{"$exists":true}}}}', + 'query': f'{{"{group}":{{"$elemMatch":{{"{group}/{question}":{{"$exists":true}}}}}}}}', 'format': 'json', } response = self.client.get(self.submission_list_url, data) @@ -710,7 +707,7 @@ def test_retrieve_submission_not_shared_as_anotheruser(self): anotheruser has no access to someuser's data someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), @@ -728,7 +725,7 @@ def test_retrieve_submission_shared_as_anotheruser(self): anotheruser has view access to someuser's data. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), @@ -747,7 +744,7 @@ def test_retrieve_submission_with_partial_permissions_as_anotheruser(self): anotheruser has partial view access to someuser's data. They can only see their own data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VIEW_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -866,7 +863,7 @@ def test_delete_submission_not_shared_as_anotheruser(self): anotheruser cannot view someuser's data, therefore they cannot delete it. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), @@ -886,7 +883,7 @@ def test_delete_submission_shared_as_anotheruser(self): anotheruser can view someuser's data but they cannot delete it. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) submission = self.submissions_submitted_by_unknownuser[0] url = reverse( self._get_endpoint('submission-detail'), @@ -917,7 +914,7 @@ def test_delete_submission_with_partial_perms_as_anotheruser(self): anotheruser has partial access to someuser's data. anotheruser can only view/delete their data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_DELETE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -1081,7 +1078,7 @@ def test_attachments_rewrite(self): asset.deployment.mock_submissions([submission]) asset.deployment.set_namespace(self.URL_NAMESPACE) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) url = reverse( self._get_endpoint('submission-detail'), kwargs={ @@ -1100,16 +1097,22 @@ def test_attachments_rewrite(self): 'group_ec9yq67/group_dq8as25[1]/group_xt0za80[2]/my_attachment', 'group_ec9yq67/group_dq8as25[2]/group_xt0za80[1]/my_attachment' ] + + submission_id = submission['_id'] + attachment_0_id = submission['_attachments'][0]['id'] + attachment_1_id = submission['_attachments'][1]['id'] + attachment_2_id = submission['_attachments'][2]['id'] + expected_new_download_urls = [ 'http://testserver/api/v2/assets/' + asset.uid - + '/data/1000/attachments/1/?format=json', + + f"/data/{submission_id}/attachments/{attachment_0_id}/?format=json", 'http://testserver/api/v2/assets/' + asset.uid - + '/data/1000/attachments/2/?format=json', + + f"/data/{submission_id}/attachments/{attachment_1_id}/?format=json", 'http://testserver/api/v2/assets/' + asset.uid - + '/data/1000/attachments/3/?format=json', + + f"/data/{submission_id}/attachments/{attachment_2_id}/?format=json", ] for idx, attachment in enumerate(attachments): @@ -1202,7 +1205,7 @@ def test_get_edit_link_submission_not_shared_as_anotheruser(self): anotheruser cannot view the project, therefore cannot edit data. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -1213,7 +1216,7 @@ def test_cannot_get_edit_link_submission_shared_with_view_as_anotheruser(self): someuser's data existence should not be revealed. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_url, {'format': 'json'}) # FIXME if anotheruser has view permissions, they should receive a 403 @@ -1227,7 +1230,7 @@ def test_get_edit_link_submission_shared_with_edit_as_anotheruser(self): anotheruser can retrieve enketo edit link """ self.asset.assign_perm(self.anotheruser, PERM_CHANGE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) ee_url = ( f'{settings.ENKETO_URL}/{settings.ENKETO_EDIT_INSTANCE_ENDPOINT}' @@ -1250,7 +1253,7 @@ def test_get_edit_link_with_partial_perms_as_anotheruser(self): anotheruser has partial permissions on someuser's data anotheruser can only view/edit their own data """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_CHANGE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -1689,7 +1692,7 @@ def test_cannot_get_view_link_submission_not_shared_as_anotheruser(self): anotheruser cannot view the project, therefore cannot retrieve enketo link. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.submission_view_link_url, {'format': 'json'}) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -1701,7 +1704,7 @@ def test_get_view_link_submission_shared_with_view_only_as_anotheruser(self): anotheruser can retrieve enketo view link. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) ee_url = ( f'{settings.ENKETO_URL}/{settings.ENKETO_VIEW_INSTANCE_ENDPOINT}' @@ -1723,7 +1726,7 @@ def test_get_view_link_with_partial_perms_as_anotheruser(self): anotheruser has partial view permissions on someuser's data anotheruser can only view their own data """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VIEW_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -1849,6 +1852,7 @@ def test_duplicate_submission_as_owner_allowed(self): someuser is the owner of the project. someuser is allowed to duplicate their own data """ + print('URL :', self.submission_url, flush=True) response = self.client.post(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_201_CREATED self._check_duplicate(response) @@ -1862,6 +1866,23 @@ def test_duplicate_submission_with_xml_encoding(self): assert submission_xml.startswith( '' ) + breakpoint() + self.test_duplicate_submission_as_owner_allowed() + + def test_duplicate_submission_without_xml_encoding(self): + submission_xml = self.asset.deployment.get_submissions( + user=self.asset.owner, + format_type=SUBMISSION_FORMAT_TYPE_XML, + submission_ids=[self.submission['_id']], + )[0] + assert submission_xml.startswith( + '' + ) + Instance.objects.filter(pk=self.submission['_id']).update( + xml=submission_xml.replace( + '', '' + ) + ) self.test_duplicate_submission_as_owner_allowed() def test_duplicate_submission_as_anotheruser_not_allowed(self): @@ -1871,7 +1892,7 @@ def test_duplicate_submission_as_anotheruser_not_allowed(self): anotheruser has no access to someuser's data and someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.post(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_404_NOT_FOUND @@ -1894,7 +1915,7 @@ def test_cannot_duplicate_submission_as_anotheruser_with_view_perm(self): edit/duplicate someuser's data. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.post(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_403_FORBIDDEN @@ -1906,7 +1927,7 @@ def test_duplicate_submission_as_anotheruser_with_change_perm_allowed(self): someuser's data. """ self.asset.assign_perm(self.anotheruser, PERM_CHANGE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.post(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_201_CREATED self._check_duplicate(response) @@ -1921,7 +1942,7 @@ def test_cannot_duplicate_submission_as_anotheruser_with_view_add_perms(self): """ for perm in [PERM_VIEW_SUBMISSIONS, PERM_ADD_SUBMISSIONS]: self.asset.assign_perm(self.anotheruser, perm) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.post(self.submission_url, {'format': 'json'}) assert response.status_code == status.HTTP_403_FORBIDDEN @@ -1932,7 +1953,7 @@ def test_duplicate_submission_as_anotheruser_with_partial_perms(self): anotheruser has partial change submissions permissions. They can edit/duplicate their own data only. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_CHANGE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] @@ -2063,7 +2084,7 @@ def test_cannot_bulk_update_submissions_as_anotheruser(self): anotheruser cannot access someuser's data. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.patch( self.submission_url, data=self.submitted_payload, format='json' ) @@ -2090,7 +2111,7 @@ def test_cannot_bulk_update_submissions_as_anotheruser_with_view_perm(self): update someuser's data """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.patch( self.submission_url, data=self.submitted_payload, format='json' ) @@ -2103,7 +2124,7 @@ def test_bulk_update_submissions_as_anotheruser_with_change_perm(self): anotheruser can edit view someuser's data """ self.asset.assign_perm(self.anotheruser, PERM_CHANGE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.patch( self.submission_url, data=self.submitted_payload, format='json' ) @@ -2116,7 +2137,7 @@ def test_bulk_update_submissions_as_anotheruser_with_partial_perms(self): The project is partially shared with anotheruser anotheruser can only edit their own data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) # Allow anotheruser to update their own data partial_perms = { @@ -2178,7 +2199,7 @@ def test_cannot_retrieve_status_of_not_shared_submission_as_anotheruser(self): anotheruser has no access to someuser's data. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -2190,7 +2211,7 @@ def test_retrieve_status_of_shared_submission_as_anotheruser(self): anotheruser can view validation status of submissions. """ self.asset.assign_perm(self.anotheruser, PERM_VIEW_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.get(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, {}) @@ -2227,7 +2248,7 @@ def test_cannot_delete_status_of_not_shared_submission_as_anotheruser(self): validation status. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.delete(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -2239,7 +2260,7 @@ def test_delete_status_of_shared_submission_as_anotheruser(self): anotheruser can delete validation status of the project. """ self.asset.assign_perm(self.anotheruser, PERM_VALIDATE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.delete(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) @@ -2284,7 +2305,7 @@ def test_cannot_edit_status_of_not_shared_submission_as_anotheruser(self): validate them. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) response = self.client.patch(self.validation_status_url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -2296,7 +2317,7 @@ def test_edit_status_of_shared_submission_as_anotheruser(self): anotheruser can edit validation status of the project. """ self.asset.assign_perm(self.anotheruser, PERM_VALIDATE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = { 'validation_status.uid': 'validation_status_not_approved' } @@ -2327,7 +2348,7 @@ def test_edit_status_with_partial_perms_as_anotheruser(self): anotheruser has partial access to someuser's data. anotheruser can only view and validate their data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VALIDATE_SUBMISSIONS: [{'_submitted_by': 'anotheruser'}] } @@ -2386,6 +2407,17 @@ def setUp(self): kwargs={'parent_lookup_asset': self.asset.uid, 'format': 'json'}, ) + # Ensure all submissions have no validation status + response = self.client.get( + self.submission_list_url, format='json' + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + emptied = [ + not s['_validation_status'] + for s in response.data['results'] + ] + self.assertTrue(all(emptied)) + # Make the owner change validation status of all submissions data = { 'payload': { @@ -2398,6 +2430,18 @@ def setUp(self): ) self.assertEqual(response.status_code, status.HTTP_200_OK) + def test_all_validation_statuses_applied(self): + # ensure all submissions are not approved + response = self.client.get( + self.submission_list_url, format='json' + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + applied = [ + s['_validation_status']['uid'] == 'validation_status_not_approved' + for s in response.data['results'] + ] + self.assertTrue(all(applied)) + def test_delete_all_status_as_owner(self): """ someuser is the owner of the project. @@ -2469,7 +2513,7 @@ def test_delete_status_of_not_shared_submissions_as_anotheruser(self): bulk delete the validation status of them. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = { 'payload': { 'validation_status.uid': None, @@ -2491,7 +2535,7 @@ def test_delete_status_of_shared_submissions_as_anotheruser(self): """ self.asset.assign_perm(self.anotheruser, PERM_VALIDATE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = { 'payload': { 'validation_status.uid': None, @@ -2616,7 +2660,7 @@ def test_cannot_edit_submission_validation_statuses_not_shared_as_anotheruser(se bulk edit the validation status of them. someuser's data existence should not be revealed. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = { 'payload': { 'validation_status.uid': 'validation_status_approved', @@ -2638,7 +2682,7 @@ def test_edit_submission_validation_statuses_as_anotheruser(self): at once. """ self.asset.assign_perm(self.anotheruser, PERM_VALIDATE_SUBMISSIONS) - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) data = { 'payload': { 'validation_status.uid': 'validation_status_approved', @@ -2693,7 +2737,7 @@ def test_edit_all_submission_validation_statuses_with_partial_perms_as_anotherus `confirm=true` must be sent when the request alters all their submissions at once. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VALIDATE_SUBMISSIONS: [ {'_submitted_by': 'anotheruser'}] @@ -2747,7 +2791,7 @@ def test_edit_some_submission_validation_statuses_with_partial_perms_as_anotheru The project is partially shared with anotheruser. anotheruser can only validate their own data. """ - self._log_in_as_another_user() + self.client.force_login(self.anotheruser) partial_perms = { PERM_VALIDATE_SUBMISSIONS: [ {'_submitted_by': 'anotheruser'}] diff --git a/kpi/tests/test_mongo_helper.py b/kpi/tests/test_mongo_helper.py index 48a1192b9a..538cfaf095 100644 --- a/kpi/tests/test_mongo_helper.py +++ b/kpi/tests/test_mongo_helper.py @@ -1,7 +1,6 @@ from __future__ import annotations import copy -import itertools from django.conf import settings from django.test import TestCase @@ -27,7 +26,7 @@ def test_get_instances(self): names = ('bob', 'alice') users = baker.make( settings.AUTH_USER_MODEL, - username=itertools.cycle(names), + username=iter(names), _quantity=2, ) assets = [] diff --git a/kpi/tests/test_utils.py b/kpi/tests/test_utils.py index 0ce45b8403..65a1092eb4 100644 --- a/kpi/tests/test_utils.py +++ b/kpi/tests/test_utils.py @@ -1,4 +1,3 @@ -# coding: utf-8 import os import re from copy import deepcopy @@ -12,6 +11,7 @@ SearchQueryTooShortException, QueryParserNotSupportedFieldLookup, ) +from kpi.tests.utils.dicts import convert_hierarchical_keys_to_nested_dict from kpi.utils.autoname import autoname_fields, autoname_fields_to_field from kpi.utils.autoname import autovalue_choices_in_place from kpi.utils.pyxform_compatibility import allow_choice_duplicates @@ -26,6 +26,145 @@ ) +class ConvertHierarchicalKeysToNestedDictTestCase(TestCase): + + def test_regular_group(self): + dict_ = { + 'group_lx4sf58/question_1': 'answer_1', + 'group_lx4sf58/question_2': 'answer_2' + } + + expected = { + 'group_lx4sf58': { + 'question_1': 'answer_1', + 'question_2': 'answer_2' + } + } + + assert convert_hierarchical_keys_to_nested_dict(dict_) == expected + + def test_nested_groups(self): + dict_ = { + 'parent_group/middle_group/inner_group/question_1': 'answer_1' + } + + expected = { + 'parent_group': { + 'middle_group': { + 'inner_group': { + 'question_1': 'answer_1' + } + } + } + } + + assert convert_hierarchical_keys_to_nested_dict(dict_) == expected + + def test_nested_repeated_groups(self): + dict_ = { + 'formhub/uuid': '61b5029a4d2e42b49a12b9a18c22449f', + 'group_lq3wx73': [ + { + 'group_lq3wx73/middle_group': [ + { + 'group_lq3wx73/middle_group/middle_q': 'middle 1.1.1.1', + 'group_lq3wx73/middle_group/inner_group': [ + { + 'group_lq3wx73/middle_group/inner_group/inner_q': 'inner 1.1.1.1' + }, + { + 'group_lq3wx73/middle_group/inner_group/inner_q': 'inner 1.1.1.2' + }, + ], + }, + { + 'group_lq3wx73/middle_group/middle_q': 'middle 1.1.2.1', + 'group_lq3wx73/middle_group/inner_group': [ + { + 'group_lq3wx73/middle_group/inner_group/inner_q': 'inner 1.1.2.1' + }, + { + 'group_lq3wx73/middle_group/inner_group/inner_q': 'inner 1.1.2.1' + }, + ], + }, + ] + }, + { + 'group_lq3wx73/middle_group': [ + { + 'group_lq3wx73/middle_group/middle_q': 'middle 1.2.1.1', + 'group_lq3wx73/middle_group/inner_group': [ + { + 'group_lq3wx73/middle_group/inner_group/inner_q': 'inner_q 1.2.1.1' + } + ], + } + ] + }, + ], + } + + expected = { + 'formhub': {'uuid': '61b5029a4d2e42b49a12b9a18c22449f'}, + 'group_lq3wx73': [ + { + 'middle_group': [ + { + 'middle_q': 'middle 1.1.1.1', + 'inner_group': [ + {'inner_q': 'inner 1.1.1.1'}, + {'inner_q': 'inner 1.1.1.2'}, + ], + }, + { + 'middle_q': 'middle 1.1.2.1', + 'inner_group': [ + {'inner_q': 'inner 1.1.2.1'}, + {'inner_q': 'inner 1.1.2.1'}, + ], + }, + ] + }, + { + 'middle_group': [ + { + 'middle_q': 'middle 1.2.1.1', + 'inner_group': [ + {'inner_q': 'inner_q 1.2.1.1'} + ], + } + ] + }, + ], + } + assert convert_hierarchical_keys_to_nested_dict(dict_) == expected + + def test_nested_repeated_groups_in_group(self): + dict_ = { + 'people/person': [ + { + 'people/person/name': 'Julius Caesar', + 'people/person/age': 55, + }, + { + 'people/person/name': 'Augustus', + 'people/person/age': 75, + }, + ], + } + + expected = { + 'people': { + 'person': [ + {'name': 'Julius Caesar', 'age': 55}, + {'name': 'Augustus', 'age': 75} + ] + } + } + assert convert_hierarchical_keys_to_nested_dict(dict_) == expected + + class UtilsTestCase(TestCase): def test_sluggify(self): diff --git a/kpi/tests/utils/dicts.py b/kpi/tests/utils/dicts.py index 3e25ddfe3c..d1e5790405 100644 --- a/kpi/tests/utils/dicts.py +++ b/kpi/tests/utils/dicts.py @@ -1,35 +1,87 @@ from __future__ import annotations -def nested_dict_from_keys(dict_: dict) -> dict: - """ - Transforms a dictionary with keys containing slashes into a nested - dictionary structure. +def convert_hierarchical_keys_to_nested_dict(dict_: dict) -> dict: """ + Converts a dictionary with flat keys containing slashes into a nested dictionary. + This function takes a dictionary where keys represent a hierarchical path, + separated by slashes (e.g., "level1/level2/level3"), and converts it into + a nested dictionary structure. Each part of the key becomes a level in the + resulting dictionary. + """ result = {} for key, value in dict_.items(): + # Split the key to get each level of hierarchy keys = key.split('/') sub_dict = result - for sub_key in keys[:-1]: - if sub_key not in sub_dict: - sub_dict[sub_key] = {} - sub_dict = sub_dict[sub_key] + # Traverse each part of the key except the last one to build the nested structure + # + # Example: + # In keys = ['a', 'b', 'c'], the sub-keys 'a' and 'b' represent intermediate + # levels in the nested dictionary structure, while 'c' is the last part, + # which corresponds to the point where we will actually assign the value + # and the appropriate depth we want. + for part in keys[:-1]: + if part not in sub_dict: + # Create an empty dictionary if the part does not exist + sub_dict[part] = {} + # Move deeper into the current level of the dictionary + sub_dict = sub_dict[part] + + # Handle the final part of the key if isinstance(value, list): - sub_dict[keys[-1]] = [ - { - sub_key.split('/')[-1]: sub_val - for sub_key, sub_val in item.items() - } - for item in value if item - ] + # If the value is a list, make sure the corresponding key exists as a list + if keys[-1] not in sub_dict: + sub_dict[keys[-1]] = [] + + # Iterate over each item in the list + for item in value: + if isinstance(item, dict): + # Clean the dictionary item and append it to the list + sub_dict[keys[-1]].append(_clean_keys(item)) + else: + # Append non-dictionary items directly to the list + sub_dict[keys[-1]].append(item) else: - sub_dict[keys[-1]] = ( - nested_dict_from_keys(value) - if isinstance(value, dict) - else value - ) + # Assign the value directly for non-list items + sub_dict[keys[-1]] = value return result + + +def _clean_keys(dict_: dict) -> dict: + """ + Removes the redundant parent segments from keys in a dictionary, + keeping only relevant parts for hierarchical nesting. + """ + + cleaned_dict = {} + + for key, value in dict_.items(): + # Get the last segment of the key after the last slash (see example + # in `convert_flat_keys_to_nested_dict` for more details). + cleaned_key = key.split('/')[-1] + + # Handle lists of dictionaries recursively + if isinstance(value, list): + cleaned_list = [] + for item in value: + if isinstance(item, dict): + # Recursively clean each dictionary in the list + cleaned_list.append(_clean_keys(item)) + else: + # Append non-dictionary items directly to the cleaned list + cleaned_list.append(item) + # Store the cleaned list under the cleaned key + cleaned_dict[cleaned_key] = cleaned_list + # Handle nested dictionaries recursively + elif isinstance(value, dict): + cleaned_dict[cleaned_key] = _clean_keys(value) + else: + # Assign the value directly if it is not a dictionary or list + cleaned_dict[cleaned_key] = value + + return cleaned_dict diff --git a/kpi/views/v2/data.py b/kpi/views/v2/data.py index 6ab9300769..982fec71a3 100644 --- a/kpi/views/v2/data.py +++ b/kpi/views/v2/data.py @@ -545,7 +545,7 @@ def duplicate(self, request, pk, *args, **kwargs): Creates a duplicate of the submission with a given `pk` """ deployment = self._get_deployment() - # Coerce to int because back end only finds matches with same type + # Coerce to int because the back end only finds matches with the same type submission_id = positive_int(pk) duplicate_response = deployment.duplicate_submission( submission_id=submission_id, request=request From 14d57a27c14dfd4b2b48941b528f42675b3bb1db Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 3 Oct 2024 16:57:31 -0400 Subject: [PATCH 10/10] Remove useless imports --- .../openrosa/apps/logger/tests/test_publish_xls.py | 1 + .../apps/project_ownership/tests/api/v2/test_api.py | 1 - .../tests/test_submission_extras_api_post.py | 1 - kobo/apps/trackers/submission_utils.py | 1 - kpi/permissions.py | 13 ------------- kpi/tests/api/v2/test_api_submissions.py | 1 - 6 files changed, 1 insertion(+), 17 deletions(-) diff --git a/kobo/apps/openrosa/apps/logger/tests/test_publish_xls.py b/kobo/apps/openrosa/apps/logger/tests/test_publish_xls.py index d5f315fb79..30647fdef6 100644 --- a/kobo/apps/openrosa/apps/logger/tests/test_publish_xls.py +++ b/kobo/apps/openrosa/apps/logger/tests/test_publish_xls.py @@ -13,6 +13,7 @@ from kobo.apps.openrosa.apps.logger.models.xform import XForm from kobo.apps.openrosa.libs.utils.logger_tools import report_exception + class TestPublishXLS(TestBase): def test_publish_xls(self): diff --git a/kobo/apps/project_ownership/tests/api/v2/test_api.py b/kobo/apps/project_ownership/tests/api/v2/test_api.py index 3420843798..7ae35e988e 100644 --- a/kobo/apps/project_ownership/tests/api/v2/test_api.py +++ b/kobo/apps/project_ownership/tests/api/v2/test_api.py @@ -13,7 +13,6 @@ InviteStatusChoices, Transfer, ) -from kobo.apps.project_ownership.tests.utils import MockServiceUsageSerializer from kobo.apps.trackers.utils import update_nlp_counter from kpi.constants import PERM_VIEW_ASSET diff --git a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py index 1502fcccf5..fcd0c2079d 100644 --- a/kobo/apps/subsequences/tests/test_submission_extras_api_post.py +++ b/kobo/apps/subsequences/tests/test_submission_extras_api_post.py @@ -1,4 +1,3 @@ -import uuid from copy import deepcopy from unittest.mock import patch diff --git a/kobo/apps/trackers/submission_utils.py b/kobo/apps/trackers/submission_utils.py index b388deebe1..f66a09474c 100644 --- a/kobo/apps/trackers/submission_utils.py +++ b/kobo/apps/trackers/submission_utils.py @@ -1,4 +1,3 @@ -import itertools import os import time import uuid diff --git a/kpi/permissions.py b/kpi/permissions.py index 2652b52bbc..4639cebc78 100644 --- a/kpi/permissions.py +++ b/kpi/permissions.py @@ -256,19 +256,6 @@ class AssetEditorSubmissionViewerPermission(AssetNestedObjectPermission): } -class AssetExportSettingsPermission(AssetNestedObjectPermission): - perms_map = { - 'GET': ['%(app_label)s.view_submissions'], - 'POST': ['%(app_label)s.manage_asset'], - } - - perms_map['OPTIONS'] = perms_map['GET'] - perms_map['HEAD'] = perms_map['GET'] - perms_map['PUT'] = perms_map['POST'] - perms_map['PATCH'] = perms_map['POST'] - perms_map['DELETE'] = perms_map['POST'] - - class AssetPermissionAssignmentPermission(AssetNestedObjectPermission): perms_map = AssetNestedObjectPermission.perms_map.copy() diff --git a/kpi/tests/api/v2/test_api_submissions.py b/kpi/tests/api/v2/test_api_submissions.py index 7101c92ac3..ab48e988a8 100644 --- a/kpi/tests/api/v2/test_api_submissions.py +++ b/kpi/tests/api/v2/test_api_submissions.py @@ -1866,7 +1866,6 @@ def test_duplicate_submission_with_xml_encoding(self): assert submission_xml.startswith( '' ) - breakpoint() self.test_duplicate_submission_as_owner_allowed() def test_duplicate_submission_without_xml_encoding(self):