Improve & test gradebook csv export/import.

course/grades.py

@@ -47,7 +47,7 @@
 from crispy_forms.layout import Submit
 from bootstrap3_datetime.widgets import DateTimePicker
 
-from course.utils import course_view, render_course_page
+from course.utils import course_view, render_course_page, RelateCsvHandler
 from course.models import (
         Participation, participation_status,
         GradingOpportunity, GradeChange, GradeStateMachine,
@@ -56,13 +56,15 @@
 from course.flow import adjust_flow_session_page_data
 from course.views import get_now_or_fake_time
 from course.constants import (
-        participation_permission as pperm,
+        participation_permission as pperm, MAX_EXTRA_CREDIT_FACTOR
         )
 
+csv_handler = RelateCsvHandler()
+
 # {{{ for mypy
 
 if False:
-    from typing import Tuple, Text, Optional, Any, Iterable, List  # noqa
+    from typing import Tuple, Text, Optional, Any, Iterable, List, Union, Dict  # noqa
     from course.utils import CoursePageContext  # noqa
     from course.content import FlowDesc  # noqa
     from course.models import Course, FlowPageVisitGrade  # noqa
@@ -203,8 +205,17 @@ def __init__(self, opportunity, grade_state_machine):
         self.grade_state_machine = grade_state_machine
 
 
-def get_grade_table(course):
-    # type: (Course) -> Tuple[List[Participation], List[GradingOpportunity], List[List[GradeInfo]]]  # noqa
+def get_grade_table(course, excluded_roles=None):
+    # type: (Course, Optional[List[Text]]) -> Tuple[List[Participation], List[GradingOpportunity], List[List[GradeInfo]]]  # noqa
+
+    participations_exlcude_kwargs = {}  # type: Dict
+    grade_changes_exclude_kwargs = {}  # type: Dict
+    if excluded_roles:
+        assert isinstance(excluded_roles, (list, tuple))
+        participations_exlcude_kwargs = {
+            "roles__identifier__in": excluded_roles}
+        grade_changes_exclude_kwargs = {
+            "participation__roles__identifier__in": excluded_roles}
 
     # NOTE: It's important that these queries are sorted consistently,
     # also consistently with the code below.
@@ -219,13 +230,15 @@ def get_grade_table(course):
             .filter(
                 course=course,
                 status=participation_status.active)
+            .exclude(**participations_exlcude_kwargs)
             .order_by("id")
             .select_related("user"))
 
     grade_changes = list(GradeChange.objects
             .filter(
                 opportunity__course=course,
                 opportunity__shown_in_grade_book=True)
+            .exclude(**grade_changes_exclude_kwargs)
             .order_by(
                 "participation__id",
                 "opportunity__identifier",
@@ -295,43 +308,195 @@ def grade_key(entry):
         })
 
 
+class ExportGradeBookForm(StyledForm):
+    def __init__(self, *args, **kwargs):
+        # type: (*Any, **Any) -> None  # noqa
+        super(ExportGradeBookForm, self).__init__(*args, **kwargs)
+
+        self.fields["user_info_fields"] = forms.ChoiceField(
+            choices=(csv_handler.export_csv_fields_options),
+            initial=csv_handler.export_csv_fields_options[0],
+            label=_("User Fields to Export"),
+        )
+
+        self.fields["exclude_instructors"] = forms.BooleanField(
+            required=False,
+            initial=True,
+            label=_("Exclude Grades of Instructors"),
+        )
+
+        self.fields["exclude_tas"] = forms.BooleanField(
+            required=False,
+            initial=True,
+            label=_("Exclude Grades of Teaching assistants"),
+        )
+
+        self.fields["zero_for_state_none"] = forms.BooleanField(
+            required=False,
+            initial=False,
+            label=_("Use 0 point for grade state `NONE`, which possibly "
+                    "means the opptunity had not been graded."),
+        )
+
+        self.fields["zero_for_graded_none"] = forms.BooleanField(
+            required=False,
+            initial=True,
+            label=_("Use 0 for graded opptunity which has `NONE` points"),
+        )
+
+        self.fields["maximum_points"] = forms.FloatField(
+            required=True,
+            min_value=0,
+            max_value=100 * MAX_EXTRA_CREDIT_FACTOR,
+            initial=100,
+            label=_("Maximum allowed points"),
+        )
+
+        self.fields["minimum_points"] = forms.FloatField(
+            required=True,
+            initial=0,
+            min_value=0,
+            max_value=100 * MAX_EXTRA_CREDIT_FACTOR,
+            label=_("Minimum points"),
+        )
+
+        self.fields["round_digits"] = forms.IntegerField(
+            required=True,
+            min_value=0,
+            initial=0,
+            max_value=2,
+            label=_("Round Digits"),
+        )
+
+        self.fields["encoding_used"] = forms.ChoiceField(
+            choices=tuple(csv_handler.export_csv_encodings_options),
+            label=_("Using Encoding"),
+            initial=csv_handler.export_csv_encodings_options[0],
+            help_text=_(
+                "The encoding used for the exported file."),
+        )
+
+        if len(csv_handler.export_csv_fields_options) == 1:
+            self.fields["user_info_fields"].widget = forms.HiddenInput()
+
+        if len(csv_handler.export_csv_encodings_options) == 1:
+            self.fields["encoding_used"].widget = forms.HiddenInput()
+
+        self.helper.add_input(Submit("download", _("Download")))
+
+    def clean_user_info_fields(self):
+        user_info_fields_str = self.cleaned_data["user_info_fields"]
+        user_info_fields = user_info_fields_str.split(",")
+        return user_info_fields
+
+    def clean(self):
+        data = super(ExportGradeBookForm, self).clean()
+        minimum_points = data.get("minimum_points")
+        maximum_points = data.get("maximum_points")
+        if minimum_points is not None and maximum_points is not None:
+            if maximum_points <= minimum_points:
+                self.add_error(
+                    'maximum_points',
+                    _("'maximum_points' must be greater than "
+                      "'minimum_points'."))
+
+
 @course_view
 def export_gradebook_csv(pctx):
     if not pctx.has_permission(pperm.batch_export_grade):
         raise PermissionDenied(_("may not batch-export grades"))
 
-    participations, grading_opps, grade_table = get_grade_table(pctx.course)
+    request = pctx.request
+
+    if request.method == "POST":
+        form = ExportGradeBookForm(request.POST)
+
+        if form.is_valid():
+            excluded_roles = []
+            if form.cleaned_data["exclude_instructors"]:
+                excluded_roles.append("instructor")
+            if form.cleaned_data["exclude_tas"]:
+                excluded_roles.append("ta")
+
+            encoding_used = form.cleaned_data["encoding_used"]
 
-    from six import StringIO
-    csvfile = StringIO()
+            participations, grading_opps, grade_table = (
+                get_grade_table(pctx.course, excluded_roles))
+
+            from six import StringIO
+            csvfile = StringIO()
+
+            if six.PY2:
+                import unicodecsv as csv
+            else:
+                import csv
+
+            info_fields = form.cleaned_data["user_info_fields"]
+
+            user_info_fields_verbose_names = (
+                csv_handler.get_user_fields_verbose_names(info_fields))
+
+            fieldnames = user_info_fields_verbose_names + [
+                    gopp.identifier for gopp in grading_opps]
+
+            writer = csv.writer(csvfile)
+
+            writer.writerow(fieldnames)
+
+            alias_for_state_none = None
+            if form.cleaned_data["zero_for_state_none"]:
+                alias_for_state_none = "0"
+
+            alias_for_graded_none = None
+            if form.cleaned_data["zero_for_graded_none"]:
+                alias_for_graded_none = "0"
+
+            def callback_for_percentage(percentage):
+                maximum_points = form.cleaned_data["maximum_points"]
+                minimum_points = form.cleaned_data["minimum_points"]
+                round_digits = form.cleaned_data["round_digits"]
+                assert maximum_points > minimum_points
+
+                if percentage > maximum_points:
+                    percentage = maximum_points
+                elif percentage < minimum_points:
+                    percentage = minimum_points
+                if round_digits == 0:
+                    return str(int(round(percentage, 0)))
+                return '{0:.{1}f}'.format(
+                    round(percentage, round_digits), round_digits)
+
+            def get_user_info_from_info_fields(user, info_fields):
+                result = []
+                for attr in info_fields:
+                    value = getattr(user, attr, "None")
+                    result.append(str(value))
+                return result
+
+            for participation, grades in zip(participations, grade_table):
+                writer.writerow(
+                    get_user_info_from_info_fields(participation.user, info_fields)
+                    + [grade_info.grade_state_machine.stringify_machine_readable_state(  # noqa
+                        alias_for_state_none=alias_for_state_none,
+                        alias_for_graded_none=alias_for_graded_none,
+                        callback_for_percentage=callback_for_percentage)
+                        for grade_info in grades])
+
+            response = http.HttpResponse(
+                    csvfile.getvalue().encode(encoding_used),
+                    content_type="text/plain; charset=utf-8")
+            response['Content-Disposition'] = (
+                    'attachment; filename="grades-%s.csv"'
+                    % pctx.course.identifier)
+            return response
 
-    if six.PY2:
-        import unicodecsv as csv
     else:
-        import csv
-
-    fieldnames = ['user_name', 'last_name', 'first_name'] + [
-            gopp.identifier for gopp in grading_opps]
-
-    writer = csv.writer(csvfile)
-
-    writer.writerow(fieldnames)
-
-    for participation, grades in zip(participations, grade_table):
-        writer.writerow([
-            participation.user.username,
-            participation.user.last_name,
-            participation.user.first_name,
-            ] + [grade_info.grade_state_machine.stringify_machine_readable_state()
-                for grade_info in grades])
-
-    response = http.HttpResponse(
-            csvfile.getvalue().encode("utf-8"),
-            content_type="text/plain; charset=utf-8")
-    response['Content-Disposition'] = (
-            'attachment; filename="grades-%s.csv"'
-            % pctx.course.identifier)
-    return response
+        form = ExportGradeBookForm()
+
+    return render_course_page(pctx, "course/generic-course-form.html", {
+        "form": form,
+        "form_description": _("Export Gradebook as a CSV File")
+        })
 
 # }}}
 

course/models.py

@@ -1759,16 +1759,27 @@ def stringify_state(self):
         else:
             return "_((other state))"
 
-    def stringify_machine_readable_state(self):
+    def stringify_machine_readable_state(self, alias_for_state_none=None,
+                                         alias_for_graded_none=None,
+                                         callback_for_percentage=None):
+        if alias_for_state_none is None:
+            alias_for_state_none = u"NONE"
+
+        if alias_for_graded_none is None:
+            alias_for_graded_none = u"NONE"
+
+        if callback_for_percentage is None:
+            callback_for_percentage = lambda x: "%.3f" % x  # noqa
+
         if self.state is None:
-            return u"NONE"
+            return alias_for_state_none
         elif self.state == grade_state_change_types.exempt:
             return "EXEMPT"
         elif self.state == grade_state_change_types.graded:
             if self.valid_percentages:
-                return "%.3f" % self.percentage()
+                return callback_for_percentage(self.percentage())
             else:
-                return u"NONE"
+                return alias_for_graded_none
         else:
             return u"OTHER_STATE"