Django 1.11 (#3815)

* Bump to django 1.11

* Fix test models

* Fix polymorphic queryset

* Adjust drf version

* Fix CustomFieldValueAdminMaxinTestCase

* Fix IPNetwork filtering

* Don't fail fast test matrix

* Fix polymorphic queryset again

* fix PermissionsSelectWidget

* fix test

* Fix serializers

* Fix isort

* fix InlineM2MAdminMixin

* Fix related_widget_wrapper

* Fix polymorphic querysets

* Adjust polymorphic querysets and prefetches

* Fix flake

* Don't run postgres. Fix flake

* Fix polymorphic for psql

* Optimize queries. Add tests

* Optimize API and improve API tests

* Fix flake. adjust query num

* Fix custom fields

* Fix different quote types in different db engines

* Fix isort

* Fix flake

* Fix missing f-string

* Fix tests

* Test API endpoints

* Test also json content

* Don't smoke test endpoints for django 1.11

* Fix transitions form

* Fix test_action_choices

* Bump django debug toolbar

* Remove empty include

* Fix default manager lookup

* Fix DC Visualization

* Fix user permissions form

* Fix user permissions form - cleanup

* Fix network terminators on save

* Cleanup

* Cleanup

---------

Co-authored-by: Olga Matyla <[email protected]>
Co-authored-by: awieckowski <[email protected]>

.github/workflows/main.yml

@@ -40,6 +40,7 @@ jobs:
     name: Missing migrations
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
         db-engine: ["psql", "mysql"]
     env:
@@ -117,7 +118,7 @@ jobs:
         run: pip3 install psycopg2-binary
 
       # Runs a set of commands using the runners shell
-      - name: Check missings migrations
+      - name: Check missing migrations
         run: "${GITHUB_WORKSPACE}/scripts/check_missing_migrations.sh"
 
   settings_test:

requirements/base.txt

@@ -1,21 +1,21 @@
 -r openstack.txt
 -r hermes.txt
-Django==1.10.8
+Django==1.11.8
 dj.choices==0.11.0
 django-extensions==1.7.5
-django-filter==0.13.0
+django-filter==1.0.4
 django-import-export==1.2.0
 django-money==0.12
 py-moneyed==1.2
 django-mptt==0.8.7
 django-reversion==2.0
 django-rq==2.0
-django-sitetree==1.7.0
+django-sitetree==1.13.0
 django-taggit==0.22.2
 django-taggit-serializer==0.1.7
 django-threadlocals==0.8
 django-cryptography==0.3
-djangorestframework==3.4.0
+djangorestframework==3.4.1
 djangorestframework_xml==1.2.0
 drf-nested-routers==0.11.1
 Markdown<3.0  # headerid extension removed in 3.0 - see #3313 for details

requirements/dev.txt

@@ -1,5 +1,5 @@
 -r test.txt
-django-debug-toolbar==1.8
+django-debug-toolbar==1.11
 werkzeug==0.16.1
 pudb
 ipython

setup.cfg

@@ -1,4 +1,4 @@
 [flake8]
 exclude = migrations,tests,docs,settings*,*local.py
 max-complexity = 15
-max-line-length = 80
+max-line-length = 100

src/ralph/accounts/admin.py

@@ -7,9 +7,9 @@
 from django.contrib.auth.admin import GroupAdmin, UserAdmin
 from django.contrib.auth.models import Group
 from django.core.exceptions import PermissionDenied
-from django.core.urlresolvers import reverse
 from django.db.models import Q
 from django.forms.models import model_to_dict
+from django.urls import reverse
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from reversion import revisions as reversion

src/ralph/accounts/helpers.py

@@ -2,7 +2,7 @@
 from urllib.parse import urlencode
 
 from django.conf import settings
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 from ralph.access_cards.models import AccessCard
 from ralph.admin.sites import ralph_site

src/ralph/accounts/tests/tests.py

@@ -5,8 +5,8 @@
 from django.conf import settings
 from django.contrib.auth.hashers import check_password
 from django.contrib.auth.models import Permission
-from django.core.urlresolvers import reverse
 from django.test import TestCase
+from django.urls import reverse
 from rest_framework import status
 
 from ralph.accounts.ldap import manager_country_attribute_populate
@@ -203,7 +203,7 @@ def setUp(self):
             settings.INVENTORY_TAG_USER,
             self.date_tag,
         ]
-    
+
     def test_tag_asset(self):
         self.assertTrue(self.login_as_user(self.user1))
         response = self.client.post(

src/ralph/accounts/views.py

@@ -3,9 +3,9 @@
 
 from django.conf import settings
 from django.contrib import messages
-from django.core.urlresolvers import reverse
 from django.http import HttpResponseForbidden, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import View
 from reversion import revisions as reversion

src/ralph/admin/filters.py

@@ -10,10 +10,10 @@
 from django.contrib.admin.filters import FieldListFilter
 from django.contrib.admin.options import IncorrectLookupParameters
 from django.contrib.admin.utils import get_model_from_relation
-from django.core.urlresolvers import reverse
 from django.db import models
 from django.db.models import Q
 from django.forms.utils import flatatt
+from django.urls import reverse
 from django.utils.encoding import smart_text
 from django.utils.formats import get_format
 from django.utils.html import conditional_escape, mark_safe

src/ralph/admin/helpers.py

@@ -3,9 +3,9 @@
 
 from django.contrib.admin.utils import get_fields_from_path
 from django.contrib.contenttypes.models import ContentType
-from django.core.urlresolvers import reverse
 from django.db.models.constants import LOOKUP_SEP
 from django.db.models.expressions import Func
+from django.urls import reverse
 
 
 def get_admin_url(obj, action):

src/ralph/admin/m2m.py

@@ -53,19 +53,16 @@ class AuthorAdmin(admin.ModelAdmin):
     modelformset_factory
 )
 from django.utils.text import get_text_list
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
-from ralph.admin.mixins import (
-    RalphAdminForm,
-    RalphStackedInline,
-    RalphTabularInline
-)
+from ralph.admin.mixins import RalphAdminForm, RalphTabularInline
 
 
 class BaseInlineM2MFormset(BaseInlineFormSet):
     """
     Adjust Django's BaseInlineFormSet to cooperate with M2M relations
     """
+
     def __init__(self, *args, **kwargs):
         self._fk = self.fk
         # m2m is assigned by m2minlineformset_factory
@@ -87,7 +84,7 @@ def save_new(self, form, commit=True):
 
     @classmethod
     def get_default_prefix(cls):
-        return cls.m2m.rel.get_accessor_name(model=cls.model).replace('+', '')
+        return cls.m2m.rel.get_accessor_name(model=cls.model).replace("+", "")
 
     def save_existing_objects(self, commit=True):
         """
@@ -115,9 +112,7 @@ def save_existing_objects(self, commit=True):
                 getattr(obj, self.m2m.attname).remove(self.instance)
             elif form.has_changed():
                 self.changed_objects.append((obj, form.changed_data))
-                saved_instances.append(
-                    self.save_existing(form, obj, commit=commit)
-                )
+                saved_instances.append(self.save_existing(form, obj, commit=commit))
                 if not commit:
                     self.saved_forms.append(form)
         return saved_instances
@@ -130,8 +125,11 @@ def get_m2m(parent_model, model):
     # need to check on both sides (m2m field will be defined only in one of the
     # models)
     for rel in parent_model._meta.get_fields(include_hidden=True):
-        if rel.many_to_many and rel.auto_created and \
-                issubclass(rel.related_model, model):
+        if (
+            rel.many_to_many
+            and rel.auto_created
+            and issubclass(rel.related_model, model)
+        ):
             return rel.field
 
     for rel in parent_model._meta.many_to_many:
@@ -148,21 +146,33 @@ def get_foreign_key_for_m2m(parent_model, m2m):
         m2m: ManyToManyField relation instance
     """
     for field in m2m.remote_field.through._meta.fields:
-        if (
-            isinstance(field, ForeignKey) and
-            issubclass(parent_model, field.rel.to)
+        if isinstance(field, ForeignKey) and issubclass(
+            parent_model, field.remote_field.model
         ):
             return field
 
 
 def m2minlineformset_factory(
-    parent_model, model, form=RalphAdminForm,
-    formset=BaseInlineM2MFormset, fk_name=None,
-    fields=None, exclude=None, extra=3, can_order=False,
-    can_delete=True, max_num=None, formfield_callback=None,
-    widgets=None, validate_max=False, localized_fields=None,
-    labels=None, help_texts=None, error_messages=None,
-    min_num=None, validate_min=False
+    parent_model,
+    model,
+    form=RalphAdminForm,
+    formset=BaseInlineM2MFormset,
+    fk_name=None,
+    fields=None,
+    exclude=None,
+    extra=3,
+    can_order=False,
+    can_delete=True,
+    max_num=None,
+    formfield_callback=None,
+    widgets=None,
+    validate_max=False,
+    localized_fields=None,
+    labels=None,
+    help_texts=None,
+    error_messages=None,
+    min_num=None,
+    validate_min=False,
 ):
     """
     Creates m2m formset. This function is very similar to
@@ -173,23 +183,23 @@ def m2minlineformset_factory(
     m2m = get_m2m(parent_model, model)
     fk = get_foreign_key_for_m2m(parent_model, m2m)
     kwargs = {
-        'form': form,
-        'formfield_callback': formfield_callback,
-        'formset': formset,
-        'extra': extra,
-        'can_delete': can_delete,
-        'can_order': can_order,
-        'fields': fields,
-        'exclude': exclude,
-        'min_num': min_num,
-        'max_num': max_num,
-        'widgets': widgets,
-        'validate_min': validate_min,
-        'validate_max': validate_max,
-        'localized_fields': localized_fields,
-        'labels': labels,
-        'help_texts': help_texts,
-        'error_messages': error_messages,
+        "form": form,
+        "formfield_callback": formfield_callback,
+        "formset": formset,
+        "extra": extra,
+        "can_delete": can_delete,
+        "can_order": can_order,
+        "fields": fields,
+        "exclude": exclude,
+        "min_num": min_num,
+        "max_num": max_num,
+        "widgets": widgets,
+        "validate_min": validate_min,
+        "validate_max": validate_max,
+        "localized_fields": localized_fields,
+        "labels": labels,
+        "help_texts": help_texts,
+        "error_messages": error_messages,
     }
     FormSet = modelformset_factory(model, **kwargs)
     FormSet.m2m = m2m
@@ -207,47 +217,36 @@ def get_formset(self, request, obj=None, **kwargs):
         The only difference here comparing to the original one is in the last
         line.
         """
-        if 'fields' in kwargs:
-            fields = kwargs.pop('fields')
+        if "fields" in kwargs:
+            fields = kwargs.pop("fields")
         else:
             fields = flatten_fieldsets(self.get_fieldsets(request, obj))
-        if self.exclude is None:
-            exclude = []
-        else:
-            exclude = list(self.exclude)
+        excluded = self.get_exclude(request, obj)
+        exclude = [] if excluded is None else list(excluded)
         exclude.extend(self.get_readonly_fields(request, obj))
-        if (
-            self.exclude is None and
-            hasattr(self.form, '_meta') and
-            self.form._meta.exclude
-        ):
+        if excluded is None and hasattr(self.form, "_meta") and self.form._meta.exclude:
             # Take the custom ModelForm's Meta.exclude into account only if the
             # InlineModelAdmin doesn't define its own.
             exclude.extend(self.form._meta.exclude)
         # If exclude is an empty list we use None, since that's the actual
         # default.
         exclude = exclude or None
-        can_delete = (
-            self.can_delete and
-            self.has_delete_permission(request, obj)
-        )
+        can_delete = self.can_delete and self.has_delete_permission(request, obj)
         defaults = {
             "form": self.form,
             "formset": self.formset,
             "fk_name": self.fk_name,
             "fields": fields,
             "exclude": exclude,
-            "formfield_callback": partial(
-                self.formfield_for_dbfield, request=request
-            ),
+            "formfield_callback": partial(self.formfield_for_dbfield, request=request),
             "extra": self.get_extra(request, obj, **kwargs),
             "min_num": self.get_min_num(request, obj, **kwargs),
             "max_num": self.get_max_num(request, obj, **kwargs),
             "can_delete": can_delete,
         }
 
         defaults.update(kwargs)
-        base_model_form = defaults['form']
+        base_model_form = defaults["form"]
 
         class DeleteProtectedModelForm(base_model_form):
             def hand_clean_DELETE(self):
@@ -266,50 +265,46 @@ def hand_clean_DELETE(self):
                         objs = []
                         for p in collector.protected:
                             objs.append(
-                                # Translators: Model verbose name and instance
-                                # representation, suitable to be an item in a
-                                # list.
-                                _('%(class_name)s %(instance)s') % {
-                                    'class_name': p._meta.verbose_name,
-                                    'instance': p}
+                                # Translators: Model verbose name and instance representation,
+                                # suitable to be an item in a list.
+                                _("%(class_name)s %(instance)s")
+                                % {"class_name": p._meta.verbose_name, "instance": p}
                             )
                         params = {
-                            'class_name': self._meta.model._meta.verbose_name,
-                            'instance': self.instance,
-                            'related_objects': get_text_list(objs, _('and'))
+                            "class_name": self._meta.model._meta.verbose_name,
+                            "instance": self.instance,
+                            "related_objects": get_text_list(objs, _("and")),
                         }
                         msg = _(
-                            "Deleting %(class_name)s %(instance)s would require"
-                            " deleting the following protected related objects:"
-                            " %(related_objects)s"
+                            "Deleting %(class_name)s %(instance)s would require "
+                            "deleting the following protected related objects: "
+                            "%(related_objects)s"
                         )
                         raise ValidationError(
-                            msg, code='deleting_protected', params=params
+                            msg, code="deleting_protected", params=params
                         )
 
             def is_valid(self):
                 result = super(DeleteProtectedModelForm, self).is_valid()
                 self.hand_clean_DELETE()
                 return result
 
-        defaults['form'] = DeleteProtectedModelForm
+        defaults["form"] = DeleteProtectedModelForm
 
-        if (
-            defaults['fields'] is None and
-            not modelform_defines_fields(defaults['form'])
+        if defaults["fields"] is None and not modelform_defines_fields(
+            defaults["form"]
         ):
-            defaults['fields'] = forms.ALL_FIELDS
+            defaults["fields"] = forms.ALL_FIELDS
 
         # THE ONLY DIFFERENCE HERE COMPARING TO ORIGINAL \/
         # return m2minlineformset_factory instead of inlineformset_factory
-        return m2minlineformset_factory(
-            self.parent_model, self.model, **defaults
-        )
 
+        # dirty fix for through queryset being passed as self.model in init in some cases
+        if hasattr(self.model._meta, "auto_created") and self.model._meta.auto_created:
+            self.model = self.model._meta.auto_created
 
-class RalphTabularM2MInline(InlineM2MAdminMixin, RalphTabularInline):
-    template = 'admin/edit_inline/tabular_m2m.html'
+        return m2minlineformset_factory(self.parent_model, self.model, **defaults)
 
 
-class RalphStackedM2MInline(InlineM2MAdminMixin, RalphStackedInline):
-    pass
+class RalphTabularM2MInline(InlineM2MAdminMixin, RalphTabularInline):
+    template = "admin/edit_inline/tabular_m2m.html"