From accc87da543e73e6709a8a921984b890ea79f579 Mon Sep 17 00:00:00 2001 From: Brian Gontowski Date: Mon, 25 Nov 2013 23:37:10 -0800 Subject: [PATCH 1/5] First pass at a DictField query --- django_mongodb_engine/contrib/__init__.py | 41 +++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/django_mongodb_engine/contrib/__init__.py b/django_mongodb_engine/contrib/__init__.py index acd3223b..ee81af1b 100644 --- a/django_mongodb_engine/contrib/__init__.py +++ b/django_mongodb_engine/contrib/__init__.py @@ -1,11 +1,18 @@ import sys +import re from django.db import models, connections from django.db.models.query import QuerySet from django.db.models.sql.query import Query as SQLQuery +from django.db.models.query_utils import Q +from django.db.models.constants import LOOKUP_SEP +from django_mongodb_engine.compiler import OPERATORS_MAP, NEGATED_OPERATORS_MAP +from djangotoolbox.fields import AbstractIterableField ON_PYPY = hasattr(sys, 'pypy_version_info') +ALL_OPERATORS = dict(list(OPERATORS_MAP.items() + NEGATED_OPERATORS_MAP.items())).keys() +MONGO_DOT_FIELDS = ('DictField',) def _compiler_for_queryset(qs, which='SQLCompiler'): @@ -84,6 +91,40 @@ def __repr__(self): class MongoDBQuerySet(QuerySet): + def _filter_or_exclude(self, negate, *args, **kwargs): + if args or kwargs: + assert self.query.can_filter(), \ + "Cannot filter a query once a slice has been taken." + + clone = self._clone() + + all_field_names = self.model._meta.get_all_field_names() + base_field_names = [] + + for f in all_field_names: + field = self.model._meta.get_field_by_name(f)[0] + if '.' not in f and field.get_internal_type() in MONGO_DOT_FIELDS: + base_field_names.append(f) + + for k, v in kwargs.items(): + if LOOKUP_SEP in k and k.split(LOOKUP_SEP)[0] in base_field_names: + del kwargs[k] + for s in ALL_OPERATORS: + if k.endswith(s): + k = re.sub(LOOKUP_SEP + s + '$', '#' + s, k) + break + k = k.replace(LOOKUP_SEP, '.').replace('#', LOOKUP_SEP) + kwargs[k] = v + f = k.split(LOOKUP_SEP)[0] + if '.' in f and f not in all_field_names: + field = AbstractIterableField(blank=True, null=True, editable=False) + field.contribute_to_class(self.model, f) + + if negate: + clone.query.add_q(~Q(*args, **kwargs)) + else: + clone.query.add_q(Q(*args, **kwargs)) + return clone def map_reduce(self, *args, **kwargs): """ From a5dd4024daf58a0526b55089b650f16976008b1e Mon Sep 17 00:00:00 2001 From: Brian Gontowski Date: Mon, 25 Nov 2013 23:53:49 -0800 Subject: [PATCH 2/5] Fixed saving after a DictField query --- django_mongodb_engine/compiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django_mongodb_engine/compiler.py b/django_mongodb_engine/compiler.py index 3f583ed2..861246ed 100644 --- a/django_mongodb_engine/compiler.py +++ b/django_mongodb_engine/compiler.py @@ -363,6 +363,9 @@ def insert(self, docs, return_id=False): doc.clear() else: raise DatabaseError("Can't save entity with _id set to None") + for d in doc.keys(): + if '.' in d: + del doc[d] collection = self.get_collection() options = self.connection.operation_flags.get('save', {}) From 13a6b5876d1e1f258ef04bf3d2179f3fbcbcd305 Mon Sep 17 00:00:00 2001 From: Brian Gontowski Date: Tue, 26 Nov 2013 19:52:50 -0800 Subject: [PATCH 3/5] Added additional field types to dot lookup --- django_mongodb_engine/contrib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mongodb_engine/contrib/__init__.py b/django_mongodb_engine/contrib/__init__.py index ee81af1b..24dabce4 100644 --- a/django_mongodb_engine/contrib/__init__.py +++ b/django_mongodb_engine/contrib/__init__.py @@ -12,7 +12,7 @@ ON_PYPY = hasattr(sys, 'pypy_version_info') ALL_OPERATORS = dict(list(OPERATORS_MAP.items() + NEGATED_OPERATORS_MAP.items())).keys() -MONGO_DOT_FIELDS = ('DictField',) +MONGO_DOT_FIELDS = ('DictField', 'ListField', 'SetField', 'EmbeddedModelField') def _compiler_for_queryset(qs, which='SQLCompiler'): From f16980e5005484c3fa6e69740925ff6d76d9bd90 Mon Sep 17 00:00:00 2001 From: Brian Gontowski Date: Tue, 26 Nov 2013 20:06:35 -0800 Subject: [PATCH 4/5] Updated authors --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 7a3bd943..aad296d0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -21,6 +21,7 @@ Contributions by * Sabin Iacob (https://github.com/m0n5t3r) * kryton (https://github.com/kryton) * Brandon Pedersen (https://github.com/bpedman) +* Brian Gontowski (https://github.com/Molanda) (For an up-to-date list of contributors, see https://github.com/django-mongodb-engine/mongodb-engine/contributors.) From f34673e44de195a83455ce70ddef58d7317a67ac Mon Sep 17 00:00:00 2001 From: Brian Gontowski Date: Wed, 27 Nov 2013 07:34:43 -0800 Subject: [PATCH 5/5] Added tests for dot queries --- django_mongodb_engine/contrib/__init__.py | 43 +++++++----- tests/dotquery/__init__.py | 0 tests/dotquery/models.py | 17 +++++ tests/dotquery/tests.py | 81 +++++++++++++++++++++++ tests/dotquery/utils.py | 35 ++++++++++ tests/settings/__init__.py | 1 + 6 files changed, 160 insertions(+), 17 deletions(-) create mode 100644 tests/dotquery/__init__.py create mode 100644 tests/dotquery/models.py create mode 100644 tests/dotquery/tests.py create mode 100644 tests/dotquery/utils.py diff --git a/django_mongodb_engine/contrib/__init__.py b/django_mongodb_engine/contrib/__init__.py index 24dabce4..67b0dd65 100644 --- a/django_mongodb_engine/contrib/__init__.py +++ b/django_mongodb_engine/contrib/__init__.py @@ -101,24 +101,33 @@ def _filter_or_exclude(self, negate, *args, **kwargs): all_field_names = self.model._meta.get_all_field_names() base_field_names = [] - for f in all_field_names: - field = self.model._meta.get_field_by_name(f)[0] - if '.' not in f and field.get_internal_type() in MONGO_DOT_FIELDS: - base_field_names.append(f) - - for k, v in kwargs.items(): - if LOOKUP_SEP in k and k.split(LOOKUP_SEP)[0] in base_field_names: - del kwargs[k] - for s in ALL_OPERATORS: - if k.endswith(s): - k = re.sub(LOOKUP_SEP + s + '$', '#' + s, k) + for name in all_field_names: + field = self.model._meta.get_field_by_name(name)[0] + if '.' not in name and field.get_internal_type() in MONGO_DOT_FIELDS: + base_field_names.append(name) + + for key, val in kwargs.items(): + if LOOKUP_SEP in key and key.split(LOOKUP_SEP)[0] in base_field_names: + del kwargs[key] + for op in ALL_OPERATORS: + if key.endswith(op): + key = re.sub(LOOKUP_SEP + op + '$', '#' + op, key) break - k = k.replace(LOOKUP_SEP, '.').replace('#', LOOKUP_SEP) - kwargs[k] = v - f = k.split(LOOKUP_SEP)[0] - if '.' in f and f not in all_field_names: - field = AbstractIterableField(blank=True, null=True, editable=False) - field.contribute_to_class(self.model, f) + key = key.replace(LOOKUP_SEP, '.').replace('#', LOOKUP_SEP) + kwargs[key] = val + name = key.split(LOOKUP_SEP)[0] + if '.' in name and name not in all_field_names: + parts = name.split('.') + column = self.model._meta.get_field_by_name(parts[0])[0].db_column + if column: + parts[0] = column + field = AbstractIterableField( + db_column = '.'.join(parts), + blank=True, + null=True, + editable=False, + ) + field.contribute_to_class(self.model, name) if negate: clone.query.add_q(~Q(*args, **kwargs)) diff --git a/tests/dotquery/__init__.py b/tests/dotquery/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/dotquery/models.py b/tests/dotquery/models.py new file mode 100644 index 00000000..980a2f31 --- /dev/null +++ b/tests/dotquery/models.py @@ -0,0 +1,17 @@ +from django.db import models +from djangotoolbox.fields import ListField, DictField, EmbeddedModelField +from django_mongodb_engine.contrib import MongoDBManager + + +class DotQueryEmbeddedModel(models.Model): + f_int = models.IntegerField() + + +class DotQueryTestModel(models.Model): + objects = MongoDBManager() + + f_id = models.IntegerField() + f_dict = DictField(db_column='test_dict') + f_list = ListField() + f_embedded = EmbeddedModelField(DotQueryEmbeddedModel) + f_embedded_list = ListField(EmbeddedModelField(DotQueryEmbeddedModel)) diff --git a/tests/dotquery/tests.py b/tests/dotquery/tests.py new file mode 100644 index 00000000..5f4d72de --- /dev/null +++ b/tests/dotquery/tests.py @@ -0,0 +1,81 @@ +from __future__ import with_statement +from models import * +from utils import * + + +class DotQueryTests(TestCase): + """Tests for querying on foo.bar using join syntax.""" + + def setUp(self): + DotQueryTestModel.objects.create( + f_id=51, + f_dict={'numbers': [1, 2, 3], 'letters': 'abc'}, + f_list=[{'color': 'red'}, {'color': 'blue'}], + f_embedded=DotQueryEmbeddedModel(f_int=10), + f_embedded_list=[ + DotQueryEmbeddedModel(f_int=100), + DotQueryEmbeddedModel(f_int=101), + ], + ) + DotQueryTestModel.objects.create( + f_id=52, + f_dict={'numbers': [2, 3], 'letters': 'bc'}, + f_list=[{'color': 'red'}, {'color': 'green'}], + f_embedded=DotQueryEmbeddedModel(f_int=11), + f_embedded_list=[ + DotQueryEmbeddedModel(f_int=110), + DotQueryEmbeddedModel(f_int=111), + ], + ) + DotQueryTestModel.objects.create( + f_id=53, + f_dict={'numbers': [3, 4], 'letters': 'cd'}, + f_list=[{'color': 'yellow'}, {'color': 'orange'}], + f_embedded=DotQueryEmbeddedModel(f_int=12), + f_embedded_list=[ + DotQueryEmbeddedModel(f_int=120), + DotQueryEmbeddedModel(f_int=121), + ], + ) + + def tearDown(self): + DotQueryTestModel.objects.all().delete() + + def test_dict_queries(self): + q = DotQueryTestModel.objects.filter(f_dict__numbers=2) + self.assertEqual(q.count(), 2) + q = DotQueryTestModel.objects.filter(f_dict__letters__contains='b') + self.assertEqual(q.count(), 2) + q = DotQueryTestModel.objects.exclude(f_dict__letters__contains='b') + self.assertEqual(q.count(), 1) + self.assertEqual(q[0].f_id, 53) + + def test_list_queries(self): + q = DotQueryTestModel.objects.filter(f_list__color='red') + q = q.exclude(f_list__color='green') + q = q.exclude(f_list__color='purple') + self.assertEqual(q.count(), 1) + self.assertEqual(q[0].f_id, 51) + + def test_embedded_queries(self): + q = DotQueryTestModel.objects.exclude(f_embedded__f_int__in=[10, 12]) + self.assertEqual(q.count(), 1) + self.assertEqual(q[0].f_id, 52) + + def test_embedded_list_queries(self): + q = DotQueryTestModel.objects.get(f_embedded_list__f_int=120) + self.assertEqual(q.f_id, 53) + + def test_save_after_query(self): + q = DotQueryTestModel.objects.get(f_dict__letters='cd') + self.assertEqual(q.f_id, 53) + q.f_id = 1053 + q.clean() + q.save() + q = DotQueryTestModel.objects.get(f_dict__letters='cd') + self.assertEqual(q.f_id, 1053) + q.f_id = 53 + q.clean() + q.save() + q = DotQueryTestModel.objects.get(f_dict__letters='cd') + self.assertEqual(q.f_id, 53) diff --git a/tests/dotquery/utils.py b/tests/dotquery/utils.py new file mode 100644 index 00000000..fada71c8 --- /dev/null +++ b/tests/dotquery/utils.py @@ -0,0 +1,35 @@ +from django.conf import settings +from django.db import connections +from django.db.models import Model +from django.test import TestCase +from django.utils.unittest import skip + + +class TestCase(TestCase): + + def setUp(self): + super(TestCase, self).setUp() + if getattr(settings, 'TEST_DEBUG', False): + settings.DEBUG = True + + def assertEqualLists(self, a, b): + self.assertEqual(list(a), list(b)) + + +def skip_all_except(*tests): + + class meta(type): + + def __new__(cls, name, bases, dict): + for attr in dict.keys(): + if attr.startswith('test_') and attr not in tests: + del dict[attr] + return type.__new__(cls, name, bases, dict) + + return meta + + +def get_collection(model_or_name): + if isinstance(model_or_name, type) and issubclass(model_or_name, Model): + model_or_name = model_or_name._meta.db_table + return connections['default'].get_collection(model_or_name) diff --git a/tests/settings/__init__.py b/tests/settings/__init__.py index eb9f2579..b3e4c323 100644 --- a/tests/settings/__init__.py +++ b/tests/settings/__init__.py @@ -17,4 +17,5 @@ 'aggregations', 'contrib', 'storage', + 'dotquery', ]