From 4137fc2efce2dde48340728b8006fc6d66b9e3a5 Mon Sep 17 00:00:00 2001 From: Matthew Schinckel Date: Mon, 27 Feb 2017 19:31:52 +1030 Subject: [PATCH] Fixed #25367 -- Allowed boolean expressions in QuerySet.filter() and exclude(). This allows using expressions that have an output_field that is a BooleanField to be used directly in a queryset filters, or in the When() clauses of a Case() expression. Thanks Josh Smeaton, Tim Graham, Simon Charette, Mariusz Felisiak, and Adam Johnson for reviews. Co-Authored-By: NyanKiyoshi --- django/db/backends/base/operations.py | 7 +++ django/db/backends/oracle/operations.py | 13 +++++ django/db/models/expressions.py | 16 +++++- django/db/models/sql/query.py | 10 ++++ docs/ref/models/conditional-expressions.txt | 50 ++++++++++++++--- docs/ref/models/expressions.txt | 38 ++++++++----- docs/releases/3.0.txt | 7 +++ tests/expressions/models.py | 1 + tests/expressions/tests.py | 60 ++++++++++++++++++++- tests/expressions_case/tests.py | 5 +- 10 files changed, 184 insertions(+), 23 deletions(-) diff --git a/django/db/backends/base/operations.py b/django/db/backends/base/operations.py index 76abc6dcb28..a0c84a8ff40 100644 --- a/django/db/backends/base/operations.py +++ b/django/db/backends/base/operations.py @@ -581,6 +581,13 @@ class BaseDatabaseOperations: """ pass + def conditional_expression_supported_in_where_clause(self, expression): + """ + Return True, if the conditional expression is supported in the WHERE + clause. + """ + return True + def combine_expression(self, connector, sub_expressions): """ Combine a list of subexpressions into a single expression, using diff --git a/django/db/backends/oracle/operations.py b/django/db/backends/oracle/operations.py index 3cad02fa412..95126d37bed 100644 --- a/django/db/backends/oracle/operations.py +++ b/django/db/backends/oracle/operations.py @@ -6,6 +6,8 @@ from functools import lru_cache from django.conf import settings from django.db.backends.base.operations import BaseDatabaseOperations from django.db.backends.utils import strip_quotes, truncate_name +from django.db.models.expressions import Exists, ExpressionWrapper +from django.db.models.query_utils import Q from django.db.utils import DatabaseError from django.utils import timezone from django.utils.encoding import force_bytes, force_str @@ -607,3 +609,14 @@ END; if fields: return self.connection.features.max_query_params // len(fields) return len(objs) + + def conditional_expression_supported_in_where_clause(self, expression): + """ + Oracle supports only EXISTS(...) or filters in the WHERE clause, others + must be compared with True. + """ + if isinstance(expression, Exists): + return True + if isinstance(expression, ExpressionWrapper) and isinstance(expression.expression, Q): + return True + return False diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 1dd061c1527..9429adf81c2 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -90,6 +90,8 @@ class Combinable: return self._combine(other, self.POW, False) def __and__(self, other): + if getattr(self, 'conditional', False) and getattr(other, 'conditional', False): + return Q(self) & Q(other) raise NotImplementedError( "Use .bitand() and .bitor() for bitwise logical operations." ) @@ -104,6 +106,8 @@ class Combinable: return self._combine(other, self.BITRIGHTSHIFT, False) def __or__(self, other): + if getattr(self, 'conditional', False) and getattr(other, 'conditional', False): + return Q(self) | Q(other) raise NotImplementedError( "Use .bitand() and .bitor() for bitwise logical operations." ) @@ -245,6 +249,10 @@ class BaseExpression: ]) return c + @property + def conditional(self): + return isinstance(self.output_field, fields.BooleanField) + @property def field(self): return self.output_field @@ -873,12 +881,17 @@ class ExpressionWrapper(Expression): class When(Expression): template = 'WHEN %(condition)s THEN %(result)s' + # This isn't a complete conditional expression, must be used in Case(). + conditional = False def __init__(self, condition=None, then=None, **lookups): if lookups and condition is None: condition, lookups = Q(**lookups), None if condition is None or not getattr(condition, 'conditional', False) or lookups: - raise TypeError("__init__() takes either a Q object or lookups as keyword arguments") + raise TypeError( + 'When() supports a Q object, a boolean expression, or lookups ' + 'as a condition.' + ) if isinstance(condition, Q) and not condition: raise ValueError("An empty Q() can't be used as a When() condition.") super().__init__(output_field=None) @@ -1090,6 +1103,7 @@ class Exists(Subquery): class OrderBy(BaseExpression): template = '%(expression)s %(ordering)s' + conditional = False def __init__(self, expression, descending=False, nulls_first=False, nulls_last=False): if nulls_first and nulls_last: diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 4ad1fb3f363..35f3c5e1ea5 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -1229,6 +1229,16 @@ class Query(BaseExpression): """ if isinstance(filter_expr, dict): raise FieldError("Cannot parse keyword query as dict") + if hasattr(filter_expr, 'resolve_expression') and getattr(filter_expr, 'conditional', False): + if connections[DEFAULT_DB_ALIAS].ops.conditional_expression_supported_in_where_clause(filter_expr): + condition = filter_expr.resolve_expression(self) + else: + # Expression is not supported in the WHERE clause, add + # comparison with True. + condition = self.build_lookup(['exact'], filter_expr.resolve_expression(self), True) + clause = self.where_class() + clause.add(condition, AND) + return clause, [] arg, value = filter_expr if not arg: raise FieldError("Cannot parse keyword query %r" % arg) diff --git a/docs/ref/models/conditional-expressions.txt b/docs/ref/models/conditional-expressions.txt index f9e681f667a..e88554dec65 100644 --- a/docs/ref/models/conditional-expressions.txt +++ b/docs/ref/models/conditional-expressions.txt @@ -42,9 +42,15 @@ We'll be using the following model in the subsequent examples:: A ``When()`` object is used to encapsulate a condition and its result for use in the conditional expression. Using a ``When()`` object is similar to using the :meth:`~django.db.models.query.QuerySet.filter` method. The condition can -be specified using :ref:`field lookups ` or -:class:`~django.db.models.Q` objects. The result is provided using the ``then`` -keyword. +be specified using :ref:`field lookups `, +:class:`~django.db.models.Q` objects, or :class:`~django.db.models.Expression` +objects that have an ``output_field`` that is a +:class:`~django.db.models.BooleanField`. The result is provided using the +``then`` keyword. + +.. versionchanged:: 3.0 + + Support for boolean :class:`~django.db.models.Expression` was added. Some examples:: @@ -60,6 +66,12 @@ Some examples:: >>> # Complex conditions can be created using Q objects >>> When(Q(name__startswith="John") | Q(name__startswith="Paul"), ... then='name') + >>> # Condition can be created using boolean expressions. + >>> from django.db.models import Exists, OuterRef + >>> non_unique_account_type = Client.objects.filter( + ... account_type=OuterRef('account_type'), + ... ).exclude(pk=OuterRef('pk')).values('pk') + >>> When(Exists(non_unique_account_type), then=Value('non unique')) Keep in mind that each of these values can be an expression. @@ -158,9 +170,9 @@ registered more than a year ago:: Advanced queries ================ -Conditional expressions can be used in annotations, aggregations, lookups, and -updates. They can also be combined and nested with other expressions. This -allows you to make powerful conditional queries. +Conditional expressions can be used in annotations, aggregations, filters, +lookups, and updates. They can also be combined and nested with other +expressions. This allows you to make powerful conditional queries. Conditional update ------------------ @@ -236,3 +248,29 @@ On other databases, this is emulated using a ``CASE`` statement: The two SQL statements are functionally equivalent but the more explicit ``FILTER`` may perform better. + +Conditional filter +------------------ + +.. versionadded:: 3.0 + +When a conditional expression returns a boolean value, it is possible to use it +directly in filters. This means that it will not be added to the ``SELECT`` +columns, but you can still use it to filter results:: + + >>> non_unique_account_type = Client.objects.filter( + ... account_type=OuterRef('account_type'), + ... ).exclude(pk=OuterRef('pk')).values('pk') + >>> Client.objects.filter(~Exists(non_unique_account_type)) + +In SQL terms, that evaluates to: + +.. code-block:: sql + + SELECT ... + FROM client c0 + WHERE NOT EXISTS ( + SELECT c1.id + FROM client c1 + WHERE c1.account_type = c0.account_type AND NOT c1.id = c0.id + ) diff --git a/docs/ref/models/expressions.txt b/docs/ref/models/expressions.txt index ccd75d62a96..16dabf8177b 100644 --- a/docs/ref/models/expressions.txt +++ b/docs/ref/models/expressions.txt @@ -5,10 +5,11 @@ Query Expressions .. currentmodule:: django.db.models Query expressions describe a value or a computation that can be used as part of -an update, create, filter, order by, annotation, or aggregate. There are a -number of built-in expressions (documented below) that can be used to help you -write queries. Expressions can be combined, or in some cases nested, to form -more complex computations. +an update, create, filter, order by, annotation, or aggregate. When an +expression outputs a boolean value, it may be used directly in filters. There +are a number of built-in expressions (documented below) that can be used to +help you write queries. Expressions can be combined, or in some cases nested, +to form more complex computations. Supported arithmetic ==================== @@ -69,6 +70,12 @@ Some examples CharField.register_lookup(Length) Company.objects.order_by('name__length') + # Boolean expression can be used directly in filters. + from django.db.models import Exists + Company.objects.filter( + Exists(Employee.objects.filter(company=OuterRef('pk'), salary__gt=10)) + ) + Built-in Expressions ==================== @@ -626,22 +633,25 @@ degrade performance, it's automatically removed. You can query using ``NOT EXISTS`` with ``~Exists()``. -Filtering on a ``Subquery`` expression -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Filtering on a ``Subquery()`` or ``Exists()`` expressions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -It's not possible to filter directly using ``Subquery`` and ``Exists``, e.g.:: +``Subquery()`` that returns a boolean value and ``Exists()`` may be used as a +``condition`` in :class:`~django.db.models.expressions.When` expressions, or to +directly filter a queryset:: + >>> recent_comments = Comment.objects.filter(...) # From above >>> Post.objects.filter(Exists(recent_comments)) - ... - TypeError: 'Exists' object is not iterable +This will ensure that the subquery will not be added to the ``SELECT`` columns, +which may result in a better performance. -You must filter on a subquery expression by first annotating the queryset -and then filtering based on that annotation:: +.. versionchanged:: 3.0 - >>> Post.objects.annotate( - ... recent_comment=Exists(recent_comments), - ... ).filter(recent_comment=True) + In previous versions of Django, it was necessary to first annotate and then + filter against the annotation. This resulted in the annotated value always + being present in the query result, and often resulted in a query that took + more time to execute. Using aggregates within a ``Subquery`` expression ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index 722baeeeaa1..f6ec8f8cc75 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -74,6 +74,13 @@ enable adding exclusion constraints on PostgreSQL. Constraints are added to models using the :attr:`Meta.constraints ` option. +Filter expressions +------------------ + +Expressions that outputs :class:`~django.db.models.BooleanField` may now be +used directly in ``QuerySet`` filters, without having to first annotate and +then filter against the annotation. + Minor features -------------- diff --git a/tests/expressions/models.py b/tests/expressions/models.py index 33f7850ac16..a81ba9e16b5 100644 --- a/tests/expressions/models.py +++ b/tests/expressions/models.py @@ -34,6 +34,7 @@ class Company(models.Model): related_name='company_point_of_contact_set', null=True, ) + based_in_eu = models.BooleanField(default=False) def __str__(self): return self.name diff --git a/tests/expressions/tests.py b/tests/expressions/tests.py index eaa34711776..f50c634014d 100644 --- a/tests/expressions/tests.py +++ b/tests/expressions/tests.py @@ -37,7 +37,7 @@ class BasicExpressionsTests(TestCase): ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10) ) cls.foobar_ltd = Company.objects.create( - name="Foobar Ltd.", num_employees=3, num_chairs=4, + name="Foobar Ltd.", num_employees=3, num_chairs=4, based_in_eu=True, ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20) ) cls.max = Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30) @@ -83,6 +83,14 @@ class BasicExpressionsTests(TestCase): 2, ) + def test_filtering_on_q_that_is_boolean(self): + self.assertEqual( + Company.objects.filter( + ExpressionWrapper(Q(num_employees__gt=3), output_field=models.BooleanField()) + ).count(), + 2, + ) + def test_filter_inter_attribute(self): # We can filter on attribute relationships on same model obj, e.g. # find companies where the number of employees is greater @@ -642,6 +650,56 @@ class BasicExpressionsTests(TestCase): with self.assertRaisesMessage(FieldError, "Cannot resolve keyword 'nope' into field."): list(Company.objects.filter(ceo__pk=F('point_of_contact__nope'))) + def test_exists_in_filter(self): + inner = Company.objects.filter(ceo=OuterRef('pk')).values('pk') + qs1 = Employee.objects.filter(Exists(inner)) + qs2 = Employee.objects.annotate(found=Exists(inner)).filter(found=True) + self.assertCountEqual(qs1, qs2) + self.assertFalse(Employee.objects.exclude(Exists(inner)).exists()) + self.assertCountEqual(qs2, Employee.objects.exclude(~Exists(inner))) + + def test_subquery_in_filter(self): + inner = Company.objects.filter(ceo=OuterRef('pk')).values('based_in_eu') + self.assertSequenceEqual( + Employee.objects.filter(Subquery(inner)), + [self.foobar_ltd.ceo], + ) + + def test_case_in_filter_if_boolean_output_field(self): + is_ceo = Company.objects.filter(ceo=OuterRef('pk')) + is_poc = Company.objects.filter(point_of_contact=OuterRef('pk')) + qs = Employee.objects.filter( + Case( + When(Exists(is_ceo), then=True), + When(Exists(is_poc), then=True), + default=False, + output_field=models.BooleanField(), + ), + ) + self.assertSequenceEqual(qs, [self.example_inc.ceo, self.foobar_ltd.ceo, self.max]) + + def test_boolean_expression_combined(self): + is_ceo = Company.objects.filter(ceo=OuterRef('pk')) + is_poc = Company.objects.filter(point_of_contact=OuterRef('pk')) + self.gmbh.point_of_contact = self.max + self.gmbh.save() + self.assertSequenceEqual( + Employee.objects.filter(Exists(is_ceo) | Exists(is_poc)), + [self.example_inc.ceo, self.foobar_ltd.ceo, self.max], + ) + self.assertSequenceEqual( + Employee.objects.filter(Exists(is_ceo) & Exists(is_poc)), + [self.max], + ) + self.assertSequenceEqual( + Employee.objects.filter(Exists(is_ceo) & Q(salary__gte=30)), + [self.max], + ) + self.assertSequenceEqual( + Employee.objects.filter(Exists(is_poc) | Q(salary__lt=15)), + [self.example_inc.ceo, self.max], + ) + class IterableLookupInnerExpressionsTests(TestCase): @classmethod diff --git a/tests/expressions_case/tests.py b/tests/expressions_case/tests.py index f1255cb8e9a..e3f4775cf26 100644 --- a/tests/expressions_case/tests.py +++ b/tests/expressions_case/tests.py @@ -1327,7 +1327,10 @@ class CaseWhenTests(SimpleTestCase): Case(When(Q(pk__in=[])), object()) def test_invalid_when_constructor_args(self): - msg = '__init__() takes either a Q object or lookups as keyword arguments' + msg = ( + 'When() supports a Q object, a boolean expression, or lookups as ' + 'a condition.' + ) with self.assertRaisesMessage(TypeError, msg): When(condition=object()) with self.assertRaisesMessage(TypeError, msg):