p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #25367 -- Allowed boolean expressions in QuerySet.filter() and exclude(). 

This allows using expressions that have an output_field that is a
BooleanField to be used directly in a queryset filters, or in the
When() clauses of a Case() expression.

Thanks Josh Smeaton, Tim Graham, Simon Charette, Mariusz Felisiak, and
Adam Johnson for reviews.

Co-Authored-By: NyanKiyoshi <hello@vanille.bid>
This commit is contained in:
Matthew Schinckel 2017-02-27 19:31:52 +10:30 committed by Mariusz Felisiak
parent 069bee7c12
commit 4137fc2efc
10 changed files with 184 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
django/db/backends/base/operations.py
View File

@ -581,6 +581,13 @@ class BaseDatabaseOperations:
"""
pass
def conditional_expression_supported_in_where_clause(self, expression):
"""
Return True, if the conditional expression is supported in the WHERE
clause.
"""
return True
def combine_expression(self, connector, sub_expressions):
"""
Combine a list of subexpressions into a single expression, using

13
django/db/backends/oracle/operations.py
View File

@ -6,6 +6,8 @@ from functools import lru_cache
from django.conf import settings
from django.db.backends.base.operations import BaseDatabaseOperations
from django.db.backends.utils import strip_quotes, truncate_name
from django.db.models.expressions import Exists, ExpressionWrapper
from django.db.models.query_utils import Q
from django.db.utils import DatabaseError
from django.utils import timezone
from django.utils.encoding import force_bytes, force_str
@ -607,3 +609,14 @@ END;
if fields:
return self.connection.features.max_query_params // len(fields)
return len(objs)
def conditional_expression_supported_in_where_clause(self, expression):
"""
Oracle supports only EXISTS(...) or filters in the WHERE clause, others
must be compared with True.
"""
if isinstance(expression, Exists):
return True
if isinstance(expression, ExpressionWrapper) and isinstance(expression.expression, Q):
return True
return False

16
django/db/models/expressions.py
View File

@ -90,6 +90,8 @@ class Combinable:
return self._combine(other, self.POW, False)
def __and__(self, other):
if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
return Q(self) & Q(other)
raise NotImplementedError(
"Use .bitand() and .bitor() for bitwise logical operations."
)
@ -104,6 +106,8 @@ class Combinable:
return self._combine(other, self.BITRIGHTSHIFT, False)
def __or__(self, other):
if getattr(self, 'conditional', False) and getattr(other, 'conditional', False):
return Q(self) | Q(other)
raise NotImplementedError(
"Use .bitand() and .bitor() for bitwise logical operations."
)
@ -245,6 +249,10 @@ class BaseExpression:
])
return c
@property
def conditional(self):
return isinstance(self.output_field, fields.BooleanField)
@property
def field(self):
return self.output_field
@ -873,12 +881,17 @@ class ExpressionWrapper(Expression):
class When(Expression):
template = 'WHEN %(condition)s THEN %(result)s'
# This isn't a complete conditional expression, must be used in Case().
conditional = False
def __init__(self, condition=None, then=None, **lookups):
if lookups and condition is None:
condition, lookups = Q(**lookups), None
if condition is None or not getattr(condition, 'conditional', False) or lookups:
raise TypeError("__init__() takes either a Q object or lookups as keyword arguments")
raise TypeError(
'When() supports a Q object, a boolean expression, or lookups '
'as a condition.'
)
if isinstance(condition, Q) and not condition:
raise ValueError("An empty Q() can't be used as a When() condition.")
super().__init__(output_field=None)
@ -1090,6 +1103,7 @@ class Exists(Subquery):
class OrderBy(BaseExpression):
template = '%(expression)s %(ordering)s'
conditional = False
def __init__(self, expression, descending=False, nulls_first=False, nulls_last=False):
if nulls_first and nulls_last:

10
django/db/models/sql/query.py
View File

@ -1229,6 +1229,16 @@ class Query(BaseExpression):
"""
if isinstance(filter_expr, dict):
raise FieldError("Cannot parse keyword query as dict")
if hasattr(filter_expr, 'resolve_expression') and getattr(filter_expr, 'conditional', False):
if connections[DEFAULT_DB_ALIAS].ops.conditional_expression_supported_in_where_clause(filter_expr):
condition = filter_expr.resolve_expression(self)
else:
# Expression is not supported in the WHERE clause, add
# comparison with True.
condition = self.build_lookup(['exact'], filter_expr.resolve_expression(self), True)
clause = self.where_class()
clause.add(condition, AND)
return clause, []
arg, value = filter_expr
if not arg:
raise FieldError("Cannot parse keyword query %r" % arg)

50
docs/ref/models/conditional-expressions.txt
View File

@ -42,9 +42,15 @@ We'll be using the following model in the subsequent examples::
A ``When()`` object is used to encapsulate a condition and its result for use
in the conditional expression. Using a ``When()`` object is similar to using
the :meth:`~django.db.models.query.QuerySet.filter` method. The condition can
be specified using :ref:`field lookups <field-lookups>` or
:class:`~django.db.models.Q` objects. The result is provided using the ``then``
keyword.
be specified using :ref:`field lookups <field-lookups>`,
:class:`~django.db.models.Q` objects, or :class:`~django.db.models.Expression`
objects that have an ``output_field`` that is a
:class:`~django.db.models.BooleanField`. The result is provided using the
``then`` keyword.
.. versionchanged:: 3.0
Support for boolean :class:`~django.db.models.Expression` was added.
Some examples::
@ -60,6 +66,12 @@ Some examples::
>>> # Complex conditions can be created using Q objects
>>> When(Q(name__startswith="John") | Q(name__startswith="Paul"),
... then='name')
>>> # Condition can be created using boolean expressions.
>>> from django.db.models import Exists, OuterRef
>>> non_unique_account_type = Client.objects.filter(
... account_type=OuterRef('account_type'),
... ).exclude(pk=OuterRef('pk')).values('pk')
>>> When(Exists(non_unique_account_type), then=Value('non unique'))
Keep in mind that each of these values can be an expression.
@ -158,9 +170,9 @@ registered more than a year ago::
Advanced queries
================
Conditional expressions can be used in annotations, aggregations, lookups, and
updates. They can also be combined and nested with other expressions. This
allows you to make powerful conditional queries.
Conditional expressions can be used in annotations, aggregations, filters,
lookups, and updates. They can also be combined and nested with other
expressions. This allows you to make powerful conditional queries.
Conditional update
------------------
@ -236,3 +248,29 @@ On other databases, this is emulated using a ``CASE`` statement:
The two SQL statements are functionally equivalent but the more explicit
``FILTER`` may perform better.
Conditional filter
------------------
.. versionadded:: 3.0
When a conditional expression returns a boolean value, it is possible to use it
directly in filters. This means that it will not be added to the ``SELECT``
columns, but you can still use it to filter results::
>>> non_unique_account_type = Client.objects.filter(
... account_type=OuterRef('account_type'),
... ).exclude(pk=OuterRef('pk')).values('pk')
>>> Client.objects.filter(~Exists(non_unique_account_type))
In SQL terms, that evaluates to:
.. code-block:: sql
SELECT ...
FROM client c0
WHERE NOT EXISTS (
SELECT c1.id
FROM client c1
WHERE c1.account_type = c0.account_type AND NOT c1.id = c0.id
)

38
docs/ref/models/expressions.txt
View File

@ -5,10 +5,11 @@ Query Expressions
.. currentmodule:: django.db.models
Query expressions describe a value or a computation that can be used as part of
an update, create, filter, order by, annotation, or aggregate. There are a
number of built-in expressions (documented below) that can be used to help you
write queries. Expressions can be combined, or in some cases nested, to form
more complex computations.
an update, create, filter, order by, annotation, or aggregate. When an
expression outputs a boolean value, it may be used directly in filters. There
are a number of built-in expressions (documented below) that can be used to
help you write queries. Expressions can be combined, or in some cases nested,
to form more complex computations.
Supported arithmetic
====================
@ -69,6 +70,12 @@ Some examples
CharField.register_lookup(Length)
Company.objects.order_by('name__length')
# Boolean expression can be used directly in filters.
from django.db.models import Exists
Company.objects.filter(
Exists(Employee.objects.filter(company=OuterRef('pk'), salary__gt=10))
)
Built-in Expressions
====================
@ -626,22 +633,25 @@ degrade performance, it's automatically removed.
You can query using ``NOT EXISTS`` with ``~Exists()``.
Filtering on a ``Subquery`` expression
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Filtering on a ``Subquery()`` or ``Exists()`` expressions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
It's not possible to filter directly using ``Subquery`` and ``Exists``, e.g.::
``Subquery()`` that returns a boolean value and ``Exists()`` may be used as a
``condition`` in :class:`~django.db.models.expressions.When` expressions, or to
directly filter a queryset::
>>> recent_comments = Comment.objects.filter(...) # From above
>>> Post.objects.filter(Exists(recent_comments))
...
TypeError: 'Exists' object is not iterable
This will ensure that the subquery will not be added to the ``SELECT`` columns,
which may result in a better performance.
You must filter on a subquery expression by first annotating the queryset
and then filtering based on that annotation::
.. versionchanged:: 3.0
>>> Post.objects.annotate(
... recent_comment=Exists(recent_comments),
... ).filter(recent_comment=True)
In previous versions of Django, it was necessary to first annotate and then
filter against the annotation. This resulted in the annotated value always
being present in the query result, and often resulted in a query that took
more time to execute.
Using aggregates within a ``Subquery`` expression
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

7
docs/releases/3.0.txt
View File

@ -74,6 +74,13 @@ enable adding exclusion constraints on PostgreSQL. Constraints are added to
models using the
:attr:`Meta.constraints <django.db.models.Options.constraints>` option.
Filter expressions
------------------
Expressions that outputs :class:`~django.db.models.BooleanField` may now be
used directly in ``QuerySet`` filters, without having to first annotate and
then filter against the annotation.
Minor features
--------------

1
tests/expressions/models.py
View File

@ -34,6 +34,7 @@ class Company(models.Model):
related_name='company_point_of_contact_set',
null=True,
)
based_in_eu = models.BooleanField(default=False)
def __str__(self):
return self.name

60
tests/expressions/tests.py
View File

@ -37,7 +37,7 @@ class BasicExpressionsTests(TestCase):
ceo=Employee.objects.create(firstname="Joe", lastname="Smith", salary=10)
)
cls.foobar_ltd = Company.objects.create(
name="Foobar Ltd.", num_employees=3, num_chairs=4,
name="Foobar Ltd.", num_employees=3, num_chairs=4, based_in_eu=True,
ceo=Employee.objects.create(firstname="Frank", lastname="Meyer", salary=20)
)
cls.max = Employee.objects.create(firstname='Max', lastname='Mustermann', salary=30)
@ -83,6 +83,14 @@ class BasicExpressionsTests(TestCase):
2,
)
def test_filtering_on_q_that_is_boolean(self):
self.assertEqual(
Company.objects.filter(
ExpressionWrapper(Q(num_employees__gt=3), output_field=models.BooleanField())
).count(),
2,
)
def test_filter_inter_attribute(self):
# We can filter on attribute relationships on same model obj, e.g.
# find companies where the number of employees is greater
@ -642,6 +650,56 @@ class BasicExpressionsTests(TestCase):
with self.assertRaisesMessage(FieldError, "Cannot resolve keyword 'nope' into field."):
list(Company.objects.filter(ceo__pk=F('point_of_contact__nope')))
def test_exists_in_filter(self):
inner = Company.objects.filter(ceo=OuterRef('pk')).values('pk')
qs1 = Employee.objects.filter(Exists(inner))
qs2 = Employee.objects.annotate(found=Exists(inner)).filter(found=True)
self.assertCountEqual(qs1, qs2)
self.assertFalse(Employee.objects.exclude(Exists(inner)).exists())
self.assertCountEqual(qs2, Employee.objects.exclude(~Exists(inner)))
def test_subquery_in_filter(self):
inner = Company.objects.filter(ceo=OuterRef('pk')).values('based_in_eu')
self.assertSequenceEqual(
Employee.objects.filter(Subquery(inner)),
[self.foobar_ltd.ceo],
)
def test_case_in_filter_if_boolean_output_field(self):
is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
qs = Employee.objects.filter(
Case(
When(Exists(is_ceo), then=True),
When(Exists(is_poc), then=True),
default=False,
output_field=models.BooleanField(),
),
)
self.assertSequenceEqual(qs, [self.example_inc.ceo, self.foobar_ltd.ceo, self.max])
def test_boolean_expression_combined(self):
is_ceo = Company.objects.filter(ceo=OuterRef('pk'))
is_poc = Company.objects.filter(point_of_contact=OuterRef('pk'))
self.gmbh.point_of_contact = self.max
self.gmbh.save()
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) | Exists(is_poc)),
[self.example_inc.ceo, self.foobar_ltd.ceo, self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) & Exists(is_poc)),
[self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_ceo) & Q(salary__gte=30)),
[self.max],
)
self.assertSequenceEqual(
Employee.objects.filter(Exists(is_poc) | Q(salary__lt=15)),
[self.example_inc.ceo, self.max],
)
class IterableLookupInnerExpressionsTests(TestCase):
@classmethod

5
tests/expressions_case/tests.py
View File

@ -1327,7 +1327,10 @@ class CaseWhenTests(SimpleTestCase):
Case(When(Q(pk__in=[])), object())
def test_invalid_when_constructor_args(self):
msg = '__init__() takes either a Q object or lookups as keyword arguments'
msg = (
'When() supports a Q object, a boolean expression, or lookups as '
'a condition.'
)
with self.assertRaisesMessage(TypeError, msg):
When(condition=object())
with self.assertRaisesMessage(TypeError, msg):