From dde6963869ba4a72b100daedd66470e080f99b75 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 9 Jan 2006 11:01:38 +0000 Subject: [PATCH] magic-removal: Fixed #1133 -- Added ability to use Q objects as args in DB lookup queries. git-svn-id: http://code.djangoproject.com/svn/django/branches/magic-removal@1884 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/db/models/manager.py | 70 ++++++++++++++++++-------- django/db/models/query.py | 2 + docs/db-api.txt | 71 ++++++++++++++++++++------- tests/modeltests/lookup/models.py | 2 +- tests/modeltests/or_lookups/models.py | 31 +++++++++++- 5 files changed, 135 insertions(+), 41 deletions(-) diff --git a/django/db/models/manager.py b/django/db/models/manager.py index 29eb371db8..1f1f301f94 100644 --- a/django/db/models/manager.py +++ b/django/db/models/manager.py @@ -5,6 +5,7 @@ from django.db.models.query import Q, parse_lookup, fill_table_cache, get_cached from django.db.models.query import handle_legacy_orderlist, orderlist2sql, orderfield2column from django.dispatch import dispatcher from django.db.models import signals +from django.utils.datastructures import SortedDict # Size of each "chunk" for get_iterator calls. # Larger values are slightly faster at the expense of more storage space. @@ -47,7 +48,7 @@ class Manager(object): self.creation_counter < klass._default_manager.creation_counter: klass._default_manager = self - def _get_sql_clause(self, **kwargs): + def _get_sql_clause(self, *args, **kwargs): def quote_only_if_word(word): if ' ' in word: return word @@ -59,12 +60,28 @@ class Manager(object): # Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z. select = ["%s.%s" % (backend.quote_name(opts.db_table), backend.quote_name(f.column)) for f in opts.fields] tables = (kwargs.get('tables') and [quote_only_if_word(t) for t in kwargs['tables']] or []) + joins = SortedDict() where = kwargs.get('where') and kwargs['where'][:] or [] params = kwargs.get('params') and kwargs['params'][:] or [] + # Convert all the args into SQL. + table_count = 0 + for arg in args: + # check that the provided argument is a Query (i.e., it has a get_sql method) + if not hasattr(arg, 'get_sql'): + raise TypeError, "'%s' is not a valid query argument" % str(arg) + + tables2, joins2, where2, params2 = arg.get_sql(opts) + tables.extend(tables2) + joins.update(joins2) + where.extend(where2) + params.extend(params2) + + # Convert the kwargs into SQL. - tables2, joins, where2, params2 = parse_lookup(kwargs.items(), opts) + tables2, joins2, where2, params2 = parse_lookup(kwargs.items(), opts) tables.extend(tables2) + joins.update(joins2) where.extend(where2) params.extend(params2) @@ -129,13 +146,13 @@ class Manager(object): return select, " ".join(sql), params - def get_iterator(self, **kwargs): + def get_iterator(self, *args, **kwargs): # kwargs['select'] is a dictionary, and dictionaries' key order is # undefined, so we convert it to a list of tuples internally. kwargs['select'] = kwargs.get('select', {}).items() cursor = connection.cursor() - select, sql, params = self._get_sql_clause(**kwargs) + select, sql, params = self._get_sql_clause(*args, **kwargs) cursor.execute("SELECT " + (kwargs.get('distinct') and "DISTINCT " or "") + ",".join(select) + sql, params) fill_cache = kwargs.get('select_related') index_end = len(self.klass._meta.fields) @@ -152,35 +169,41 @@ class Manager(object): setattr(obj, k[0], row[index_end+i]) yield obj - def get_list(self, **kwargs): - return list(self.get_iterator(**kwargs)) + def get_list(self, *args, **kwargs): + return list(self.get_iterator(*args, **kwargs)) - def get_count(self, **kwargs): + def get_count(self, *args, **kwargs): kwargs['order_by'] = [] kwargs['offset'] = None kwargs['limit'] = None kwargs['select_related'] = False - _, sql, params = self._get_sql_clause(**kwargs) + _, sql, params = self._get_sql_clause(*args, **kwargs) cursor = connection.cursor() cursor.execute("SELECT COUNT(*)" + sql, params) return cursor.fetchone()[0] - def get_object(self, **kwargs): - obj_list = self.get_list(**kwargs) + def get_object(self, *args, **kwargs): + obj_list = self.get_list(*args, **kwargs) if len(obj_list) < 1: raise self.klass.DoesNotExist, "%s does not exist for %s" % (self.klass._meta.object_name, kwargs) assert len(obj_list) == 1, "get_object() returned more than one %s -- it returned %s! Lookup parameters were %s" % (self.klass._meta.object_name, len(obj_list), kwargs) return obj_list[0] def get_in_bulk(self, *args, **kwargs): - id_list = args and args[0] or kwargs.get('id_list', []) - assert id_list != [], "get_in_bulk() cannot be passed an empty list." + # Separate any list arguments: the first list will be used as the id list; subsequent + # lists will be ignored. + id_args = filter(lambda arg: isinstance(arg, list), args) + # Separate any non-list arguments: these are assumed to be query arguments + sql_args = filter(lambda arg: not isinstance(arg, list), args) + + id_list = id_args and id_args[0] or kwargs.get('id_list', []) + assert id_list != [], "get_in_bulk() cannot be passed an empty ID list." kwargs['where'] = ["%s.%s IN (%s)" % (backend.quote_name(self.klass._meta.db_table), backend.quote_name(self.klass._meta.pk.column), ",".join(['%s'] * len(id_list)))] kwargs['params'] = id_list - obj_list = self.get_list(**kwargs) + obj_list = self.get_list(*sql_args, **kwargs) return dict([(getattr(o, self.klass._meta.pk.attname), o) for o in obj_list]) - def get_values_iterator(self, **kwargs): + def get_values_iterator(self, *args, **kwargs): # select_related and select aren't supported in get_values(). kwargs['select_related'] = False kwargs['select'] = {} @@ -192,7 +215,7 @@ class Manager(object): fields = [f.column for f in self.klass._meta.fields] cursor = connection.cursor() - _, sql, params = self._get_sql_clause(**kwargs) + _, sql, params = self._get_sql_clause(*args, **kwargs) select = ['%s.%s' % (backend.quote_name(self.klass._meta.db_table), backend.quote_name(f)) for f in fields] cursor.execute("SELECT " + (kwargs.get('distinct') and "DISTINCT " or "") + ",".join(select) + sql, params) while 1: @@ -202,17 +225,22 @@ class Manager(object): for row in rows: yield dict(zip(fields, row)) - def get_values(self, **kwargs): - return list(self.get_values_iterator(**kwargs)) + def get_values(self, *args, **kwargs): + return list(self.get_values_iterator(*args, **kwargs)) - def __get_latest(self, **kwargs): + def __get_latest(self, *args, **kwargs): kwargs['order_by'] = ('-' + self.klass._meta.get_latest_by,) kwargs['limit'] = 1 - return self.get_object(**kwargs) + return self.get_object(*args, **kwargs) def __get_date_list(self, field, *args, **kwargs): + # Separate any string arguments: the first will be used as the kind + kind_args = filter(lambda arg: isinstance(arg, str), args) + # Separate any non-list arguments: these are assumed to be query arguments + sql_args = filter(lambda arg: not isinstance(arg, str), args) + from django.db.backends.util import typecast_timestamp - kind = args and args[0] or kwargs['kind'] + kind = kind_args and kind_args[0] or kwargs.get(['kind'],"") assert kind in ("month", "year", "day"), "'kind' must be one of 'year', 'month' or 'day'." order = 'ASC' if kwargs.has_key('order'): @@ -223,7 +251,7 @@ class Manager(object): if field.null: kwargs.setdefault('where', []).append('%s.%s IS NOT NULL' % \ (backend.quote_name(self.klass._meta.db_table), backend.quote_name(field.column))) - select, sql, params = self._get_sql_clause(**kwargs) + select, sql, params = self._get_sql_clause(*sql_args, **kwargs) sql = 'SELECT %s %s GROUP BY 1 ORDER BY 1 %s' % \ (backend.get_date_trunc_sql(kind, '%s.%s' % (backend.quote_name(self.klass._meta.db_table), backend.quote_name(field.column))), sql, order) diff --git a/django/db/models/query.py b/django/db/models/query.py index 8d52be3232..01fe4eda00 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -198,6 +198,8 @@ def parse_lookup(kwarg_items, opts): elif value is None: pass elif kwarg == 'complex': + if not hasattr(value, 'get_sql'): + raise TypeError, "'%s' is not a valid query argument" % str(arg) tables2, joins2, where2, params2 = value.get_sql(opts) tables.extend(tables2) joins.update(joins2) diff --git a/docs/db-api.txt b/docs/db-api.txt index d09d2a14e7..e6b2238225 100644 --- a/docs/db-api.txt +++ b/docs/db-api.txt @@ -224,32 +224,67 @@ OR lookups **New in Django development version.** -By default, multiple lookups are "AND"ed together. If you'd like to use ``OR`` -statements in your queries, use the ``complex`` lookup type. +By default, keyword argument queries are "AND"ed together. If you have more complex query +requirements (for example, you need to include an ``OR`` statement in your query), you need +to use ``Q`` objects. -``complex`` takes an expression of clauses, each of which is an instance of -``django.core.meta.Q``. ``Q`` takes an arbitrary number of keyword arguments in -the standard Django lookup format. And you can use Python's "and" (``&``) and -"or" (``|``) operators to combine ``Q`` instances. For example:: +A ``Q`` object is an instance of ``django.core.meta.Q``, used to encapsulate a collection of +keyword arguments. These keyword arguments are specified in the same way as keyword arguments to +the basic lookup functions like get_object() and get_list(). For example:: - from django.core.meta import Q - polls.get_object(complex=(Q(question__startswith='Who') | Q(question__startswith='What'))) + Q(question__startswith='What') -The ``|`` symbol signifies an "OR", so this (roughly) translates into:: +``Q`` objects can be combined using the ``&`` and ``|`` operators. When an operator is used on two +``Q`` objects, it yields a new ``Q`` object. For example the statement:: - SELECT * FROM polls - WHERE question LIKE 'Who%' OR question LIKE 'What%'; + Q(question__startswith='Who') | Q(question__startswith='What') -You can use ``&`` and ``|`` operators together, and use parenthetical grouping. -Example:: +... yields a single ``Q`` object that represents the "OR" of two "question__startswith" queries, equivalent to the SQL WHERE clause:: - polls.get_object(complex=(Q(question__startswith='Who') & (Q(pub_date__exact=date(2005, 5, 2)) | Q(pub_date__exact=date(2005, 5, 6)))) + ... WHERE question LIKE 'Who%' OR question LIKE 'What%' -This roughly translates into:: +You can compose statements of arbitrary complexity by combining ``Q`` objects with the ``&`` and ``|`` operators. Parenthetical grouping can also be used. - SELECT * FROM polls - WHERE question LIKE 'Who%' - AND (pub_date = '2005-05-02' OR pub_date = '2005-05-06'); +One or more ``Q`` objects can then provided as arguments to the lookup functions. If multiple +``Q`` object arguments are provided to a lookup function, they will be "AND"ed together. +For example:: + + polls.get_object( + Q(question__startswith='Who'), + Q(pub_date__exact=date(2005, 5, 2)) | Q(pub_date__exact=date(2005, 5, 6)) + ) + +... roughly translates into the SQL:: + + SELECT * from polls WHERE question LIKE 'Who%' + AND (pub_date = '2005-05-02' OR pub_date = '2005-05-06') + +If necessary, lookup functions can mix the use of ``Q`` objects and keyword arguments. All arguments +provided to a lookup function (be they keyword argument or ``Q`` object) are "AND"ed together. +However, if a ``Q`` object is provided, it must precede the definition of any keyword arguments. +For example:: + + polls.get_object( + Q(pub_date__exact=date(2005, 5, 2)) | Q(pub_date__exact=date(2005, 5, 6)), + question__startswith='Who') + +... would be a valid query, equivalent to the previous example; but:: + + # INVALID QUERY + polls.get_object( + question__startswith='Who', + Q(pub_date__exact=date(2005, 5, 2)) | Q(pub_date__exact=date(2005, 5, 6))) + +... would not be valid. + +A ``Q`` objects can also be provided to the ``complex`` keyword argument. For example:: + + polls.get_object( + complex=Q(question__startswith='Who') & + (Q(pub_date__exact=date(2005, 5, 2)) | + Q(pub_date__exact=date(2005, 5, 6)) + ) + ) See the `OR lookups examples page`_ for more examples. diff --git a/tests/modeltests/lookup/models.py b/tests/modeltests/lookup/models.py index 9d5c5af07d..dcc0b3a826 100644 --- a/tests/modeltests/lookup/models.py +++ b/tests/modeltests/lookup/models.py @@ -68,7 +68,7 @@ Article 4 >>> Article.objects.get_in_bulk([]) Traceback (most recent call last): ... -AssertionError: get_in_bulk() cannot be passed an empty list. +AssertionError: get_in_bulk() cannot be passed an empty ID list. # get_values() is just like get_list(), except it returns a list of # dictionaries instead of object instances -- and you can specify which fields diff --git a/tests/modeltests/or_lookups/models.py b/tests/modeltests/or_lookups/models.py index 0a5fb541be..42553d6b34 100644 --- a/tests/modeltests/or_lookups/models.py +++ b/tests/modeltests/or_lookups/models.py @@ -3,7 +3,7 @@ To perform an OR lookup, or a lookup that combines ANDs and ORs, use the ``complex`` keyword argument, and pass it an expression of clauses using the -variable ``django.db.models.Q``. +variable ``django.db.models.Q`` (or any object with a get_sql method). """ from django.db import models @@ -54,4 +54,33 @@ API_TESTS = """ >>> Article.objects.get_list(complex=(Q(pk=1) | Q(pk=2) | Q(pk=3))) [Hello, Goodbye, Hello and goodbye] +# Queries can use Q objects as args +>>> Article.objects.get_list(Q(headline__startswith='Hello')) +[Hello, Hello and goodbye] + +# Q arg objects are ANDed +>>> Article.objects.get_list(Q(headline__startswith='Hello'), Q(headline__contains='bye')) +[Hello and goodbye] + +# Q arg AND order is irrelevant +>>> Article.objects.get_list(Q(headline__contains='bye'), headline__startswith='Hello') +[Hello and goodbye] + +# QOrs are ok, as they ultimately resolve to a Q +>>> Article.objects.get_list(Q(headline__contains='Hello') | Q(headline__contains='bye')) +[Hello, Goodbye, Hello and goodbye] + +# Try some arg queries with operations other than get_list +>>> Article.objects.get_object(Q(headline__startswith='Hello'), Q(headline__contains='bye')) +Hello and goodbye + +>>> Article.objects.get_count(Q(headline__startswith='Hello') | Q(headline__contains='bye')) +3 + +>>> Article.objects.get_values(Q(headline__startswith='Hello'), Q(headline__contains='bye')) +[{'headline': 'Hello and goodbye', 'pub_date': datetime.datetime(2005, 11, 29, 0, 0), 'id': 3}] + +>>> Article.objects.get_in_bulk([1,2], Q(headline__startswith='Hello')) +{1: Hello} + """