from django.conf import settings from django.db import connection, transaction from django.db.models.fields import DateField, FieldDoesNotExist from django.db.models import signals, loading from django.dispatch import dispatcher from django.utils.datastructures import SortedDict from django.utils.encoding import smart_unicode from django.contrib.contenttypes import generic import datetime import operator import re try: set except NameError: from sets import Set as set # Python 2.3 fallback # The string constant used to separate query parts LOOKUP_SEPARATOR = '__' # The list of valid query types QUERY_TERMS = ( 'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in', 'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year', 'month', 'day', 'isnull', 'search', 'regex', 'iregex', ) # Size of each "chunk" for get_iterator calls. # Larger values are slightly faster at the expense of more storage space. GET_ITERATOR_CHUNK_SIZE = 100 class EmptyResultSet(Exception): pass #################### # HELPER FUNCTIONS # #################### # Django currently supports two forms of ordering. # Form 1 (deprecated) example: # order_by=(('pub_date', 'DESC'), ('headline', 'ASC'), (None, 'RANDOM')) # Form 2 (new-style) example: # order_by=('-pub_date', 'headline', '?') # Form 1 is deprecated and will no longer be supported for Django's first # official release. The following code converts from Form 1 to Form 2. LEGACY_ORDERING_MAPPING = {'ASC': '_', 'DESC': '-_', 'RANDOM': '?'} def handle_legacy_orderlist(order_list): if not order_list or isinstance(order_list[0], basestring): return order_list else: import warnings new_order_list = [LEGACY_ORDERING_MAPPING[j.upper()].replace('_', smart_unicode(i)) for i, j in order_list] warnings.warn("%r ordering syntax is deprecated. Use %r instead." % (order_list, new_order_list), DeprecationWarning) return new_order_list def orderfield2column(f, opts): try: return opts.get_field(f, False).column except FieldDoesNotExist: return f def orderlist2sql(order_list, opts, prefix=''): qn = connection.ops.quote_name if prefix.endswith('.'): prefix = qn(prefix[:-1]) + '.' output = [] for f in handle_legacy_orderlist(order_list): if f.startswith('-'): output.append('%s%s DESC' % (prefix, qn(orderfield2column(f[1:], opts)))) elif f == '?': output.append(connection.ops.random_function_sql()) else: output.append('%s%s ASC' % (prefix, qn(orderfield2column(f, opts)))) return ', '.join(output) def quote_only_if_word(word): if re.search('\W', word): # Don't quote if there are spaces or non-word chars. return word else: return connection.ops.quote_name(word) class _QuerySet(object): "Represents a lazy database lookup for a set of objects" def __init__(self, model=None): self.model = model self._filters = Q() self._order_by = None # Ordering, e.g. ('date', '-name'). If None, use model's ordering. self._select_related = False # Whether to fill cache for related objects. self._max_related_depth = 0 # Maximum "depth" for select_related self._distinct = False # Whether the query should use SELECT DISTINCT. self._select = {} # Dictionary of attname -> SQL. self._where = [] # List of extra WHERE clauses to use. self._params = [] # List of params to use for extra WHERE clauses. self._tables = [] # List of extra tables to use. self._offset = None # OFFSET clause. self._limit = None # LIMIT clause. self._result_cache = None ######################## # PYTHON MAGIC METHODS # ######################## def __repr__(self): return repr(self._get_data()) def __len__(self): return len(self._get_data()) def __iter__(self): return iter(self._get_data()) def __getitem__(self, k): "Retrieve an item or slice from the set of results." if not isinstance(k, (slice, int, long)): raise TypeError assert (not isinstance(k, slice) and (k >= 0)) \ or (isinstance(k, slice) and (k.start is None or k.start >= 0) and (k.stop is None or k.stop >= 0)), \ "Negative indexing is not supported." if self._result_cache is None: if isinstance(k, slice): # Offset: if self._offset is None: offset = k.start elif k.start is None: offset = self._offset else: offset = self._offset + k.start # Now adjust offset to the bounds of any existing limit: if self._limit is not None and k.start is not None: limit = self._limit - k.start else: limit = self._limit # Limit: if k.stop is not None and k.start is not None: if limit is None: limit = k.stop - k.start else: limit = min((k.stop - k.start), limit) else: if limit is None: limit = k.stop else: if k.stop is not None: limit = min(k.stop, limit) if k.step is None: return self._clone(_offset=offset, _limit=limit) else: return list(self._clone(_offset=offset, _limit=limit))[::k.step] else: try: return list(self._clone(_offset=k, _limit=1))[0] except self.model.DoesNotExist, e: raise IndexError, e.args else: return self._result_cache[k] def __and__(self, other): combined = self._combine(other) combined._filters = self._filters & other._filters return combined def __or__(self, other): combined = self._combine(other) combined._filters = self._filters | other._filters return combined #################################### # METHODS THAT DO DATABASE QUERIES # #################################### def iterator(self): "Performs the SELECT database lookup of this QuerySet." try: select, sql, params = self._get_sql_clause() except EmptyResultSet: raise StopIteration # self._select is a dictionary, and dictionaries' key order is # undefined, so we convert it to a list of tuples. extra_select = self._select.items() cursor = connection.cursor() cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params) fill_cache = self._select_related fields = self.model._meta.fields index_end = len(fields) has_resolve_columns = hasattr(self, 'resolve_columns') while 1: rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE) if not rows: raise StopIteration for row in rows: if has_resolve_columns: row = self.resolve_columns(row, fields) if fill_cache: obj, index_end = get_cached_row(klass=self.model, row=row, index_start=0, max_depth=self._max_related_depth) else: obj = self.model(*row[:index_end]) for i, k in enumerate(extra_select): setattr(obj, k[0], row[index_end+i]) yield obj def count(self): """ Performs a SELECT COUNT() and returns the number of records as an integer. If the queryset is already cached (i.e. self._result_cache is set) this simply returns the length of the cached results set to avoid multiple SELECT COUNT(*) calls. """ if self._result_cache is not None: return len(self._result_cache) counter = self._clone() counter._order_by = () counter._select_related = False offset = counter._offset limit = counter._limit counter._offset = None counter._limit = None try: select, sql, params = counter._get_sql_clause() except EmptyResultSet: return 0 cursor = connection.cursor() if self._distinct: id_col = "%s.%s" % (connection.ops.quote_name(self.model._meta.db_table), connection.ops.quote_name(self.model._meta.pk.column)) cursor.execute("SELECT COUNT(DISTINCT(%s))" % id_col + sql, params) else: cursor.execute("SELECT COUNT(*)" + sql, params) count = cursor.fetchone()[0] # Apply any offset and limit constraints manually, since using LIMIT or # OFFSET in SQL doesn't change the output of COUNT. if offset: count = max(0, count - offset) if limit: count = min(limit, count) return count def get(self, *args, **kwargs): "Performs the SELECT and returns a single object matching the given keyword arguments." clone = self.filter(*args, **kwargs) # clean up SQL by removing unneeded ORDER BY if not clone._order_by: clone._order_by = () obj_list = list(clone) if len(obj_list) < 1: raise self.model.DoesNotExist, "%s matching query does not exist." % self.model._meta.object_name assert len(obj_list) == 1, "get() returned more than one %s -- it returned %s! Lookup parameters were %s" % (self.model._meta.object_name, len(obj_list), kwargs) return obj_list[0] def create(self, **kwargs): """ Create a new object with the given kwargs, saving it to the database and returning the created object. """ obj = self.model(**kwargs) obj.save() return obj def get_or_create(self, **kwargs): """ Looks up an object with the given kwargs, creating one if necessary. Returns a tuple of (object, created), where created is a boolean specifying whether an object was created. """ assert len(kwargs), 'get_or_create() must be passed at least one keyword argument' defaults = kwargs.pop('defaults', {}) try: return self.get(**kwargs), False except self.model.DoesNotExist: params = dict([(k, v) for k, v in kwargs.items() if '__' not in k]) params.update(defaults) obj = self.model(**params) obj.save() return obj, True def latest(self, field_name=None): """ Returns the latest object, according to the model's 'get_latest_by' option or optional given field_name. """ latest_by = field_name or self.model._meta.get_latest_by assert bool(latest_by), "latest() requires either a field_name parameter or 'get_latest_by' in the model" assert self._limit is None and self._offset is None, \ "Cannot change a query once a slice has been taken." return self._clone(_limit=1, _order_by=('-'+latest_by,)).get() def in_bulk(self, id_list): """ Returns a dictionary mapping each of the given IDs to the object with that ID. """ assert self._limit is None and self._offset is None, \ "Cannot use 'limit' or 'offset' with in_bulk" assert isinstance(id_list, (tuple, list)), "in_bulk() must be provided with a list of IDs." qn = connection.ops.quote_name id_list = list(id_list) if id_list == []: return {} qs = self._clone() qs._where.append("%s.%s IN (%s)" % (qn(self.model._meta.db_table), qn(self.model._meta.pk.column), ",".join(['%s'] * len(id_list)))) qs._params.extend(id_list) return dict([(obj._get_pk_val(), obj) for obj in qs.iterator()]) def delete(self): """ Deletes the records in the current QuerySet. """ assert self._limit is None and self._offset is None, \ "Cannot use 'limit' or 'offset' with delete." del_query = self._clone() # disable non-supported fields del_query._select_related = False del_query._order_by = [] # Delete objects in chunks to prevent an the list of # related objects from becoming too long more_objects = True while more_objects: # Collect all the objects to be deleted in this chunk, and all the objects # that are related to the objects that are to be deleted seen_objs = SortedDict() more_objects = False for object in del_query[0:GET_ITERATOR_CHUNK_SIZE]: more_objects = True object._collect_sub_objects(seen_objs) # If one or more objects were found, delete them. # Otherwise, stop looping. if more_objects: delete_objects(seen_objs) # Clear the result cache, in case this QuerySet gets reused. self._result_cache = None delete.alters_data = True ################################################## # PUBLIC METHODS THAT RETURN A QUERYSET SUBCLASS # ################################################## def values(self, *fields): return self._clone(klass=ValuesQuerySet, _fields=fields) def dates(self, field_name, kind, order='ASC'): """ Returns a list of datetime objects representing all available dates for the given field_name, scoped to 'kind'. """ assert kind in ("month", "year", "day"), "'kind' must be one of 'year', 'month' or 'day'." assert order in ('ASC', 'DESC'), "'order' must be either 'ASC' or 'DESC'." # Let the FieldDoesNotExist exception propagate. field = self.model._meta.get_field(field_name, many_to_many=False) assert isinstance(field, DateField), "%r isn't a DateField." % field_name return self._clone(klass=DateQuerySet, _field=field, _kind=kind, _order=order) ################################################################## # PUBLIC METHODS THAT ALTER ATTRIBUTES AND RETURN A NEW QUERYSET # ################################################################## def filter(self, *args, **kwargs): "Returns a new QuerySet instance with the args ANDed to the existing set." return self._filter_or_exclude(None, *args, **kwargs) def exclude(self, *args, **kwargs): "Returns a new QuerySet instance with NOT (args) ANDed to the existing set." return self._filter_or_exclude(QNot, *args, **kwargs) def _filter_or_exclude(self, mapper, *args, **kwargs): # mapper is a callable used to transform Q objects, # or None for identity transform if mapper is None: mapper = lambda x: x if len(args) > 0 or len(kwargs) > 0: assert self._limit is None and self._offset is None, \ "Cannot filter a query once a slice has been taken." clone = self._clone() if len(kwargs) > 0: clone._filters = clone._filters & mapper(Q(**kwargs)) if len(args) > 0: clone._filters = clone._filters & reduce(operator.and_, map(mapper, args)) return clone def complex_filter(self, filter_obj): """Returns a new QuerySet instance with filter_obj added to the filters. filter_obj can be a Q object (has 'get_sql' method) or a dictionary of keyword lookup arguments.""" # This exists to support framework features such as 'limit_choices_to', # and usually it will be more natural to use other methods. if hasattr(filter_obj, 'get_sql'): return self._filter_or_exclude(None, filter_obj) else: return self._filter_or_exclude(None, **filter_obj) def select_related(self, true_or_false=True, depth=0): "Returns a new QuerySet instance with '_select_related' modified." return self._clone(_select_related=true_or_false, _max_related_depth=depth) def order_by(self, *field_names): "Returns a new QuerySet instance with the ordering changed." assert self._limit is None and self._offset is None, \ "Cannot reorder a query once a slice has been taken." return self._clone(_order_by=field_names) def distinct(self, true_or_false=True): "Returns a new QuerySet instance with '_distinct' modified." return self._clone(_distinct=true_or_false) def extra(self, select=None, where=None, params=None, tables=None): assert self._limit is None and self._offset is None, \ "Cannot change a query once a slice has been taken" clone = self._clone() if select: clone._select.update(select) if where: clone._where.extend(where) if params: clone._params.extend(params) if tables: clone._tables.extend(tables) return clone ################### # PRIVATE METHODS # ################### def _clone(self, klass=None, **kwargs): if klass is None: klass = self.__class__ c = klass() c.model = self.model c._filters = self._filters c._order_by = self._order_by c._select_related = self._select_related c._max_related_depth = self._max_related_depth c._distinct = self._distinct c._select = self._select.copy() c._where = self._where[:] c._params = self._params[:] c._tables = self._tables[:] c._offset = self._offset c._limit = self._limit c.__dict__.update(kwargs) return c def _combine(self, other): assert self._limit is None and self._offset is None \ and other._limit is None and other._offset is None, \ "Cannot combine queries once a slice has been taken." assert self._distinct == other._distinct, \ "Cannot combine a unique query with a non-unique query" # use 'other's order by # (so that A.filter(args1) & A.filter(args2) does the same as # A.filter(args1).filter(args2) combined = other._clone() if self._select: combined._select.update(self._select) if self._where: combined._where.extend(self._where) if self._params: combined._params.extend(self._params) if self._tables: combined._tables.extend(self._tables) # If 'self' is ordered and 'other' isn't, propagate 'self's ordering if (self._order_by is not None and len(self._order_by) > 0) and \ (combined._order_by is None or len(combined._order_by) == 0): combined._order_by = self._order_by return combined def _get_data(self): if self._result_cache is None: self._result_cache = list(self.iterator()) return self._result_cache def _get_sql_clause(self): qn = connection.ops.quote_name opts = self.model._meta # Construct the fundamental parts of the query: SELECT X FROM Y WHERE Z. select = ["%s.%s" % (qn(opts.db_table), qn(f.column)) for f in opts.fields] tables = [quote_only_if_word(t) for t in self._tables] joins = SortedDict() where = self._where[:] params = self._params[:] # Convert self._filters into SQL. joins2, where2, params2 = self._filters.get_sql(opts) joins.update(joins2) where.extend(where2) params.extend(params2) # Add additional tables and WHERE clauses based on select_related. if self._select_related: fill_table_cache(opts, select, tables, where, old_prefix=opts.db_table, cache_tables_seen=[opts.db_table], max_depth=self._max_related_depth) # Add any additional SELECTs. if self._select: select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), qn(s[0])) for s in self._select.items()]) # Start composing the body of the SQL statement. sql = [" FROM", qn(opts.db_table)] # Compose the join dictionary into SQL describing the joins. if joins: sql.append(" ".join(["%s %s AS %s ON %s" % (join_type, table, alias, condition) for (alias, (table, join_type, condition)) in joins.items()])) # Compose the tables clause into SQL. if tables: sql.append(", " + ", ".join(tables)) # Compose the where clause into SQL. if where: sql.append(where and "WHERE " + " AND ".join(where)) # ORDER BY clause order_by = [] if self._order_by is not None: ordering_to_use = self._order_by else: ordering_to_use = opts.ordering for f in handle_legacy_orderlist(ordering_to_use): if f == '?': # Special case. order_by.append(connection.ops.random_function_sql()) else: if f.startswith('-'): col_name = f[1:] order = "DESC" else: col_name = f order = "ASC" if "." in col_name: table_prefix, col_name = col_name.split('.', 1) table_prefix = qn(table_prefix) + '.' else: # Use the database table as a column prefix if it wasn't given, # and if the requested column isn't a custom SELECT. if "." not in col_name and col_name not in (self._select or ()): table_prefix = qn(opts.db_table) + '.' else: table_prefix = '' order_by.append('%s%s %s' % (table_prefix, qn(orderfield2column(col_name, opts)), order)) if order_by: sql.append("ORDER BY " + ", ".join(order_by)) # LIMIT and OFFSET clauses if self._limit is not None: sql.append("%s " % connection.ops.limit_offset_sql(self._limit, self._offset)) else: assert self._offset is None, "'offset' is not allowed without 'limit'" return select, " ".join(sql), params # Use the backend's QuerySet class if it defines one. Otherwise, use _QuerySet. if connection.features.uses_custom_queryset: QuerySet = connection.ops.query_set_class(_QuerySet) else: QuerySet = _QuerySet class ValuesQuerySet(QuerySet): def __init__(self, *args, **kwargs): super(ValuesQuerySet, self).__init__(*args, **kwargs) # select_related isn't supported in values(). self._select_related = False def iterator(self): try: select, sql, params = self._get_sql_clause() except EmptyResultSet: raise StopIteration qn = connection.ops.quote_name # self._select is a dictionary, and dictionaries' key order is # undefined, so we convert it to a list of tuples. extra_select = self._select.items() # Construct two objects -- fields and field_names. # fields is a list of Field objects to fetch. # field_names is a list of field names, which will be the keys in the # resulting dictionaries. if self._fields: if not extra_select: fields = [self.model._meta.get_field(f, many_to_many=False) for f in self._fields] field_names = self._fields else: fields = [] field_names = [] for f in self._fields: if f in [field.name for field in self.model._meta.fields]: fields.append(self.model._meta.get_field(f, many_to_many=False)) field_names.append(f) elif not self._select.has_key(f): raise FieldDoesNotExist('%s has no field named %r' % (self.model._meta.object_name, f)) else: # Default to all fields. fields = self.model._meta.fields field_names = [f.attname for f in fields] columns = [f.column for f in fields] select = ['%s.%s' % (qn(self.model._meta.db_table), qn(c)) for c in columns] if extra_select: select.extend(['(%s) AS %s' % (quote_only_if_word(s[1]), qn(s[0])) for s in extra_select]) field_names.extend([f[0] for f in extra_select]) cursor = connection.cursor() cursor.execute("SELECT " + (self._distinct and "DISTINCT " or "") + ",".join(select) + sql, params) has_resolve_columns = hasattr(self, 'resolve_columns') while 1: rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE) if not rows: raise StopIteration for row in rows: if has_resolve_columns: row = self.resolve_columns(row, fields) yield dict(zip(field_names, row)) def _clone(self, klass=None, **kwargs): c = super(ValuesQuerySet, self)._clone(klass, **kwargs) c._fields = self._fields[:] return c class DateQuerySet(QuerySet): def iterator(self): from django.db.backends.util import typecast_timestamp from django.db.models.fields import DateTimeField qn = connection.ops.quote_name self._order_by = () # Clear this because it'll mess things up otherwise. if self._field.null: self._where.append('%s.%s IS NOT NULL' % \ (qn(self.model._meta.db_table), qn(self._field.column))) try: select, sql, params = self._get_sql_clause() except EmptyResultSet: raise StopIteration table_name = qn(self.model._meta.db_table) field_name = qn(self._field.column) if connection.features.allows_group_by_ordinal: group_by = '1' else: group_by = connection.ops.date_trunc_sql(self._kind, '%s.%s' % (table_name, field_name)) sql = 'SELECT %s %s GROUP BY %s ORDER BY 1 %s' % \ (connection.ops.date_trunc_sql(self._kind, '%s.%s' % (qn(self.model._meta.db_table), qn(self._field.column))), sql, group_by, self._order) cursor = connection.cursor() cursor.execute(sql, params) has_resolve_columns = hasattr(self, 'resolve_columns') needs_datetime_string_cast = connection.features.needs_datetime_string_cast dates = [] # It would be better to use self._field here instead of DateTimeField(), # but in Oracle that will result in a list of datetime.date instead of # datetime.datetime. fields = [DateTimeField()] while 1: rows = cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE) if not rows: return dates for row in rows: date = row[0] if has_resolve_columns: date = self.resolve_columns([date], fields)[0] elif needs_datetime_string_cast: date = typecast_timestamp(str(date)) dates.append(date) def _clone(self, klass=None, **kwargs): c = super(DateQuerySet, self)._clone(klass, **kwargs) c._field = self._field c._kind = self._kind c._order = self._order return c class EmptyQuerySet(QuerySet): def __init__(self, model=None): super(EmptyQuerySet, self).__init__(model) self._result_cache = [] def count(self): return 0 def delete(self): pass def _clone(self, klass=None, **kwargs): c = super(EmptyQuerySet, self)._clone(klass, **kwargs) c._result_cache = [] return c def _get_sql_clause(self): raise EmptyResultSet class QOperator(object): "Base class for QAnd and QOr" def __init__(self, *args): self.args = args def get_sql(self, opts): joins, where, params = SortedDict(), [], [] for val in self.args: try: joins2, where2, params2 = val.get_sql(opts) joins.update(joins2) where.extend(where2) params.extend(params2) except EmptyResultSet: if not isinstance(self, QOr): raise EmptyResultSet if where: return joins, ['(%s)' % self.operator.join(where)], params return joins, [], params class QAnd(QOperator): "Encapsulates a combined query that uses 'AND'." operator = ' AND ' def __or__(self, other): return QOr(self, other) def __and__(self, other): if isinstance(other, QAnd): return QAnd(*(self.args+other.args)) elif isinstance(other, (Q, QOr)): return QAnd(*(self.args+(other,))) else: raise TypeError, other class QOr(QOperator): "Encapsulates a combined query that uses 'OR'." operator = ' OR ' def __and__(self, other): return QAnd(self, other) def __or__(self, other): if isinstance(other, QOr): return QOr(*(self.args+other.args)) elif isinstance(other, (Q, QAnd)): return QOr(*(self.args+(other,))) else: raise TypeError, other class Q(object): "Encapsulates queries as objects that can be combined logically." def __init__(self, **kwargs): self.kwargs = kwargs def __and__(self, other): return QAnd(self, other) def __or__(self, other): return QOr(self, other) def get_sql(self, opts): return parse_lookup(self.kwargs.items(), opts) class QNot(Q): "Encapsulates NOT (...) queries as objects" def __init__(self, q): "Creates a negation of the q object passed in." self.q = q def get_sql(self, opts): try: joins, where, params = self.q.get_sql(opts) where2 = ['(NOT (%s))' % " AND ".join(where)] except EmptyResultSet: return SortedDict(), [], [] return joins, where2, params def get_where_clause(lookup_type, table_prefix, field_name, value, db_type): if table_prefix.endswith('.'): table_prefix = connection.ops.quote_name(table_prefix[:-1])+'.' field_name = connection.ops.quote_name(field_name) if type(value) == datetime.datetime and connection.ops.datetime_cast_sql(): cast_sql = connection.ops.datetime_cast_sql() else: cast_sql = '%s' field_sql = connection.ops.field_cast_sql(db_type) % (table_prefix + field_name) if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith') and connection.features.needs_upper_for_iops: format = 'UPPER(%s) %s' else: format = '%s %s' try: return format % (field_sql, connection.operators[lookup_type] % cast_sql) except KeyError: pass if lookup_type == 'in': in_string = ','.join(['%s' for id in value]) if in_string: return '%s IN (%s)' % (field_sql, in_string) else: raise EmptyResultSet elif lookup_type in ('range', 'year'): return '%s BETWEEN %%s AND %%s' % field_sql elif lookup_type in ('month', 'day'): return "%s = %%s" % connection.ops.date_extract_sql(lookup_type, field_sql) elif lookup_type == 'isnull': return "%s IS %sNULL" % (field_sql, (not value and 'NOT ' or '')) elif lookup_type == 'search': return connection.ops.fulltext_search_sql(field_sql) elif lookup_type in ('regex', 'iregex'): if settings.DATABASE_ENGINE == 'oracle': if lookup_type == 'regex': match_option = 'c' else: match_option = 'i' return "REGEXP_LIKE(%s, %s, '%s')" % (field_sql, cast_sql, match_option) else: raise NotImplementedError raise TypeError, "Got invalid lookup_type: %s" % repr(lookup_type) def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0): """Helper function that recursively returns an object with cache filled""" # If we've got a max_depth set and we've exceeded that depth, bail now. if max_depth and cur_depth > max_depth: return None index_end = index_start + len(klass._meta.fields) obj = klass(*row[index_start:index_end]) for f in klass._meta.fields: if f.rel and not f.null: cached_row = get_cached_row(f.rel.to, row, index_end, max_depth, cur_depth+1) if cached_row: rel_obj, index_end = cached_row setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end def fill_table_cache(opts, select, tables, where, old_prefix, cache_tables_seen, max_depth=0, cur_depth=0): """ Helper function that recursively populates the select, tables and where (in place) for select_related queries. """ # If we've got a max_depth set and we've exceeded that depth, bail now. if max_depth and cur_depth > max_depth: return None qn = connection.ops.quote_name for f in opts.fields: if f.rel and not f.null: db_table = f.rel.to._meta.db_table if db_table not in cache_tables_seen: tables.append(qn(db_table)) else: # The table was already seen, so give it a table alias. new_prefix = '%s%s' % (db_table, len(cache_tables_seen)) tables.append('%s %s' % (qn(db_table), qn(new_prefix))) db_table = new_prefix cache_tables_seen.append(db_table) where.append('%s.%s = %s.%s' % \ (qn(old_prefix), qn(f.column), qn(db_table), qn(f.rel.get_related_field().column))) select.extend(['%s.%s' % (qn(db_table), qn(f2.column)) for f2 in f.rel.to._meta.fields]) fill_table_cache(f.rel.to._meta, select, tables, where, db_table, cache_tables_seen, max_depth, cur_depth+1) def parse_lookup(kwarg_items, opts): # Helper function that handles converting API kwargs # (e.g. "name__exact": "tom") to SQL. # Returns a tuple of (joins, where, params). # 'joins' is a sorted dictionary describing the tables that must be joined # to complete the query. The dictionary is sorted because creation order # is significant; it is a dictionary to ensure uniqueness of alias names. # # Each key-value pair follows the form # alias: (table, join_type, condition) # where # alias is the AS alias for the joined table # table is the actual table name to be joined # join_type is the type of join (INNER JOIN, LEFT OUTER JOIN, etc) # condition is the where-like statement over which narrows the join. # alias will be derived from the lookup list name. # # At present, this method only every returns INNER JOINs; the option is # there for others to implement custom Q()s, etc that return other join # types. joins, where, params = SortedDict(), [], [] for kwarg, value in kwarg_items: path = kwarg.split(LOOKUP_SEPARATOR) # Extract the last elements of the kwarg. # The very-last is the lookup_type (equals, like, etc). # The second-last is the table column on which the lookup_type is # to be performed. If this name is 'pk', it will be substituted with # the name of the primary key. # If there is only one part, or the last part is not a query # term, assume that the query is an __exact lookup_type = path.pop() if lookup_type == 'pk': lookup_type = 'exact' path.append(None) elif len(path) == 0 or lookup_type not in QUERY_TERMS: path.append(lookup_type) lookup_type = 'exact' if len(path) < 1: raise TypeError, "Cannot parse keyword query %r" % kwarg if value is None: # Interpret '__exact=None' as the sql '= NULL'; otherwise, reject # all uses of None as a query value. if lookup_type != 'exact': raise ValueError, "Cannot use None as a query value" elif callable(value): value = value() joins2, where2, params2 = lookup_inner(path, lookup_type, value, opts, opts.db_table, None) joins.update(joins2) where.extend(where2) params.extend(params2) return joins, where, params class FieldFound(Exception): "Exception used to short circuit field-finding operations." pass def find_field(name, field_list, related_query): """ Finds a field with a specific name in a list of field instances. Returns None if there are no matches, or several matches. """ if related_query: matches = [f for f in field_list if f.field.related_query_name() == name] else: matches = [f for f in field_list if f.name == name] if len(matches) != 1: return None return matches[0] def field_choices(field_list, related_query): if related_query: choices = [f.field.related_query_name() for f in field_list] else: choices = [f.name for f in field_list] return choices def lookup_inner(path, lookup_type, value, opts, table, column): qn = connection.ops.quote_name joins, where, params = SortedDict(), [], [] current_opts = opts current_table = table current_column = column intermediate_table = None join_required = False name = path.pop(0) # Has the primary key been requested? If so, expand it out # to be the name of the current class' primary key if name is None or name == 'pk': name = current_opts.pk.name # Try to find the name in the fields associated with the current class try: # Does the name belong to a defined many-to-many field? field = find_field(name, current_opts.many_to_many, False) if field: new_table = current_table + '__' + name new_opts = field.rel.to._meta new_column = new_opts.pk.column # Need to create an intermediate table join over the m2m table # This process hijacks current_table/column to point to the # intermediate table. current_table = "m2m_" + new_table intermediate_table = field.m2m_db_table() join_column = field.m2m_reverse_name() intermediate_column = field.m2m_column_name() raise FieldFound # Does the name belong to a reverse defined many-to-many field? field = find_field(name, current_opts.get_all_related_many_to_many_objects(), True) if field: new_table = current_table + '__' + name new_opts = field.opts new_column = new_opts.pk.column # Need to create an intermediate table join over the m2m table. # This process hijacks current_table/column to point to the # intermediate table. current_table = "m2m_" + new_table intermediate_table = field.field.m2m_db_table() join_column = field.field.m2m_column_name() intermediate_column = field.field.m2m_reverse_name() raise FieldFound # Does the name belong to a one-to-many field? field = find_field(name, current_opts.get_all_related_objects(), True) if field: new_table = table + '__' + name new_opts = field.opts new_column = field.field.column join_column = opts.pk.column # 1-N fields MUST be joined, regardless of any other conditions. join_required = True raise FieldFound # Does the name belong to a one-to-one, many-to-one, or regular field? field = find_field(name, current_opts.fields, False) if field: if field.rel: # One-to-One/Many-to-one field new_table = current_table + '__' + name new_opts = field.rel.to._meta new_column = new_opts.pk.column join_column = field.column raise FieldFound elif path: # For regular fields, if there are still items on the path, # an error has been made. We munge "name" so that the error # properly identifies the cause of the problem. name += LOOKUP_SEPARATOR + path[0] else: raise FieldFound except FieldFound: # Match found, loop has been shortcut. pass else: # No match found. choices = field_choices(current_opts.many_to_many, False) + \ field_choices(current_opts.get_all_related_many_to_many_objects(), True) + \ field_choices(current_opts.get_all_related_objects(), True) + \ field_choices(current_opts.fields, False) raise TypeError, "Cannot resolve keyword '%s' into field. Choices are: %s" % (name, ", ".join(choices)) # Check whether an intermediate join is required between current_table # and new_table. if intermediate_table: joins[qn(current_table)] = ( qn(intermediate_table), "LEFT OUTER JOIN", "%s.%s = %s.%s" % (qn(table), qn(current_opts.pk.column), qn(current_table), qn(intermediate_column)) ) if path: # There are elements left in the path. More joins are required. if len(path) == 1 and path[0] in (new_opts.pk.name, None) \ and lookup_type in ('exact', 'isnull') and not join_required: # If the next and final name query is for a primary key, # and the search is for isnull/exact, then the current # (for N-1) or intermediate (for N-N) table can be used # for the search. No need to join an extra table just # to check the primary key. new_table = current_table else: # There are 1 or more name queries pending, and we have ruled out # any shortcuts; therefore, a join is required. joins[qn(new_table)] = ( qn(new_opts.db_table), "INNER JOIN", "%s.%s = %s.%s" % (qn(current_table), qn(join_column), qn(new_table), qn(new_column)) ) # If we have made the join, we don't need to tell subsequent # recursive calls about the column name we joined on. join_column = None # There are name queries remaining. Recurse deeper. joins2, where2, params2 = lookup_inner(path, lookup_type, value, new_opts, new_table, join_column) joins.update(joins2) where.extend(where2) params.extend(params2) else: # No elements left in path. Current element is the element on which # the search is being performed. db_type = None if join_required: # Last query term is a RelatedObject if field.field.rel.multiple: # RelatedObject is from a 1-N relation. # Join is required; query operates on joined table. column = new_opts.pk.name joins[qn(new_table)] = ( qn(new_opts.db_table), "INNER JOIN", "%s.%s = %s.%s" % (qn(current_table), qn(join_column), qn(new_table), qn(new_column)) ) current_table = new_table else: # RelatedObject is from a 1-1 relation, # No need to join; get the pk value from the related object, # and compare using that. column = current_opts.pk.name elif intermediate_table: # Last query term is a related object from an N-N relation. # Join from intermediate table is sufficient. column = join_column elif name == current_opts.pk.name and lookup_type in ('exact', 'isnull') and current_column: # Last query term is for a primary key. If previous iterations # introduced a current/intermediate table that can be used to # optimize the query, then use that table and column name. column = current_column else: # Last query term was a normal field. column = field.column db_type = field.db_type() where.append(get_where_clause(lookup_type, current_table + '.', column, value, db_type)) params.extend(field.get_db_prep_lookup(lookup_type, value)) return joins, where, params def delete_objects(seen_objs): "Iterate through a list of seen classes, and remove any instances that are referred to" qn = connection.ops.quote_name ordered_classes = seen_objs.keys() ordered_classes.reverse() cursor = connection.cursor() for cls in ordered_classes: seen_objs[cls] = seen_objs[cls].items() seen_objs[cls].sort() # Pre notify all instances to be deleted for pk_val, instance in seen_objs[cls]: dispatcher.send(signal=signals.pre_delete, sender=cls, instance=instance) pk_list = [pk for pk,instance in seen_objs[cls]] for related in cls._meta.get_all_related_many_to_many_objects(): if not isinstance(related.field, generic.GenericRelation): for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ (qn(related.field.m2m_db_table()), qn(related.field.m2m_reverse_name()), ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])), pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]) for f in cls._meta.many_to_many: if isinstance(f, generic.GenericRelation): from django.contrib.contenttypes.models import ContentType query_extra = 'AND %s=%%s' % f.rel.to._meta.get_field(f.content_type_field_name).column args_extra = [ContentType.objects.get_for_model(cls).id] else: query_extra = '' args_extra = [] for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): cursor.execute(("DELETE FROM %s WHERE %s IN (%s)" % \ (qn(f.m2m_db_table()), qn(f.m2m_column_name()), ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]]))) + query_extra, pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE] + args_extra) for field in cls._meta.fields: if field.rel and field.null and field.rel.to in seen_objs: for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): cursor.execute("UPDATE %s SET %s=NULL WHERE %s IN (%s)" % \ (qn(cls._meta.db_table), qn(field.column), qn(cls._meta.pk.column), ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])), pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]) # Now delete the actual data for cls in ordered_classes: seen_objs[cls].reverse() pk_list = [pk for pk,instance in seen_objs[cls]] for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): cursor.execute("DELETE FROM %s WHERE %s IN (%s)" % \ (qn(cls._meta.db_table), qn(cls._meta.pk.column), ','.join(['%s' for pk in pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]])), pk_list[offset:offset+GET_ITERATOR_CHUNK_SIZE]) # Last cleanup; set NULLs where there once was a reference to the object, # NULL the primary key of the found objects, and perform post-notification. for pk_val, instance in seen_objs[cls]: for field in cls._meta.fields: if field.rel and field.null and field.rel.to in seen_objs: setattr(instance, field.attname, None) setattr(instance, cls._meta.pk.attname, None) dispatcher.send(signal=signals.post_delete, sender=cls, instance=instance) transaction.commit_unless_managed()