Fixed #10109 -- Removed the use of raw SQL in many-to-many fields by introducing an autogenerated through model.

This is the first part of Alex Gaynor's GSoC project to add Multi-db support to Django.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11710 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/contrib/admin/options.py

@@ -153,8 +153,9 @@ class BaseModelAdmin(object):
         """
         Get a form Field for a ManyToManyField.
         """
-        # If it uses an intermediary model, don't show field in admin.
+        # If it uses an intermediary model that isn't auto created, don't show
-        if db_field.rel.through is not None:
+        # a field in admin.
+        if not db_field.rel.through._meta.auto_created:
             return None
 
         if db_field.name in self.raw_id_fields:

django/contrib/contenttypes/generic.py

@@ -105,8 +105,6 @@ class GenericRelation(RelatedField, Field):
                             limit_choices_to=kwargs.pop('limit_choices_to', None),
                             symmetrical=kwargs.pop('symmetrical', True))
 
-        # By its very nature, a GenericRelation doesn't create a table.
-        self.creates_table = False
 
         # Override content-type/object-id field names on the related class
         self.object_id_field_name = kwargs.pop("object_id_field", "object_id")

django/core/management/commands/syncdb.py

@@ -57,12 +57,15 @@ class Command(NoArgsCommand):
         # Create the tables for each model
         for app in models.get_apps():
             app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
+            model_list = models.get_models(app, include_auto_created=True)
             for model in model_list:
                 # Create the model's database table, if it doesn't already exist.
                 if verbosity >= 2:
                     print "Processing %s.%s model" % (app_name, model._meta.object_name)
-                if connection.introspection.table_name_converter(model._meta.db_table) in tables:
+                opts = model._meta
+                if (connection.introspection.table_name_converter(opts.db_table) in tables or
+                    (opts.auto_created and
+                    connection.introspection.table_name_converter(opts.auto_created._meta.db_table in tables))):
                     continue
                 sql, references = connection.creation.sql_create_model(model, self.style, seen_models)
                 seen_models.add(model)
@@ -78,19 +81,6 @@ class Command(NoArgsCommand):
                     cursor.execute(statement)
                 tables.append(connection.introspection.table_name_converter(model._meta.db_table))
 
-        # Create the m2m tables. This must be done after all tables have been created
-        # to ensure that all referred tables will exist.
-        for app in models.get_apps():
-            app_name = app.__name__.split('.')[-2]
-            model_list = models.get_models(app)
-            for model in model_list:
-                if model in created_models:
-                    sql = connection.creation.sql_for_many_to_many(model, self.style)
-                    if sql:
-                        if verbosity >= 2:
-                            print "Creating many-to-many tables for %s.%s model" % (app_name, model._meta.object_name)
-                        for statement in sql:
-                            cursor.execute(statement)
 
         transaction.commit_unless_managed()
 

django/core/management/sql.py

@@ -23,7 +23,7 @@ def sql_create(app, style):
     # We trim models from the current app so that the sqlreset command does not
     # generate invalid SQL (leaving models out of known_models is harmless, so
     # we can be conservative).
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     final_output = []
     tables = connection.introspection.table_names()
     known_models = set([model for model in connection.introspection.installed_models(tables) if model not in app_models])
@@ -40,10 +40,6 @@ def sql_create(app, style):
         # Keep track of the fact that we've created the table for this model.
         known_models.add(model)
 
-    # Create the many-to-many join tables.
-    for model in app_models:
-        final_output.extend(connection.creation.sql_for_many_to_many(model, style))
-
     # Handle references to tables that are from other apps
     # but don't exist physically.
     not_installed_models = set(pending_references.keys())
@@ -82,7 +78,7 @@ def sql_delete(app, style):
     to_delete = set()
 
     references_to_delete = {}
-    app_models = models.get_models(app)
+    app_models = models.get_models(app, include_auto_created=True)
     for model in app_models:
         if cursor and connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             # The table exists, so it needs to be dropped
@@ -97,13 +93,6 @@ def sql_delete(app, style):
         if connection.introspection.table_name_converter(model._meta.db_table) in table_names:
             output.extend(connection.creation.sql_destroy_model(model, references_to_delete, style))
 
-    # Output DROP TABLE statements for many-to-many tables.
-    for model in app_models:
-        opts = model._meta
-        for f in opts.local_many_to_many:
-            if cursor and connection.introspection.table_name_converter(f.m2m_db_table()) in table_names:
-                output.extend(connection.creation.sql_destroy_many_to_many(model, f, style))
-
     # Close database connection explicitly, in case this output is being piped
     # directly into a database client, to avoid locking issues.
     if cursor:

django/core/management/validation.py

@@ -79,6 +79,7 @@ def get_validation_errors(outfile, app=None):
                 rel_opts = f.rel.to._meta
                 rel_name = RelatedObject(f.rel.to, cls, f).get_accessor_name()
                 rel_query_name = f.related_query_name()
+                if not f.rel.is_hidden():
                     for r in rel_opts.fields:
                         if r.name == rel_name:
                             e.add(opts, "Accessor for field '%s' clashes with field '%s.%s'. Add a related_name argument to the definition for '%s'." % (f.name, rel_opts.object_name, r.name, f.name))
@@ -117,48 +118,80 @@ def get_validation_errors(outfile, app=None):
             if f.unique:
                 e.add(opts, "ManyToManyFields cannot be unique.  Remove the unique argument on '%s'." % f.name)
 
-            if getattr(f.rel, 'through', None) is not None:
+            if f.rel.through is not None and not isinstance(f.rel.through, basestring):
-                if hasattr(f.rel, 'through_model'):
                 from_model, to_model = cls, f.rel.to
-                    if from_model == to_model and f.rel.symmetrical:
+                if from_model == to_model and f.rel.symmetrical and not f.rel.through._meta.auto_created:
                     e.add(opts, "Many-to-many fields with intermediate tables cannot be symmetrical.")
                 seen_from, seen_to, seen_self = False, False, 0
-                    for inter_field in f.rel.through_model._meta.fields:
+                for inter_field in f.rel.through._meta.fields:
                     rel_to = getattr(inter_field.rel, 'to', None)
                     if from_model == to_model: # relation to self
                         if rel_to == from_model:
                             seen_self += 1
                         if seen_self > 2:
-                                e.add(opts, "Intermediary model %s has more than two foreign keys to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, from_model._meta.object_name))
+                            e.add(opts, "Intermediary model %s has more than "
+                                "two foreign keys to %s, which is ambiguous "
+                                "and is not permitted." % (
+                                    f.rel.through._meta.object_name,
+                                    from_model._meta.object_name
+                                )
+                            )
                     else:
                         if rel_to == from_model:
                             if seen_from:
-                                    e.add(opts, "Intermediary model %s has more than one foreign key to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, from_model._meta.object_name))
+                                e.add(opts, "Intermediary model %s has more "
+                                    "than one foreign key to %s, which is "
+                                    "ambiguous and is not permitted." % (
+                                        f.rel.through._meta.object_name,
+                                         from_model._meta.object_name
+                                     )
+                                 )
                             else:
                                 seen_from = True
                         elif rel_to == to_model:
                             if seen_to:
-                                    e.add(opts, "Intermediary model %s has more than one foreign key to %s, which is ambiguous and is not permitted." % (f.rel.through_model._meta.object_name, rel_to._meta.object_name))
+                                e.add(opts, "Intermediary model %s has more "
+                                    "than one foreign key to %s, which is "
+                                    "ambiguous and is not permitted." % (
+                                        f.rel.through._meta.object_name,
+                                        rel_to._meta.object_name
+                                    )
+                                )
                             else:
                                 seen_to = True
-                    if f.rel.through_model not in models.get_models():
+                if f.rel.through not in models.get_models(include_auto_created=True):
-                        e.add(opts, "'%s' specifies an m2m relation through model %s, which has not been installed." % (f.name, f.rel.through))
+                    e.add(opts, "'%s' specifies an m2m relation through model "
-                    signature = (f.rel.to, cls, f.rel.through_model)
+                        "%s, which has not been installed." % (f.name, f.rel.through)
+                    )
+                signature = (f.rel.to, cls, f.rel.through)
                 if signature in seen_intermediary_signatures:
-                        e.add(opts, "The model %s has two manually-defined m2m relations through the model %s, which is not permitted. Please consider using an extra field on your intermediary model instead." % (cls._meta.object_name, f.rel.through_model._meta.object_name))
+                    e.add(opts, "The model %s has two manually-defined m2m "
+                        "relations through the model %s, which is not "
+                        "permitted. Please consider using an extra field on "
+                        "your intermediary model instead." % (
+                            cls._meta.object_name,
+                            f.rel.through._meta.object_name
+                        )
+                    )
                 else:
                     seen_intermediary_signatures.append(signature)
                 seen_related_fk, seen_this_fk = False, False
-                    for field in f.rel.through_model._meta.fields:
+                for field in f.rel.through._meta.fields:
                     if field.rel:
                         if not seen_related_fk and field.rel.to == f.rel.to:
                             seen_related_fk = True
                         elif field.rel.to == cls:
                             seen_this_fk = True
                 if not seen_related_fk or not seen_this_fk:
-                        e.add(opts, "'%s' has a manually-defined m2m relation through model %s, which does not have foreign keys to %s and %s" % (f.name, f.rel.through, f.rel.to._meta.object_name, cls._meta.object_name))
+                    e.add(opts, "'%s' has a manually-defined m2m relation "
-                else:
+                        "through model %s, which does not have foreign keys "
-                    e.add(opts, "'%s' specifies an m2m relation through model %s, which has not been installed" % (f.name, f.rel.through))
+                        "to %s and %s" % (f.name, f.rel.through._meta.object_name,
+                            f.rel.to._meta.object_name, cls._meta.object_name)
+                    )
+            elif isinstance(f.rel.through, basestring):
+                e.add(opts, "'%s' specifies an m2m relation through model %s, "
+                    "which has not been installed" % (f.name, f.rel.through)
+                )
 
             rel_opts = f.rel.to._meta
             rel_name = RelatedObject(f.rel.to, cls, f).get_accessor_name()

django/core/serializers/python.py

@@ -56,7 +56,7 @@ class Serializer(base.Serializer):
         self._current[field.name] = smart_unicode(related, strings_only=True)
 
     def handle_m2m_field(self, obj, field):
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
                                for related in getattr(obj, field.name).iterator()]
 

django/core/serializers/xml_serializer.py

@@ -98,7 +98,7 @@ class Serializer(base.Serializer):
         serialized as references to the object's PK (i.e. the related *data*
         is not dumped, just the relation).
         """
-        if field.creates_table:
+        if field.rel.through._meta.auto_created:
             self._start_relational_field(field)
             for relobj in getattr(obj, field.name).iterator():
                 self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
@@ -233,4 +233,3 @@ def getInnerText(node):
         else:
            pass
     return u"".join(inner_text)
-

django/db/models/base.py

@@ -434,7 +434,7 @@ class Model(object):
         else:
             meta = cls._meta
 
-        if origin:
+        if origin and not meta.auto_created:
             signals.pre_save.send(sender=origin, instance=self, raw=raw)
 
         # If we are in a raw save, save the object exactly as presented.
@@ -507,7 +507,7 @@ class Model(object):
                     setattr(self, meta.pk.attname, result)
             transaction.commit_unless_managed()
 
-        if origin:
+        if origin and not meta.auto_created:
             signals.post_save.send(sender=origin, instance=self,
                 created=(not record_exists), raw=raw)
 
@@ -544,7 +544,12 @@ class Model(object):
                         rel_descriptor = cls.__dict__[rel_opts_name]
                         break
                 else:
+                    # in the case of a hidden fkey just skip it, it'll get
+                    # processed as an m2m
+                    if not related.field.rel.is_hidden():
                         raise AssertionError("Should never get here.")
+                    else:
+                        continue
                 delete_qs = rel_descriptor.delete_manager(self).all()
                 for sub_obj in delete_qs:
                     sub_obj._collect_sub_objects(seen_objs, self.__class__, related.field.null)

django/db/models/fields/related.py

@@ -58,6 +58,10 @@ def add_lazy_relation(cls, field, relation, operation):
             # If we can't split, assume a model in current app
             app_label = cls._meta.app_label
             model_name = relation
+        except AttributeError:
+            # If it doesn't have a split it's actually a model class
+            app_label = relation._meta.app_label
+            model_name = relation._meta.object_name
 
     # Try to look up the related model, and if it's already loaded resolve the
     # string right away. If get_model returns None, it means that the related
@@ -96,7 +100,7 @@ class RelatedField(object):
             self.rel.related_name = self.rel.related_name % {'class': cls.__name__.lower()}
 
         other = self.rel.to
-        if isinstance(other, basestring):
+        if isinstance(other, basestring) or other._meta.pk is None:
             def resolve_related_class(field, model, cls):
                 field.rel.to = model
                 field.do_related_class(model, cls)
@@ -401,22 +405,22 @@ class ForeignRelatedObjectsDescriptor(object):
 
         return manager
 
-def create_many_related_manager(superclass, through=False):
+def create_many_related_manager(superclass, rel=False):
     """Creates a manager that subclasses 'superclass' (which is a Manager)
     and adds behavior for many-to-many related objects."""
+    through = rel.through
     class ManyRelatedManager(superclass):
         def __init__(self, model=None, core_filters=None, instance=None, symmetrical=None,
-                join_table=None, source_col_name=None, target_col_name=None):
+                join_table=None, source_field_name=None, target_field_name=None):
             super(ManyRelatedManager, self).__init__()
             self.core_filters = core_filters
             self.model = model
             self.symmetrical = symmetrical
             self.instance = instance
-            self.join_table = join_table
+            self.source_field_name = source_field_name
-            self.source_col_name = source_col_name
+            self.target_field_name = target_field_name
-            self.target_col_name = target_col_name
             self.through = through
-            self._pk_val = self.instance._get_pk_val()
+            self._pk_val = self.instance.pk
             if self._pk_val is None:
                 raise ValueError("%r instance needs to have a primary key value before a many-to-many relationship can be used." % instance.__class__.__name__)
 
@@ -425,36 +429,37 @@ def create_many_related_manager(superclass, through=False):
 
         # If the ManyToMany relation has an intermediary model,
         # the add and remove methods do not exist.
-        if through is None:
+        if rel.through._meta.auto_created:
             def add(self, *objs):
-                self._add_items(self.source_col_name, self.target_col_name, *objs)
+                self._add_items(self.source_field_name, self.target_field_name, *objs)
 
                 # If this is a symmetrical m2m relation to self, add the mirror entry in the m2m table
                 if self.symmetrical:
-                    self._add_items(self.target_col_name, self.source_col_name, *objs)
+                    self._add_items(self.target_field_name, self.source_field_name, *objs)
             add.alters_data = True
 
             def remove(self, *objs):
-                self._remove_items(self.source_col_name, self.target_col_name, *objs)
+                self._remove_items(self.source_field_name, self.target_field_name, *objs)
 
                 # If this is a symmetrical m2m relation to self, remove the mirror entry in the m2m table
                 if self.symmetrical:
-                    self._remove_items(self.target_col_name, self.source_col_name, *objs)
+                    self._remove_items(self.target_field_name, self.source_field_name, *objs)
             remove.alters_data = True
 
         def clear(self):
-            self._clear_items(self.source_col_name)
+            self._clear_items(self.source_field_name)
 
             # If this is a symmetrical m2m relation to self, clear the mirror entry in the m2m table
             if self.symmetrical:
-                self._clear_items(self.target_col_name)
+                self._clear_items(self.target_field_name)
         clear.alters_data = True
 
         def create(self, **kwargs):
             # This check needs to be done here, since we can't later remove this
             # from the method lookup table, as we do with add and remove.
-            if through is not None:
+            if not rel.through._meta.auto_created:
-                raise AttributeError, "Cannot use create() on a ManyToManyField which specifies an intermediary model. Use %s's Manager instead." % through
+                opts = through._meta
+                raise AttributeError, "Cannot use create() on a ManyToManyField which specifies an intermediary model. Use %s.%s's Manager instead." % (opts.app_label, opts.object_name)
             new_obj = super(ManyRelatedManager, self).create(**kwargs)
             self.add(new_obj)
             return new_obj
@@ -470,41 +475,38 @@ def create_many_related_manager(superclass, through=False):
             return obj, created
         get_or_create.alters_data = True
 
-        def _add_items(self, source_col_name, target_col_name, *objs):
+        def _add_items(self, source_field_name, target_field_name, *objs):
             # join_table: name of the m2m link table
-            # source_col_name: the PK colname in join_table for the source object
+            # source_field_name: the PK fieldname in join_table for the source object
-            # target_col_name: the PK colname in join_table for the target object
+            # target_col_name: the PK fieldname in join_table for the target object
             # *objs - objects to add. Either object instances, or primary keys of object instances.
 
             # If there aren't any objects, there is nothing to do.
+            from django.db.models import Model
             if objs:
-                from django.db.models.base import Model
-                # Check that all the objects are of the right type
                 new_ids = set()
                 for obj in objs:
                     if isinstance(obj, self.model):
-                        new_ids.add(obj._get_pk_val())
+                        new_ids.add(obj.pk)
                     elif isinstance(obj, Model):
                         raise TypeError, "'%s' instance expected" % self.model._meta.object_name
                     else:
                         new_ids.add(obj)
-                # Add the newly created or already existing objects to the join table.
+                vals = self.through._default_manager.values_list(target_field_name, flat=True)
-                # First find out which items are already added, to avoid adding them twice
+                vals = vals.filter(**{
-                cursor = connection.cursor()
+                    source_field_name: self._pk_val,
-                cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \
+                    '%s__in' % target_field_name: new_ids,
-                    (target_col_name, self.join_table, source_col_name,
+                })
-                    target_col_name, ",".join(['%s'] * len(new_ids))),
+                vals = set(vals)
-                    [self._pk_val] + list(new_ids))
-                existing_ids = set([row[0] for row in cursor.fetchall()])
 
                 # Add the ones that aren't there already
-                for obj_id in (new_ids - existing_ids):
+                for obj_id in (new_ids - vals):
-                    cursor.execute("INSERT INTO %s (%s, %s) VALUES (%%s, %%s)" % \
+                    self.through._default_manager.create(**{
-                        (self.join_table, source_col_name, target_col_name),
+                        '%s_id' % source_field_name: self._pk_val,
-                        [self._pk_val, obj_id])
+                        '%s_id' % target_field_name: obj_id,
-                transaction.commit_unless_managed()
+                    })
 
-        def _remove_items(self, source_col_name, target_col_name, *objs):
+        def _remove_items(self, source_field_name, target_field_name, *objs):
             # source_col_name: the PK colname in join_table for the source object
             # target_col_name: the PK colname in join_table for the target object
             # *objs - objects to remove
@@ -515,24 +517,20 @@ def create_many_related_manager(superclass, through=False):
                 old_ids = set()
                 for obj in objs:
                     if isinstance(obj, self.model):
-                        old_ids.add(obj._get_pk_val())
+                        old_ids.add(obj.pk)
                     else:
                         old_ids.add(obj)
                 # Remove the specified objects from the join table
-                cursor = connection.cursor()
+                self.through._default_manager.filter(**{
-                cursor.execute("DELETE FROM %s WHERE %s = %%s AND %s IN (%s)" % \
+                    source_field_name: self._pk_val,
-                    (self.join_table, source_col_name,
+                    '%s__in' % target_field_name: old_ids
-                    target_col_name, ",".join(['%s'] * len(old_ids))),
+                }).delete()
-                    [self._pk_val] + list(old_ids))
-                transaction.commit_unless_managed()
 
-        def _clear_items(self, source_col_name):
+        def _clear_items(self, source_field_name):
             # source_col_name: the PK colname in join_table for the source object
-            cursor = connection.cursor()
+            self.through._default_manager.filter(**{
-            cursor.execute("DELETE FROM %s WHERE %s = %%s" % \
+                source_field_name: self._pk_val
-                (self.join_table, source_col_name),
+            }).delete()
-                [self._pk_val])
-            transaction.commit_unless_managed()
 
     return ManyRelatedManager
 
@@ -554,17 +552,15 @@ class ManyRelatedObjectsDescriptor(object):
         # model's default manager.
         rel_model = self.related.model
         superclass = rel_model._default_manager.__class__
-        RelatedManager = create_many_related_manager(superclass, self.related.field.rel.through)
+        RelatedManager = create_many_related_manager(superclass, self.related.field.rel)
 
-        qn = connection.ops.quote_name
         manager = RelatedManager(
             model=rel_model,
             core_filters={'%s__pk' % self.related.field.name: instance._get_pk_val()},
             instance=instance,
             symmetrical=False,
-            join_table=qn(self.related.field.m2m_db_table()),
+            source_field_name=self.related.field.m2m_reverse_field_name(),
-            source_col_name=qn(self.related.field.m2m_reverse_name()),
+            target_field_name=self.related.field.m2m_field_name()
-            target_col_name=qn(self.related.field.m2m_column_name())
         )
 
         return manager
@@ -573,9 +569,9 @@ class ManyRelatedObjectsDescriptor(object):
         if instance is None:
             raise AttributeError, "Manager must be accessed via instance"
 
-        through = getattr(self.related.field.rel, 'through', None)
+        if not self.related.field.rel.through._meta.auto_created:
-        if through is not None:
+            opts = self.related.field.rel.through._meta
-            raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model. Use %s's Manager instead." % through
+            raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model. Use %s.%s's Manager instead." % (opts.app_label, opts.object_name)
 
         manager = self.__get__(instance)
         manager.clear()
@@ -599,17 +595,15 @@ class ReverseManyRelatedObjectsDescriptor(object):
         # model's default manager.
         rel_model=self.field.rel.to
         superclass = rel_model._default_manager.__class__
-        RelatedManager = create_many_related_manager(superclass, self.field.rel.through)
+        RelatedManager = create_many_related_manager(superclass, self.field.rel)
 
-        qn = connection.ops.quote_name
         manager = RelatedManager(
             model=rel_model,
             core_filters={'%s__pk' % self.field.related_query_name(): instance._get_pk_val()},
             instance=instance,
             symmetrical=(self.field.rel.symmetrical and isinstance(instance, rel_model)),
-            join_table=qn(self.field.m2m_db_table()),
+            source_field_name=self.field.m2m_field_name(),
-            source_col_name=qn(self.field.m2m_column_name()),
+            target_field_name=self.field.m2m_reverse_field_name()
-            target_col_name=qn(self.field.m2m_reverse_name())
         )
 
         return manager
@@ -618,9 +612,9 @@ class ReverseManyRelatedObjectsDescriptor(object):
         if instance is None:
             raise AttributeError, "Manager must be accessed via instance"
 
-        through = getattr(self.field.rel, 'through', None)
+        if not self.field.rel.through._meta.auto_created:
-        if through is not None:
+            opts = self.field.rel.through._meta
-            raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model.  Use %s's Manager instead." % through
+            raise AttributeError, "Cannot set values on a ManyToManyField which specifies an intermediary model.  Use %s.%s's Manager instead." % (opts.app_label, opts.object_name)
 
         manager = self.__get__(instance)
         manager.clear()
@@ -642,6 +636,10 @@ class ManyToOneRel(object):
         self.multiple = True
         self.parent_link = parent_link
 
+    def is_hidden(self):
+        "Should the related object be hidden?"
+        return self.related_name and self.related_name[-1] == '+'
+
     def get_related_field(self):
         """
         Returns the Field in the 'to' object to which this relationship is
@@ -673,6 +671,10 @@ class ManyToManyRel(object):
         self.multiple = True
         self.through = through
 
+    def is_hidden(self):
+        "Should the related object be hidden?"
+        return self.related_name and self.related_name[-1] == '+'
+
     def get_related_field(self):
         """
         Returns the field in the to' object to which this relationship is tied
@@ -690,7 +692,6 @@ class ForeignKey(RelatedField, Field):
             assert isinstance(to, basestring), "%s(%r) is invalid. First parameter to ForeignKey must be either a model, a model name, or the string %r" % (self.__class__.__name__, to, RECURSIVE_RELATIONSHIP_CONSTANT)
         else:
             assert not to._meta.abstract, "%s cannot define a relation with abstract class %s" % (self.__class__.__name__, to._meta.object_name)
-            to_field = to_field or to._meta.pk.name
         kwargs['verbose_name'] = kwargs.get('verbose_name', None)
 
         kwargs['rel'] = rel_class(to, to_field,
@@ -743,7 +744,12 @@ class ForeignKey(RelatedField, Field):
         cls._meta.duplicate_targets[self.column] = (target, "o2m")
 
     def contribute_to_related_class(self, cls, related):
+        # Internal FK's - i.e., those with a related name ending with '+' -
+        # don't get a related descriptor.
+        if not self.rel.is_hidden():
             setattr(cls, related.get_accessor_name(), ForeignRelatedObjectsDescriptor(related))
+        if self.rel.field_name is None:
+            self.rel.field_name = cls._meta.pk.name
 
     def formfield(self, **kwargs):
         defaults = {
@@ -790,6 +796,43 @@ class OneToOneField(ForeignKey):
             return None
         return super(OneToOneField, self).formfield(**kwargs)
 
+def create_many_to_many_intermediary_model(field, klass):
+    from django.db import models
+    managed = True
+    if isinstance(field.rel.to, basestring) and field.rel.to != RECURSIVE_RELATIONSHIP_CONSTANT:
+        to = field.rel.to
+        to_model = field.rel.to
+        def set_managed(field, model, cls):
+            field.rel.through._meta.managed = model._meta.managed or cls._meta.managed
+        add_lazy_relation(klass, field, to_model, set_managed)
+    elif isinstance(field.rel.to, basestring):
+        to = klass._meta.object_name
+        to_model = klass
+        managed = klass._meta.managed
+    else:
+        to = field.rel.to._meta.object_name
+        to_model = field.rel.to
+        managed = klass._meta.managed or to_model._meta.managed
+    name = '%s_%s' % (klass._meta.object_name, field.name)
+    if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT or field.rel.to == klass._meta.object_name:
+        from_ = 'from_%s' % to.lower()
+        to = to.lower()
+    else:
+        from_ = klass._meta.object_name.lower()
+        to = to.lower()
+    meta = type('Meta', (object,), {
+        'db_table': field._get_m2m_db_table(klass._meta),
+        'managed': managed,
+        'auto_created': klass,
+        'unique_together': (from_, to)
+    })
+    return type(name, (models.Model,), {
+        'Meta': meta,
+        '__module__': klass.__module__,
+        from_: models.ForeignKey(klass, related_name='%s+' % name),
+        to: models.ForeignKey(to_model, related_name='%s+' % name)
+    })
+
 class ManyToManyField(RelatedField, Field):
     def __init__(self, to, **kwargs):
         try:
@@ -806,10 +849,7 @@ class ManyToManyField(RelatedField, Field):
 
         self.db_table = kwargs.pop('db_table', None)
         if kwargs['rel'].through is not None:
-            self.creates_table = False
             assert self.db_table is None, "Cannot specify a db_table if an intermediary model is used."
-        else:
-            self.creates_table = True
 
         Field.__init__(self, **kwargs)
 
@@ -822,40 +862,30 @@ class ManyToManyField(RelatedField, Field):
     def _get_m2m_db_table(self, opts):
         "Function that can be curried to provide the m2m table name for this relation"
         if self.rel.through is not None:
-            return self.rel.through_model._meta.db_table
+            return self.rel.through._meta.db_table
         elif self.db_table:
             return self.db_table
         else:
             return util.truncate_name('%s_%s' % (opts.db_table, self.name),
                                       connection.ops.max_name_length())
 
-    def _get_m2m_column_name(self, related):
+    def _get_m2m_attr(self, related, attr):
         "Function that can be curried to provide the source column name for the m2m table"
-        try:
+        cache_attr = '_m2m_%s_cache' % attr
-            return self._m2m_column_name_cache
+        if hasattr(self, cache_attr):
-        except:
+            return getattr(self, cache_attr)
-            if self.rel.through is not None:
+        for f in self.rel.through._meta.fields:
-                for f in self.rel.through_model._meta.fields:
             if hasattr(f,'rel') and f.rel and f.rel.to == related.model:
-                        self._m2m_column_name_cache = f.column
+                setattr(self, cache_attr, getattr(f, attr))
-                        break
+                return getattr(self, cache_attr)
-            # If this is an m2m relation to self, avoid the inevitable name clash
-            elif related.model == related.parent_model:
-                self._m2m_column_name_cache = 'from_' + related.model._meta.object_name.lower() + '_id'
-            else:
-                self._m2m_column_name_cache = related.model._meta.object_name.lower() + '_id'
 
-            # Return the newly cached value
+    def _get_m2m_reverse_attr(self, related, attr):
-            return self._m2m_column_name_cache
-
-    def _get_m2m_reverse_name(self, related):
         "Function that can be curried to provide the related column name for the m2m table"
-        try:
+        cache_attr = '_m2m_reverse_%s_cache' % attr
-            return self._m2m_reverse_name_cache
+        if hasattr(self, cache_attr):
-        except:
+            return getattr(self, cache_attr)
-            if self.rel.through is not None:
         found = False
-                for f in self.rel.through_model._meta.fields:
+        for f in self.rel.through._meta.fields:
             if hasattr(f,'rel') and f.rel and f.rel.to == related.parent_model:
                 if related.model == related.parent_model:
                     # If this is an m2m-intermediate to self,
@@ -863,21 +893,14 @@ class ManyToManyField(RelatedField, Field):
                     # the source column. Keep searching for
                     # the second foreign key.
                     if found:
-                                self._m2m_reverse_name_cache = f.column
+                        setattr(self, cache_attr, getattr(f, attr))
                         break
                     else:
                         found = True
                 else:
-                            self._m2m_reverse_name_cache = f.column
+                    setattr(self, cache_attr, getattr(f, attr))
                     break
-            # If this is an m2m relation to self, avoid the inevitable name clash
+        return getattr(self, cache_attr)
-            elif related.model == related.parent_model:
-                self._m2m_reverse_name_cache = 'to_' + related.parent_model._meta.object_name.lower() + '_id'
-            else:
-                self._m2m_reverse_name_cache = related.parent_model._meta.object_name.lower() + '_id'
-
-            # Return the newly cached value
-            return self._m2m_reverse_name_cache
 
     def isValidIDList(self, field_data, all_data):
         "Validates that the value is a valid list of foreign keys"
@@ -919,10 +942,17 @@ class ManyToManyField(RelatedField, Field):
         # specify *what* on my non-reversible relation?!"), so we set it up
         # automatically. The funky name reduces the chance of an accidental
         # clash.
-        if self.rel.symmetrical and self.rel.to == "self" and self.rel.related_name is None:
+        if self.rel.symmetrical and (self.rel.to == "self" or self.rel.to == cls._meta.object_name):
             self.rel.related_name = "%s_rel_+" % name
 
         super(ManyToManyField, self).contribute_to_class(cls, name)
+
+        # The intermediate m2m model is not auto created if:
+        #  1) There is a manually specified intermediate, or
+        #  2) The class owning the m2m field is abstract.
+        if not self.rel.through and not cls._meta.abstract:
+            self.rel.through = create_many_to_many_intermediary_model(self, cls)
+
         # Add the descriptor for the m2m relation
         setattr(cls, self.name, ReverseManyRelatedObjectsDescriptor(self))
 
@@ -933,11 +963,8 @@ class ManyToManyField(RelatedField, Field):
         # work correctly.
         if isinstance(self.rel.through, basestring):
             def resolve_through_model(field, model, cls):
-                field.rel.through_model = model
+                field.rel.through = model
             add_lazy_relation(cls, self, self.rel.through, resolve_through_model)
-        elif self.rel.through:
-            self.rel.through_model = self.rel.through
-            self.rel.through = self.rel.through._meta.object_name
 
         if isinstance(self.rel.to, basestring):
             target = self.rel.to
@@ -946,15 +973,17 @@ class ManyToManyField(RelatedField, Field):
         cls._meta.duplicate_targets[self.column] = (target, "m2m")
 
     def contribute_to_related_class(self, cls, related):
-        # m2m relations to self do not have a ManyRelatedObjectsDescriptor,
+        # Internal M2Ms (i.e., those with a related name ending with '+')
-        # as it would be redundant - unless the field is non-symmetrical.
+        # don't get a related descriptor.
-        if related.model != related.parent_model or not self.rel.symmetrical:
+        if not self.rel.is_hidden():
-            # Add the descriptor for the m2m relation
             setattr(cls, related.get_accessor_name(), ManyRelatedObjectsDescriptor(related))
 
         # Set up the accessors for the column names on the m2m table
-        self.m2m_column_name = curry(self._get_m2m_column_name, related)
+        self.m2m_column_name = curry(self._get_m2m_attr, related, 'column')
-        self.m2m_reverse_name = curry(self._get_m2m_reverse_name, related)
+        self.m2m_reverse_name = curry(self._get_m2m_reverse_attr, related, 'column')
+
+        self.m2m_field_name = curry(self._get_m2m_attr, related, 'name')
+        self.m2m_reverse_field_name = curry(self._get_m2m_reverse_attr, related, 'name')
 
     def set_attributes_from_rel(self):
         pass

django/db/models/loading.py

@@ -131,18 +131,24 @@ class AppCache(object):
         self._populate()
         return self.app_errors
 
-    def get_models(self, app_mod=None):
+    def get_models(self, app_mod=None, include_auto_created=False):
         """
         Given a module containing models, returns a list of the models.
         Otherwise returns a list of all installed models.
+
+        By default, auto-created models (i.e., m2m models without an
+        explicit intermediate table) are not included. However, if you
+        specify include_auto_created=True, they will be.
         """
         self._populate()
         if app_mod:
-            return self.app_models.get(app_mod.__name__.split('.')[-2], SortedDict()).values()
+            model_list = self.app_models.get(app_mod.__name__.split('.')[-2], SortedDict()).values()
         else:
             model_list = []
             for app_entry in self.app_models.itervalues():
                 model_list.extend(app_entry.values())
+        if not include_auto_created:
+            return filter(lambda o: not o._meta.auto_created, model_list)
         return model_list
 
     def get_model(self, app_label, model_name, seed_cache=True):

django/db/models/options.py

@@ -21,7 +21,7 @@ get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|
 DEFAULT_NAMES = ('verbose_name', 'db_table', 'ordering',
                  'unique_together', 'permissions', 'get_latest_by',
                  'order_with_respect_to', 'app_label', 'db_tablespace',
-                 'abstract', 'managed', 'proxy')
+                 'abstract', 'managed', 'proxy', 'auto_created')
 
 class Options(object):
     def __init__(self, meta, app_label=None):
@@ -47,6 +47,7 @@ class Options(object):
         self.proxy_for_model = None
         self.parents = SortedDict()
         self.duplicate_targets = {}
+        self.auto_created = False
 
         # To handle various inheritance situations, we need to track where
         # managers came from (concrete or abstract base classes).
@@ -487,4 +488,3 @@ class Options(object):
         Returns the index of the primary key field in the self.fields list.
         """
         return self.fields.index(self.pk)
-

django/db/models/query.py

@@ -1028,6 +1028,7 @@ def delete_objects(seen_objs):
 
             # Pre-notify all instances to be deleted.
             for pk_val, instance in items:
+                if not cls._meta.auto_created:
                     signals.pre_delete.send(sender=cls, instance=instance)
 
             pk_list = [pk for pk,instance in items]
@@ -1062,6 +1063,7 @@ def delete_objects(seen_objs):
                     if field.rel and field.null and field.rel.to in seen_objs:
                         setattr(instance, field.attname, None)
 
+                if not cls._meta.auto_created:
                     signals.post_delete.send(sender=cls, instance=instance)
                 setattr(instance, cls._meta.pk.attname, None)
 

docs/internals/deprecation.txt

@@ -25,6 +25,9 @@ their deprecation, as per the :ref:`Django deprecation policy
         * ``SMTPConnection``. The 1.2 release deprecated the ``SMTPConnection``
           class in favor of a generic E-mail backend API.
 
+        * The many to many SQL generation functions on the database backends
+          will be removed.  These have been deprecated since the 1.2 release.
+
     * 2.0
         * ``django.views.defaults.shortcut()``. This function has been moved
           to ``django.contrib.contenttypes.views.shortcut()`` as part of the

tests/modeltests/invalid_models/models.py

@@ -182,6 +182,7 @@ class UniqueM2M(models.Model):
     """ Model to test for unique ManyToManyFields, which are invalid. """
     unique_people = models.ManyToManyField( Person, unique=True )
 
+
 model_errors = """invalid_models.fielderrors: "charfield": CharFields require a "max_length" attribute.
 invalid_models.fielderrors: "decimalfield": DecimalFields require a "decimal_places" attribute.
 invalid_models.fielderrors: "decimalfield": DecimalFields require a "max_digits" attribute.

tests/modeltests/m2m_through/models.py

@@ -133,7 +133,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'add'
 >>> rock.members.create(name='Anne')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use m2m_through.Membership's Manager instead.
 
 # Remove has similar complications, and is not provided either.
 >>> rock.members.remove(jim)
@@ -160,7 +160,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'remove'
 >>> rock.members = backup
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through.Membership's Manager instead.
 
 # Let's re-save those instances that we've cleared.
 >>> m1.save()
@@ -184,7 +184,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'add'
 >>> bob.group_set.create(name='Funk')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model. Use m2m_through.Membership's Manager instead.
 
 # Remove has similar complications, and is not provided either.
 >>> jim.group_set.remove(rock)
@@ -209,7 +209,7 @@ AttributeError: 'ManyRelatedManager' object has no attribute 'remove'
 >>> jim.group_set = backup
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through.Membership's Manager instead.
 
 # Let's re-save those instances that we've cleared.
 >>> m1.save()

tests/regressiontests/m2m_through_regress/models.py

@@ -84,22 +84,22 @@ __test__ = {'API_TESTS':"""
 >>> bob.group_set = []
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> roll.members = []
 Traceback (most recent call last):
 ...
-AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot set values on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> rock.members.create(name='Anne')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 >>> bob.group_set.create(name='Funk')
 Traceback (most recent call last):
 ...
-AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use Membership's Manager instead.
+AttributeError: Cannot use create() on a ManyToManyField which specifies an intermediary model.  Use m2m_through_regress.Membership's Manager instead.
 
 # Now test that the intermediate with a relationship outside
 # the current app (i.e., UserMembership) workds

tests/regressiontests/model_inheritance_regress/models.py

@@ -110,6 +110,36 @@ class DerivedM(BaseM):
         return "PK = %d, base_name = %s, derived_name = %s" \
                 % (self.customPK, self.base_name, self.derived_name)
 
+# Check that abstract classes don't get m2m tables autocreated.
+class Person(models.Model):
+    name = models.CharField(max_length=100)
+
+    class Meta:
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+class AbstractEvent(models.Model):
+    name = models.CharField(max_length=100)
+    attendees = models.ManyToManyField(Person, related_name="%(class)s_set")
+
+    class Meta:
+        abstract = True
+        ordering = ('name',)
+
+    def __unicode__(self):
+        return self.name
+
+class BirthdayParty(AbstractEvent):
+    pass
+
+class BachelorParty(AbstractEvent):
+    pass
+
+class MessyBachelorParty(BachelorParty):
+    pass
+
 __test__ = {'API_TESTS':"""
 # Regression for #7350, #7202
 # Check that when you create a Parent object with a specific reference to an
@@ -318,5 +348,41 @@ True
 >>> ParkingLot3._meta.get_ancestor_link(Place).name  # the child->parent link
 "parent"
 
+# Check that many-to-many relations defined on an abstract base class
+# are correctly inherited (and created) on the child class.
+>>> p1 = Person.objects.create(name='Alice')
+>>> p2 = Person.objects.create(name='Bob')
+>>> p3 = Person.objects.create(name='Carol')
+>>> p4 = Person.objects.create(name='Dave')
+
+>>> birthday = BirthdayParty.objects.create(name='Birthday party for Alice')
+>>> birthday.attendees = [p1, p3]
+
+>>> bachelor = BachelorParty.objects.create(name='Bachelor party for Bob')
+>>> bachelor.attendees = [p2, p4]
+
+>>> print p1.birthdayparty_set.all()
+[<BirthdayParty: Birthday party for Alice>]
+
+>>> print p1.bachelorparty_set.all()
+[]
+
+>>> print p2.bachelorparty_set.all()
+[<BachelorParty: Bachelor party for Bob>]
+
+# Check that a subclass of a subclass of an abstract model
+# doesn't get it's own accessor.
+>>> p2.messybachelorparty_set.all()
+Traceback (most recent call last):
+...
+AttributeError: 'Person' object has no attribute 'messybachelorparty_set'
+
+# ... but it does inherit the m2m from it's parent
+>>> messy = MessyBachelorParty.objects.create(name='Bachelor party for Dave')
+>>> messy.attendees = [p4]
+
+>>> p4.bachelorparty_set.all()
+[<BachelorParty: Bachelor party for Bob>, <BachelorParty: Bachelor party for Dave>]
+
 """}
 

tests/regressiontests/serializers_regress/models.py

@@ -105,6 +105,9 @@ class Anchor(models.Model):
 
     data = models.CharField(max_length=30)
 
+    class Meta:
+        ordering = ('id',)
+
 class UniqueAnchor(models.Model):
     """This is a model that can be used as
     something for other models to point at"""
@@ -252,4 +255,3 @@ class InheritBaseModel(BaseModel):
 class ExplicitInheritBaseModel(BaseModel):
     parent = models.OneToOneField(BaseModel)
     child_data = models.IntegerField()
-    

tests/regressiontests/signals_regress/models.py

@@ -0,0 +1,89 @@
+"""
+Testing signals before/after saving and deleting.
+"""
+
+from django.db import models
+
+class Author(models.Model):
+    name = models.CharField(max_length=20)
+
+    def __unicode__(self):
+        return self.name
+
+class Book(models.Model):
+    name = models.CharField(max_length=20)
+    authors = models.ManyToManyField(Author)
+
+    def __unicode__(self):
+        return self.name
+
+def pre_save_test(signal, sender, instance, **kwargs):
+    print 'pre_save signal,', instance
+    if kwargs.get('raw'):
+        print 'Is raw'
+
+def post_save_test(signal, sender, instance, **kwargs):
+    print 'post_save signal,', instance
+    if 'created' in kwargs:
+        if kwargs['created']:
+            print 'Is created'
+        else:
+            print 'Is updated'
+    if kwargs.get('raw'):
+        print 'Is raw'
+
+def pre_delete_test(signal, sender, instance, **kwargs):
+    print 'pre_delete signal,', instance
+    print 'instance.id is not None: %s' % (instance.id != None)
+
+def post_delete_test(signal, sender, instance, **kwargs):
+    print 'post_delete signal,', instance
+    print 'instance.id is not None: %s' % (instance.id != None)
+
+__test__ = {'API_TESTS':"""
+
+# Save up the number of connected signals so that we can check at the end
+# that all the signals we register get properly unregistered (#9989)
+>>> pre_signals = (len(models.signals.pre_save.receivers),
+...                len(models.signals.post_save.receivers),
+...                len(models.signals.pre_delete.receivers),
+...                len(models.signals.post_delete.receivers))
+
+>>> models.signals.pre_save.connect(pre_save_test)
+>>> models.signals.post_save.connect(post_save_test)
+>>> models.signals.pre_delete.connect(pre_delete_test)
+>>> models.signals.post_delete.connect(post_delete_test)
+
+>>> a1 = Author(name='Neal Stephenson')
+>>> a1.save()
+pre_save signal, Neal Stephenson
+post_save signal, Neal Stephenson
+Is created
+
+>>> b1 = Book(name='Snow Crash')
+>>> b1.save()
+pre_save signal, Snow Crash
+post_save signal, Snow Crash
+Is created
+
+# Assigning to m2m shouldn't generate an m2m signal
+>>> b1.authors = [a1]
+
+# Removing an author from an m2m shouldn't generate an m2m signal
+>>> b1.authors = []
+
+>>> models.signals.post_delete.disconnect(post_delete_test)
+>>> models.signals.pre_delete.disconnect(pre_delete_test)
+>>> models.signals.post_save.disconnect(post_save_test)
+>>> models.signals.pre_save.disconnect(pre_save_test)
+
+# Check that all our signals got disconnected properly.
+>>> post_signals = (len(models.signals.pre_save.receivers),
+...                 len(models.signals.post_save.receivers),
+...                 len(models.signals.pre_delete.receivers),
+...                 len(models.signals.post_delete.receivers))
+
+>>> pre_signals == post_signals
+True
+
+"""}