diff --git a/django/contrib/auth/models.py b/django/contrib/auth/models.py index 053761cb56..b20a2caf17 100644 --- a/django/contrib/auth/models.py +++ b/django/contrib/auth/models.py @@ -47,6 +47,13 @@ def check_password(raw_password, enc_password): class SiteProfileNotAvailable(Exception): pass +class PermissionManager(models.Manager): + def get_by_natural_key(self, codename, app_label, model): + return self.get( + codename=codename, + content_type=ContentType.objects.get_by_natural_key(app_label, model) + ) + class Permission(models.Model): """The permissions system provides a way to assign permissions to specific users and groups of users. @@ -63,6 +70,7 @@ class Permission(models.Model): name = models.CharField(_('name'), max_length=50) content_type = models.ForeignKey(ContentType) codename = models.CharField(_('codename'), max_length=100) + objects = PermissionManager() class Meta: verbose_name = _('permission') @@ -76,6 +84,10 @@ class Permission(models.Model): unicode(self.content_type), unicode(self.name)) + def natural_key(self): + return (self.codename,) + self.content_type.natural_key() + natural_key.dependencies = ['contenttypes.contenttype'] + class Group(models.Model): """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups. diff --git a/django/contrib/contenttypes/models.py b/django/contrib/contenttypes/models.py index def7ce6986..69d0806385 100644 --- a/django/contrib/contenttypes/models.py +++ b/django/contrib/contenttypes/models.py @@ -8,6 +8,13 @@ class ContentTypeManager(models.Manager): # This cache is shared by all the get_for_* methods. _cache = {} + def get_by_natural_key(self, app_label, model): + try: + ct = self.__class__._cache[(app_label, model)] + except KeyError: + ct = self.get(app_label=app_label, model=model) + return ct + def get_for_model(self, model): """ Returns the ContentType object for a given model, creating the @@ -93,3 +100,6 @@ class ContentType(models.Model): so code that calls this method should catch it. """ return self.model_class()._default_manager.get(**kwargs) + + def natural_key(self): + return (self.app_label, self.model) diff --git a/django/core/management/commands/dumpdata.py b/django/core/management/commands/dumpdata.py index 7e05b89160..14b9b00dfd 100644 --- a/django/core/management/commands/dumpdata.py +++ b/django/core/management/commands/dumpdata.py @@ -13,6 +13,8 @@ class Command(BaseCommand): help='Specifies the indent level to use when pretty-printing output'), make_option('-e', '--exclude', dest='exclude',action='append', default=[], help='App to exclude (use multiple --exclude to exclude multiple apps).'), + make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False, + help='Use natural keys if they are available.'), ) help = 'Output the contents of the database as a fixture of the given format.' args = '[appname ...]' @@ -24,6 +26,7 @@ class Command(BaseCommand): indent = options.get('indent',None) exclude = options.get('exclude',[]) show_traceback = options.get('traceback', False) + use_natural_keys = options.get('use_natural_keys', False) excluded_apps = [get_app(app_label) for app_label in exclude] @@ -67,18 +70,86 @@ class Command(BaseCommand): except KeyError: raise CommandError("Unknown serialization format: %s" % format) + # Now collate the objects to be serialized. objects = [] - for app, model_list in app_list.items(): - if model_list is None: - model_list = get_models(app) - - for model in model_list: - if not model._meta.proxy: - objects.extend(model._default_manager.all()) + for model in sort_dependencies(app_list.items()): + if not model._meta.proxy: + objects.extend(model._default_manager.all()) try: - return serializers.serialize(format, objects, indent=indent) + return serializers.serialize(format, objects, indent=indent, + use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e) + +def sort_dependencies(app_list): + """Sort a list of app,modellist pairs into a single list of models. + + The single list of models is sorted so that any model with a natural key + is serialized before a normal model, and any model with a natural key + dependency has it's dependencies serialized first. + """ + from django.db.models import get_model, get_models + # Process the list of models, and get the list of dependencies + model_dependencies = [] + models = set() + for app, model_list in app_list: + if model_list is None: + model_list = get_models(app) + + for model in model_list: + models.add(model) + # Add any explicitly defined dependencies + if hasattr(model, 'natural_key'): + deps = getattr(model.natural_key, 'dependencies', []) + if deps: + deps = [get_model(*d.split('.')) for d in deps] + else: + deps = [] + + # Now add a dependency for any FK or M2M relation with + # a model that defines a natural key + for field in model._meta.fields: + if hasattr(field.rel, 'to'): + rel_model = field.rel.to + if hasattr(rel_model, 'natural_key'): + deps.append(rel_model) + for field in model._meta.many_to_many: + rel_model = field.rel.to + if hasattr(rel_model, 'natural_key'): + deps.append(rel_model) + model_dependencies.append((model, deps)) + + model_dependencies.reverse() + # Now sort the models to ensure that dependencies are met. This + # is done by repeatedly iterating over the input list of models. + # If all the dependencies of a given model are in the final list, + # that model is promoted to the end of the final list. This process + # continues until the input list is empty, or we do a full iteration + # over the input models without promoting a model to the final list. + # If we do a full iteration without a promotion, that means there are + # circular dependencies in the list. + model_list = [] + while model_dependencies: + skipped = [] + changed = False + while model_dependencies: + model, deps = model_dependencies.pop() + if all((d not in models or d in model_list) for d in deps): + # If all of the models in the dependency list are either already + # on the final model list, or not on the original serialization list, + # then we've found another model with all it's dependencies satisfied. + model_list.append(model) + changed = True + else: + skipped.append((model, deps)) + if not changed: + raise CommandError("Can't resolve dependencies for %s in serialized app list." % + ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name) + for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__)) + ) + model_dependencies = skipped + + return model_list diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py index 22de2d70d0..eda998e0f8 100644 --- a/django/core/serializers/base.py +++ b/django/core/serializers/base.py @@ -33,6 +33,7 @@ class Serializer(object): self.stream = options.get("stream", StringIO()) self.selected_fields = options.get("fields") + self.use_natural_keys = options.get("use_natural_keys", False) self.start_serialization() for obj in queryset: diff --git a/django/core/serializers/json.py b/django/core/serializers/json.py index 97e5bc9b26..d5872fefc3 100644 --- a/django/core/serializers/json.py +++ b/django/core/serializers/json.py @@ -24,6 +24,7 @@ class Serializer(PythonSerializer): def end_serialization(self): self.options.pop('stream', None) self.options.pop('fields', None) + self.options.pop('use_natural_keys', None) simplejson.dump(self.objects, self.stream, cls=DjangoJSONEncoder, **self.options) def getvalue(self): diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py index 7b77804009..c6c2457258 100644 --- a/django/core/serializers/python.py +++ b/django/core/serializers/python.py @@ -47,17 +47,24 @@ class Serializer(base.Serializer): def handle_fk_field(self, obj, field): related = getattr(obj, field.name) if related is not None: - if field.rel.field_name == related._meta.pk.name: - # Related to remote object via primary key - related = related._get_pk_val() + if self.use_natural_keys and hasattr(related, 'natural_key'): + related = related.natural_key() else: - # Related to remote object via other field - related = getattr(related, field.rel.field_name) - self._current[field.name] = smart_unicode(related, strings_only=True) + if field.rel.field_name == related._meta.pk.name: + # Related to remote object via primary key + related = related._get_pk_val() + else: + # Related to remote object via other field + related = smart_unicode(getattr(related, field.rel.field_name), strings_only=True) + self._current[field.name] = related def handle_m2m_field(self, obj, field): if field.rel.through._meta.auto_created: - self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True) + if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'): + m2m_value = lambda value: value.natural_key() + else: + m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True) + self._current[field.name] = [m2m_value(related) for related in getattr(obj, field.name).iterator()] def getvalue(self): @@ -86,13 +93,28 @@ def Deserializer(object_list, **options): # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): - m2m_convert = field.rel.to._meta.pk.to_python - m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value] + if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): + def m2m_convert(value): + if hasattr(value, '__iter__'): + return field.rel.to._default_manager.get_by_natural_key(*value).pk + else: + return smart_unicode(field.rel.to._meta.pk.to_python(value)) + else: + m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) + m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: - data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) + if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): + if hasattr(field_value, '__iter__'): + obj = field.rel.to._default_manager.get_by_natural_key(*field_value) + value = getattr(obj, field.rel.field_name) + else: + value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) + data[field.attname] = value + else: + data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None diff --git a/django/core/serializers/pyyaml.py b/django/core/serializers/pyyaml.py index 34f8118d38..7a302e615e 100644 --- a/django/core/serializers/pyyaml.py +++ b/django/core/serializers/pyyaml.py @@ -26,9 +26,9 @@ class Serializer(PythonSerializer): """ Convert a queryset to YAML. """ - + internal_use_only = False - + def handle_field(self, obj, field): # A nasty special case: base YAML doesn't support serialization of time # types (as opposed to dates or datetimes, which it does support). Since @@ -40,10 +40,11 @@ class Serializer(PythonSerializer): self._current[field.name] = str(getattr(obj, field.name)) else: super(Serializer, self).handle_field(obj, field) - + def end_serialization(self): self.options.pop('stream', None) self.options.pop('fields', None) + self.options.pop('use_natural_keys', None) yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options) def getvalue(self): diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index 4cde0b039d..7e4f9a2eb8 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -81,13 +81,22 @@ class Serializer(base.Serializer): self._start_relational_field(field) related = getattr(obj, field.name) if related is not None: - if field.rel.field_name == related._meta.pk.name: - # Related to remote object via primary key - related = related._get_pk_val() + if self.use_natural_keys and hasattr(related, 'natural_key'): + # If related object has a natural key, use it + related = related.natural_key() + # Iterable natural keys are rolled out as subelements + for key_value in related: + self.xml.startElement("natural", {}) + self.xml.characters(smart_unicode(key_value)) + self.xml.endElement("natural") else: - # Related to remote object via other field - related = getattr(related, field.rel.field_name) - self.xml.characters(smart_unicode(related)) + if field.rel.field_name == related._meta.pk.name: + # Related to remote object via primary key + related = related._get_pk_val() + else: + # Related to remote object via other field + related = getattr(related, field.rel.field_name) + self.xml.characters(smart_unicode(related)) else: self.xml.addQuickElement("None") self.xml.endElement("field") @@ -100,8 +109,25 @@ class Serializer(base.Serializer): """ if field.rel.through._meta.auto_created: self._start_relational_field(field) + if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'): + # If the objects in the m2m have a natural key, use it + def handle_m2m(value): + natural = value.natural_key() + # Iterable natural keys are rolled out as subelements + self.xml.startElement("object", {}) + for key_value in natural: + self.xml.startElement("natural", {}) + self.xml.characters(smart_unicode(key_value)) + self.xml.endElement("natural") + self.xml.endElement("object") + else: + def handle_m2m(value): + self.xml.addQuickElement("object", attrs={ + 'pk' : smart_unicode(value._get_pk_val()) + }) for relobj in getattr(obj, field.name).iterator(): - self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())}) + handle_m2m(relobj) + self.xml.endElement("field") def _start_relational_field(self, field): @@ -187,16 +213,40 @@ class Deserializer(base.Deserializer): if node.getElementsByTagName('None'): return None else: - return field.rel.to._meta.get_field(field.rel.field_name).to_python( - getInnerText(node).strip()) + if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): + keys = node.getElementsByTagName('natural') + if keys: + # If there are 'natural' subelements, it must be a natural key + field_value = [getInnerText(k).strip() for k in keys] + obj = field.rel.to._default_manager.get_by_natural_key(*field_value) + obj_pk = getattr(obj, field.rel.field_name) + else: + # Otherwise, treat like a normal PK + field_value = getInnerText(node).strip() + obj_pk = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) + return obj_pk + else: + field_value = getInnerText(node).strip() + return field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) def _handle_m2m_field_node(self, node, field): """ Handle a node for a ManyToManyField. """ - return [field.rel.to._meta.pk.to_python( - c.getAttribute("pk")) - for c in node.getElementsByTagName("object")] + if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): + def m2m_convert(n): + keys = n.getElementsByTagName('natural') + if keys: + # If there are 'natural' subelements, it must be a natural key + field_value = [getInnerText(k).strip() for k in keys] + obj_pk = field.rel.to._default_manager.get_by_natural_key(*field_value).pk + else: + # Otherwise, treat like a normal PK value. + obj_pk = field.rel.to._meta.pk.to_python(n.getAttribute('pk')) + return obj_pk + else: + m2m_convert = lambda n: field.rel.to._meta.pk.to_python(n.getAttribute('pk')) + return [m2m_convert(c) for c in node.getElementsByTagName("object")] def _get_model_from_node(self, node, attr): """ diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index dc18f350c4..2b7af17f1e 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -234,6 +234,17 @@ name to ``dumpdata``, the dumped output will be restricted to that model, rather than the entire application. You can also mix application names and model names. +.. django-admin-option:: --natural + +.. versionadded:: 1.2 + +Use :ref:`natural keys ` to represent +any foreign key and many-to-many relationship with a model that provides +a natural key definition. If you are dumping ``contrib.auth`` ``Permission`` +objects or ``contrib.contenttypes`` ``ContentType`` objects, you should +probably be using this flag. + + flush ----- @@ -701,7 +712,7 @@ information. .. versionadded:: 1.2 -Use the ``--failfast`` option to stop running tests and report the failure +Use the ``--failfast`` option to stop running tests and report the failure immediately after a test fails. testserver diff --git a/docs/releases/1.2.txt b/docs/releases/1.2.txt index 62f536d733..9e8816e148 100644 --- a/docs/releases/1.2.txt +++ b/docs/releases/1.2.txt @@ -267,3 +267,13 @@ include %}`` tags). As a side effect, it is now much easier to support non-Django template languages. For more details, see the :ref:`notes on supporting non-Django template languages`. + +Natural keys in fixtures +------------------------ + +Fixtures can refer to remote objects using +:ref:`topics-serialization-natural-keys`. This lookup scheme is an +alternative to the normal primary-key based object references in a +fixture, improving readability, and resolving problems referring to +objects whose primary key value may not be predictable or known. + diff --git a/docs/topics/serialization.txt b/docs/topics/serialization.txt index 751ff27b79..b33e4effe3 100644 --- a/docs/topics/serialization.txt +++ b/docs/topics/serialization.txt @@ -154,10 +154,10 @@ to install third-party Python modules: .. _PyYAML: http://www.pyyaml.org/ Notes for specific serialization formats ----------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ json -~~~~ +^^^^ If you're using UTF-8 (or any other non-ASCII encoding) data with the JSON serializer, you must pass ``ensure_ascii=False`` as a parameter to the @@ -191,3 +191,191 @@ them. Something like this will work:: .. _special encoder: http://svn.red-bean.com/bob/simplejson/tags/simplejson-1.7/docs/index.html +.. _topics-serialization-natural-keys: + +Natural keys +------------ + +The default serialization strategy for foreign keys and many-to-many +relations is to serialize the value of the primary key(s) of the +objects in the relation. This strategy works well for most types of +object, but it can cause difficulty in some circumstances. + +Consider the case of a list of objects that have foreign key on +:class:`ContentType`. If you're going to serialize an object that +refers to a content type, you need to have a way to refer to that +content type. Content Types are automatically created by Django as +part of the database synchronization process, so you don't need to +include content types in a fixture or other serialized data. As a +result, the primary key of any given content type isn't easy to +predict - it will depend on how and when :djadmin:`syncdb` was +executed to create the content types. + +There is also the matter of convenience. An integer id isn't always +the most convenient way to refer to an object; sometimes, a +more natural reference would be helpful. + +Deserialization of natural keys +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is for these reasons that Django provides `natural keys`. A natural +key is a tuple of values that can be used to uniquely identify an +object instance without using the primary key value. + +Consider the following two models:: + + from django.db import models + + class Person(models.Model): + first_name = models.CharField(max_length=100) + last_name = models.CharField(max_length=100) + + birthdate = models.DateField() + + class Book(models.Model): + name = models.CharField(max_length=100) + author = models.ForeignKey(Person) + +Ordinarily, serialized data for ``Book`` would use an integer to refer to +the author. For example, in JSON, a Book might be serialized as:: + + ... + { + "pk": 1, + "model": "store.book", + "fields": { + "name": "Mostly Harmless", + "author": 42 + } + } + ... + +This isn't a particularly natural way to refer to an author. It +requires that you know the primary key value for the author; it also +requires that this primary key value is stable and predictable. + +However, if we add natural key handling to Person, the fixture becomes +much more humane. To add natural key handling, you define a default +Manager for Person with a ``get_by_natural_key()`` method. In the case +of a Person, a good natural key might be the pair of first and last +name:: + + from django.db import models + + class PersonManager(models.Manager): + def get_by_natural_key(self, first_name, last_name): + return self.filter(first_name=first_name, last_name=last_name) + + class Person(models.Model): + objects = PersonManager() + + first_name = models.CharField(max_length=100) + last_name = models.CharField(max_length=100) + + birthdate = models.DateField() + +Now books can use that natural key to refer to ``Person`` objects:: + + ... + { + "pk": 1, + "model": "store.book", + "fields": { + "name": "Mostly Harmless", + "author": ["Douglas", "Adams"] + } + } + ... + +When you try to load this serialized data, Django will use the +``get_by_natural_key()`` method to resolve ``["Douglas", "Adams"]`` +into the primary key of an actual ``Person`` object. + +Serialization of natural keys +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +So how do you get Django to emit a natural key when serializing an object? +Firstly, you need to add another method -- this time to the model itself:: + + class Person(models.Model): + objects = PersonManager() + + first_name = models.CharField(max_length=100) + last_name = models.CharField(max_length=100) + + birthdate = models.DateField() + + def natural_key(self): + return (self.first_name, self.last_name) + +Then, when you call ``serializers.serialize()``, you provide a +``use_natural_keys=True`` argument:: + + >>> serializers.serialize([book1, book2], format='json', indent=2, use_natural_keys=True) + +When ``use_natural_keys=True`` is specified, Django will use the +``natural_key()`` method to serialize any reference to objects of the +type that defines the method. + +If you are using :djadmin:`dumpdata` to generate serialized data, you +use the `--natural` command line flag to generate natural keys. + +.. note:: + + You don't need to define both ``natural_key()`` and + ``get_by_natural_key()``. If you don't want Django to output + natural keys during serialization, but you want to retain the + ability to load natural keys, then you can opt to not implement + the ``natural_key()`` method. + + Conversely, if (for some strange reason) you want Django to output + natural keys during serialization, but *not* be able to load those + key values, just don't define the ``get_by_natural_key()`` method. + +Dependencies during serialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since natural keys rely on database lookups to resolve references, it +is important that data exists before it is referenced. You can't make +a `forward reference` with natural keys - the data you are referencing +must exist before you include a natural key reference to that data. + +To accommodate this limitation, calls to :djadmin:`dumpdata` that use +the :djadminopt:`--natural` optionwill serialize any model with a +``natural_key()`` method before it serializes normal key objects. + +However, this may not always be enough. If your natural key refers to +another object (by using a foreign key or natural key to another object +as part of a natural key), then you need to be able to ensure that +the objects on which a natural key depends occur in the serialized data +before the natural key requires them. + +To control this ordering, you can define dependencies on your +``natural_key()`` methods. You do this by setting a ``dependencies`` +attribute on the ``natural_key()`` method itself. + +For example, consider the ``Permission`` model in ``contrib.auth``. +The following is a simplified version of the ``Permission`` model:: + + class Permission(models.Model): + name = models.CharField(max_length=50) + content_type = models.ForeignKey(ContentType) + codename = models.CharField(max_length=100) + # ... + def natural_key(self): + return (self.codename,) + self.content_type.natural_key() + +The natural key for a ``Permission`` is a combination of the codename for the +``Permission``, and the ``ContentType`` to which the ``Permission`` applies. This means +that ``ContentType`` must be serialized before ``Permission``. To define this +dependency, we add one extra line:: + + class Permission(models.Model): + # ... + def natural_key(self): + return (self.codename,) + self.content_type.natural_key() + natural_key.dependencies = ['contenttypes.contenttype'] + +This definition ensures that ``ContentType`` models are serialized before +``Permission`` models. In turn, any object referencing ``Permission`` will +be serialized after both ``ContentType`` and ``Permission``. diff --git a/tests/modeltests/fixtures/fixtures/fixture6.json b/tests/modeltests/fixtures/fixtures/fixture6.json new file mode 100644 index 0000000000..60e4733c71 --- /dev/null +++ b/tests/modeltests/fixtures/fixtures/fixture6.json @@ -0,0 +1,41 @@ +[ + { + "pk": "1", + "model": "fixtures.tag", + "fields": { + "name": "copyright", + "tagged_type": ["fixtures", "article"], + "tagged_id": "3" + } + }, + { + "pk": "2", + "model": "fixtures.tag", + "fields": { + "name": "law", + "tagged_type": ["fixtures", "article"], + "tagged_id": "3" + } + }, + { + "pk": "1", + "model": "fixtures.person", + "fields": { + "name": "Django Reinhardt" + } + }, + { + "pk": "2", + "model": "fixtures.person", + "fields": { + "name": "Stephane Grappelli" + } + }, + { + "pk": "3", + "model": "fixtures.person", + "fields": { + "name": "Prince" + } + } +] diff --git a/tests/modeltests/fixtures/fixtures/fixture7.xml b/tests/modeltests/fixtures/fixtures/fixture7.xml new file mode 100644 index 0000000000..547cba10d2 --- /dev/null +++ b/tests/modeltests/fixtures/fixtures/fixture7.xml @@ -0,0 +1,27 @@ + + + + legal + + fixtures + article + + 3 + + + django + + fixtures + article + + 4 + + + world domination + + fixtures + article + + 4 + + diff --git a/tests/modeltests/fixtures/fixtures/fixture8.json b/tests/modeltests/fixtures/fixtures/fixture8.json new file mode 100644 index 0000000000..bc113aa00e --- /dev/null +++ b/tests/modeltests/fixtures/fixtures/fixture8.json @@ -0,0 +1,32 @@ +[ + { + "pk": "1", + "model": "fixtures.visa", + "fields": { + "person": ["Django Reinhardt"], + "permissions": [ + ["add_user", "auth", "user"], + ["change_user", "auth", "user"], + ["delete_user", "auth", "user"] + ] + } + }, + { + "pk": "2", + "model": "fixtures.visa", + "fields": { + "person": ["Stephane Grappelli"], + "permissions": [ + ["add_user", "auth", "user"] + ] + } + }, + { + "pk": "3", + "model": "fixtures.visa", + "fields": { + "person": ["Prince"], + "permissions": [] + } + } +] diff --git a/tests/modeltests/fixtures/fixtures/fixture9.xml b/tests/modeltests/fixtures/fixtures/fixture9.xml new file mode 100644 index 0000000000..100f63d106 --- /dev/null +++ b/tests/modeltests/fixtures/fixtures/fixture9.xml @@ -0,0 +1,48 @@ + + + + + Stephane Grappelli + + + + add_user + auth + user + + + delete_user + auth + user + + + + + + Artist formerly known as "Prince" + + + + + Artist formerly known as "Prince" + + + + change_user + auth + user + + + + + Music for all ages + + + Django Reinhardt + + + Artist formerly known as "Prince" + + + + diff --git a/tests/modeltests/fixtures/models.py b/tests/modeltests/fixtures/models.py index 75ff99867a..bb02cd921d 100644 --- a/tests/modeltests/fixtures/models.py +++ b/tests/modeltests/fixtures/models.py @@ -8,6 +8,9 @@ in the application directory, on in one of the directories named in the ``FIXTURE_DIRS`` setting. """ +from django.contrib.auth.models import Permission +from django.contrib.contenttypes import generic +from django.contrib.contenttypes.models import ContentType from django.db import models from django.conf import settings @@ -31,6 +34,62 @@ class Article(models.Model): class Meta: ordering = ('-pub_date', 'headline') +class Blog(models.Model): + name = models.CharField(max_length=100) + featured = models.ForeignKey(Article, related_name='fixtures_featured_set') + articles = models.ManyToManyField(Article, blank=True, + related_name='fixtures_articles_set') + + def __unicode__(self): + return self.name + + +class Tag(models.Model): + name = models.CharField(max_length=100) + tagged_type = models.ForeignKey(ContentType, related_name="fixtures_tag_set") + tagged_id = models.PositiveIntegerField(default=0) + tagged = generic.GenericForeignKey(ct_field='tagged_type', + fk_field='tagged_id') + + def __unicode__(self): + return '<%s: %s> tagged "%s"' % (self.tagged.__class__.__name__, + self.tagged, self.name) + +class PersonManager(models.Manager): + def get_by_natural_key(self, name): + return self.get(name=name) + +class Person(models.Model): + objects = PersonManager() + name = models.CharField(max_length=100) + def __unicode__(self): + return self.name + + class Meta: + ordering = ('name',) + + def natural_key(self): + return (self.name,) + +class Visa(models.Model): + person = models.ForeignKey(Person) + permissions = models.ManyToManyField(Permission, blank=True) + + def __unicode__(self): + return '%s %s' % (self.person.name, + ', '.join(p.name for p in self.permissions.all())) + +class Book(models.Model): + name = models.CharField(max_length=100) + authors = models.ManyToManyField(Person) + + def __unicode__(self): + return '%s by %s' % (self.name, + ' and '.join(a.name for a in self.authors.all())) + + class Meta: + ordering = ('name',) + __test__ = {'API_TESTS': """ >>> from django.core import management >>> from django.db.models import get_app @@ -90,12 +149,53 @@ __test__ = {'API_TESTS': """ >>> Article.objects.all() [, , , , ] +# Load fixture 6, JSON file with dynamic ContentType fields. Testing ManyToOne. +>>> management.call_command('loaddata', 'fixture6.json', verbosity=0) +>>> Tag.objects.all() +[ tagged "copyright">, tagged "law">] + +# Load fixture 7, XML file with dynamic ContentType fields. Testing ManyToOne. +>>> management.call_command('loaddata', 'fixture7.xml', verbosity=0) +>>> Tag.objects.all() +[ tagged "copyright">, tagged "legal">, tagged "django">, tagged "world domination">] + +# Load fixture 8, JSON file with dynamic Permission fields. Testing ManyToMany. +>>> management.call_command('loaddata', 'fixture8.json', verbosity=0) +>>> Visa.objects.all() +[, , ] + +# Load fixture 9, XML file with dynamic Permission fields. Testing ManyToMany. +>>> management.call_command('loaddata', 'fixture9.xml', verbosity=0) +>>> Visa.objects.all() +[, , ] + +>>> Book.objects.all() +[] + # Load a fixture that doesn't exist >>> management.call_command('loaddata', 'unknown.json', verbosity=0) # object list is unaffected >>> Article.objects.all() [, , , , ] + +# By default, you get raw keys on dumpdata +>>> management.call_command('dumpdata', 'fixtures.book', format='json') +[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [3, 1]}}] + +# But you can get natural keys if you ask for them and they are available +>>> management.call_command('dumpdata', 'fixtures.book', format='json', use_natural_keys=True) +[{"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}] + +# Dump the current contents of the database as a JSON fixture +>>> management.call_command('dumpdata', 'fixtures', format='json', use_natural_keys=True) +[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 5, "model": "fixtures.article", "fields": {"headline": "XML identified as leading cause of cancer", "pub_date": "2006-06-16 16:00:00"}}, {"pk": 4, "model": "fixtures.article", "fields": {"headline": "Django conquers world!", "pub_date": "2006-06-16 15:00:00"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Copyright is fine the way it is", "pub_date": "2006-06-16 14:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker on TV is great!", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "legal", "tagged_id": 3}}, {"pk": 3, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "django", "tagged_id": 4}}, {"pk": 4, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "world domination", "tagged_id": 4}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Artist formerly known as \\"Prince\\""}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}, {"pk": 1, "model": "fixtures.visa", "fields": {"person": ["Django Reinhardt"], "permissions": [["add_user", "auth", "user"], ["change_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 2, "model": "fixtures.visa", "fields": {"person": ["Stephane Grappelli"], "permissions": [["add_user", "auth", "user"], ["delete_user", "auth", "user"]]}}, {"pk": 3, "model": "fixtures.visa", "fields": {"person": ["Artist formerly known as \\"Prince\\""], "permissions": [["change_user", "auth", "user"]]}}, {"pk": 1, "model": "fixtures.book", "fields": {"name": "Music for all ages", "authors": [["Artist formerly known as \\"Prince\\""], ["Django Reinhardt"]]}}] + +# Dump the current contents of the database as an XML fixture +>>> management.call_command('dumpdata', 'fixtures', format='xml', use_natural_keys=True) + +News StoriesLatest news storiesXML identified as leading cause of cancer2006-06-16 16:00:00Django conquers world!2006-06-16 15:00:00Copyright is fine the way it is2006-06-16 14:00:00Poker on TV is great!2006-06-16 11:00:00Python program becomes self aware2006-06-16 11:00:00copyrightfixturesarticle3legalfixturesarticle3djangofixturesarticle4world dominationfixturesarticle4Artist formerly known as "Prince"Django ReinhardtStephane GrappelliDjango Reinhardtadd_userauthuserchange_userauthuserdelete_userauthuserStephane Grappelliadd_userauthuserdelete_userauthuserArtist formerly known as "Prince"change_userauthuserMusic for all agesArtist formerly known as "Prince"Django Reinhardt + """} # Database flushing does not work on MySQL with the default storage engine @@ -159,6 +259,25 @@ Multiple fixtures named 'fixture2' in '...fixtures'. Aborting. >>> management.call_command('loaddata', 'fixture5', verbosity=0) # doctest: +ELLIPSIS Multiple fixtures named 'fixture5' in '...fixtures'. Aborting. +>>> management.call_command('flush', verbosity=0, interactive=False) + +# Load back in fixture 1, we need the articles from it +>>> management.call_command('loaddata', 'fixture1', verbosity=0) + +# Try to load fixture 6 using format discovery +>>> management.call_command('loaddata', 'fixture6', verbosity=0) +>>> Tag.objects.all() +[ tagged "copyright">, tagged "law">] + +# Dump the current contents of the database as a JSON fixture +>>> management.call_command('dumpdata', 'fixtures', format='json', use_natural_keys=True) +[{"pk": 1, "model": "fixtures.category", "fields": {"description": "Latest news stories", "title": "News Stories"}}, {"pk": 3, "model": "fixtures.article", "fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16 13:00:00"}}, {"pk": 2, "model": "fixtures.article", "fields": {"headline": "Poker has no place on ESPN", "pub_date": "2006-06-16 12:00:00"}}, {"pk": 1, "model": "fixtures.article", "fields": {"headline": "Python program becomes self aware", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 1, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "copyright", "tagged_id": 3}}, {"pk": 2, "model": "fixtures.tag", "fields": {"tagged_type": ["fixtures", "article"], "name": "law", "tagged_id": 3}}, {"pk": 1, "model": "fixtures.person", "fields": {"name": "Django Reinhardt"}}, {"pk": 3, "model": "fixtures.person", "fields": {"name": "Prince"}}, {"pk": 2, "model": "fixtures.person", "fields": {"name": "Stephane Grappelli"}}] + +# Dump the current contents of the database as an XML fixture +>>> management.call_command('dumpdata', 'fixtures', format='xml', use_natural_keys=True) + +News StoriesLatest news storiesTime to reform copyright2006-06-16 13:00:00Poker has no place on ESPN2006-06-16 12:00:00Python program becomes self aware2006-06-16 11:00:00copyrightfixturesarticle3lawfixturesarticle3Django ReinhardtPrinceStephane Grappelli + """ from django.test import TestCase diff --git a/tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json b/tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json new file mode 100644 index 0000000000..fe50c653cc --- /dev/null +++ b/tests/regressiontests/fixtures_regress/fixtures/forward_ref_lookup.json @@ -0,0 +1,32 @@ +[ + { + "pk": "4", + "model": "fixtures_regress.person", + "fields": { + "name": "Neal Stephenson" + } + }, + { + "pk": "2", + "model": "fixtures_regress.store", + "fields": { + "name": "Amazon" + } + }, + { + "pk": "3", + "model": "fixtures_regress.store", + "fields": { + "name": "Borders" + } + }, + { + "pk": 1, + "model": "fixtures_regress.book", + "fields": { + "name": "Cryptonomicon", + "author": ["Neal Stephenson"], + "stores": [["Amazon"], ["Borders"]] + } + } +] \ No newline at end of file diff --git a/tests/regressiontests/fixtures_regress/fixtures/non_natural_1.json b/tests/regressiontests/fixtures_regress/fixtures/non_natural_1.json new file mode 100644 index 0000000000..4bce792e35 --- /dev/null +++ b/tests/regressiontests/fixtures_regress/fixtures/non_natural_1.json @@ -0,0 +1,25 @@ +[ + { + "pk": 12, + "model": "fixtures_regress.person", + "fields": { + "name": "Greg Egan" + } + }, + { + "pk": 11, + "model": "fixtures_regress.store", + "fields": { + "name": "Angus and Robertson" + } + }, + { + "pk": 10, + "model": "fixtures_regress.book", + "fields": { + "name": "Permutation City", + "author": 12, + "stores": [11] + } + } +] \ No newline at end of file diff --git a/tests/regressiontests/fixtures_regress/fixtures/non_natural_2.xml b/tests/regressiontests/fixtures_regress/fixtures/non_natural_2.xml new file mode 100644 index 0000000000..280ad3758b --- /dev/null +++ b/tests/regressiontests/fixtures_regress/fixtures/non_natural_2.xml @@ -0,0 +1,16 @@ + + + + Orson Scott Card + + + Collins Bookstore + + + Ender's Game + 22 + + + + + \ No newline at end of file diff --git a/tests/regressiontests/fixtures_regress/models.py b/tests/regressiontests/fixtures_regress/models.py index e33471646c..6e733ec256 100644 --- a/tests/regressiontests/fixtures_regress/models.py +++ b/tests/regressiontests/fixtures_regress/models.py @@ -13,7 +13,7 @@ class Animal(models.Model): specimens = models.Manager() def __unicode__(self): - return self.common_name + return self.name def animal_pre_save_check(signal, sender, instance, **kwargs): "A signal that is used to check the type of data loaded from fixtures" @@ -69,10 +69,66 @@ class Article(models.Model): class Widget(models.Model): name = models.CharField(max_length=255) + class Meta: + ordering = ('name',) + + def __unicode__(self): + return self.name + class WidgetProxy(Widget): class Meta: proxy = True +# Check for forward references in FKs and M2Ms with natural keys + +class TestManager(models.Manager): + def get_by_natural_key(self, key): + return self.get(name=key) + +class Store(models.Model): + objects = TestManager() + name = models.CharField(max_length=255) + + class Meta: + ordering = ('name',) + + def __unicode__(self): + return self.name + + def natural_key(self): + return (self.name,) + +class Person(models.Model): + objects = TestManager() + name = models.CharField(max_length=255) + + class Meta: + ordering = ('name',) + + def __unicode__(self): + return self.name + + # Person doesn't actually have a dependency on store, but we need to define + # one to test the behaviour of the dependency resolution algorithm. + def natural_key(self): + return (self.name,) + natural_key.dependencies = ['fixtures_regress.store'] + +class Book(models.Model): + name = models.CharField(max_length=255) + author = models.ForeignKey(Person) + stores = models.ManyToManyField(Store) + + class Meta: + ordering = ('name',) + + def __unicode__(self): + return u'%s by %s (available at %s)' % ( + self.name, + self.author.name, + ', '.join(s.name for s in self.stores.all()) + ) + __test__ = {'API_TESTS':""" >>> from django.core import management @@ -192,4 +248,121 @@ Weight = 1.2 () >>> management.call_command('dumpdata', 'fixtures_regress', format='json') [{"pk": 1, "model": "fixtures_regress.widget", "fields": {"name": "grommet"}}] +############################################### +# Check that natural key requirements are taken into account +# when serializing models +>>> management.call_command('loaddata', 'forward_ref_lookup.json', verbosity=0) + +>>> management.call_command('dumpdata', 'fixtures_regress.book', 'fixtures_regress.person', 'fixtures_regress.store', verbosity=0, use_natural_keys=True) +[{"pk": 2, "model": "fixtures_regress.store", "fields": {"name": "Amazon"}}, {"pk": 3, "model": "fixtures_regress.store", "fields": {"name": "Borders"}}, {"pk": 4, "model": "fixtures_regress.person", "fields": {"name": "Neal Stephenson"}}, {"pk": 1, "model": "fixtures_regress.book", "fields": {"stores": [["Amazon"], ["Borders"]], "name": "Cryptonomicon", "author": ["Neal Stephenson"]}}] + +# Now lets check the dependency sorting explicitly + +# First Some models with pathological circular dependencies +>>> class Circle1(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle2'] + +>>> class Circle2(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle1'] + +>>> class Circle3(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle3'] + +>>> class Circle4(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle5'] + +>>> class Circle5(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle6'] + +>>> class Circle6(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.circle4'] + +>>> class ExternalDependency(models.Model): +... name = models.CharField(max_length=255) +... def natural_key(self): +... return self.name +... natural_key.dependencies = ['fixtures_regress.book'] + +# It doesn't matter what order you mention the models +# Store *must* be serialized before then Person, and both +# must be serialized before Book. +>>> from django.core.management.commands.dumpdata import sort_dependencies +>>> sort_dependencies([('fixtures_regress', [Book, Person, Store])]) +[, , ] + +>>> sort_dependencies([('fixtures_regress', [Book, Store, Person])]) +[, , ] + +>>> sort_dependencies([('fixtures_regress', [Store, Book, Person])]) +[, , ] + +>>> sort_dependencies([('fixtures_regress', [Store, Person, Book])]) +[, , ] + +>>> sort_dependencies([('fixtures_regress', [Person, Book, Store])]) +[, , ] + +>>> sort_dependencies([('fixtures_regress', [Person, Store, Book])]) +[, , ] + +# A dangling dependency - assume the user knows what they are doing. +>>> sort_dependencies([('fixtures_regress', [Person, Circle1, Store, Book])]) +[, , , ] + +# A tight circular dependency +>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Store, Book])]) +Traceback (most recent call last): +... +CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2 in serialized app list. + +>>> sort_dependencies([('fixtures_regress', [Circle1, Book, Circle2])]) +Traceback (most recent call last): +... +CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2 in serialized app list. + +# A self referential dependency +>>> sort_dependencies([('fixtures_regress', [Book, Circle3])]) +Traceback (most recent call last): +... +CommandError: Can't resolve dependencies for fixtures_regress.Circle3 in serialized app list. + +# A long circular dependency +>>> sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Circle3, Store, Book])]) +Traceback (most recent call last): +... +CommandError: Can't resolve dependencies for fixtures_regress.Circle1, fixtures_regress.Circle2, fixtures_regress.Circle3 in serialized app list. + +# A dependency on a normal, non-natural-key model +>>> sort_dependencies([('fixtures_regress', [Person, ExternalDependency, Book])]) +[, , ] + +############################################### +# Check that normal primary keys still work +# on a model with natural key capabilities + +>>> management.call_command('loaddata', 'non_natural_1.json', verbosity=0) +>>> management.call_command('loaddata', 'non_natural_2.xml', verbosity=0) + +>>> Book.objects.all() +[, , ] + """} +