diff --git a/AUTHORS b/AUTHORS index bf63b1057c..0cd8838cf1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -251,6 +251,7 @@ answer newbie questions, and generally made Django that much better: Erik Karulf Ben Dean Kawamura Ian G. Kelly + Niall Kelly Ryan Kelly Thomas Kerpe Wiley Kestner diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index 5fef3b6936..bcf5631e00 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -42,10 +42,16 @@ class Serializer(base.Serializer): raise base.SerializationError("Non-model object (%s) encountered during serialization" % type(obj)) self.indent(1) - self.xml.startElement("object", { - "pk" : smart_unicode(obj._get_pk_val()), - "model" : smart_unicode(obj._meta), - }) + obj_pk = obj._get_pk_val() + if obj_pk is None: + attrs = {"model": smart_unicode(obj._meta),} + else: + attrs = { + "pk": smart_unicode(obj._get_pk_val()), + "model": smart_unicode(obj._meta), + } + + self.xml.startElement("object", attrs) def end_object(self, obj): """ @@ -166,11 +172,12 @@ class Deserializer(base.Deserializer): # bail. Model = self._get_model_from_node(node, "model") - # Start building a data dictionary from the object. If the node is - # missing the pk attribute, bail. - pk = node.getAttribute("pk") - if not pk: - raise base.DeserializationError(" node is missing the 'pk' attribute") + # Start building a data dictionary from the object. + # If the node is missing the pk set it to None + if node.hasAttribute("pk"): + pk = node.getAttribute("pk") + else: + pk = None data = {Model._meta.pk.attname : Model._meta.pk.to_python(pk)} diff --git a/tests/modeltests/serializers/models.py b/tests/modeltests/serializers/models.py index aac945077b..c12e73fd6e 100644 --- a/tests/modeltests/serializers/models.py +++ b/tests/modeltests/serializers/models.py @@ -18,6 +18,7 @@ class Category(models.Model): def __unicode__(self): return self.name + class Author(models.Model): name = models.CharField(max_length=20) @@ -27,6 +28,7 @@ class Author(models.Model): def __unicode__(self): return self.name + class Article(models.Model): author = models.ForeignKey(Author) headline = models.CharField(max_length=50) @@ -39,6 +41,7 @@ class Article(models.Model): def __unicode__(self): return self.headline + class AuthorProfile(models.Model): author = models.OneToOneField(Author, primary_key=True) date_of_birth = models.DateField() @@ -46,6 +49,7 @@ class AuthorProfile(models.Model): def __unicode__(self): return u"Profile of %s" % self.author + class Actor(models.Model): name = models.CharField(max_length=20, primary_key=True) @@ -55,6 +59,7 @@ class Actor(models.Model): def __unicode__(self): return self.name + class Movie(models.Model): actor = models.ForeignKey(Actor) title = models.CharField(max_length=50) @@ -66,6 +71,7 @@ class Movie(models.Model): def __unicode__(self): return self.title + class Score(models.Model): score = models.FloatField() @@ -83,6 +89,7 @@ class Team(object): def to_string(self): return "%s" % self.title + class TeamField(models.CharField): __metaclass__ = models.SubfieldBase @@ -100,6 +107,7 @@ class TeamField(models.CharField): def value_to_string(self, obj): return self._get_val_from_obj(obj).to_string() + class Player(models.Model): name = models.CharField(max_length=50) rank = models.IntegerField() @@ -107,238 +115,3 @@ class Player(models.Model): def __unicode__(self): return u'%s (%d) playing for %s' % (self.name, self.rank, self.team.to_string()) - -__test__ = {'API_TESTS':""" -# Create some data: ->>> from datetime import datetime ->>> sports = Category(name="Sports") ->>> music = Category(name="Music") ->>> op_ed = Category(name="Op-Ed") ->>> sports.save(); music.save(); op_ed.save() - ->>> joe = Author(name="Joe") ->>> jane = Author(name="Jane") ->>> joe.save(); jane.save() - ->>> a1 = Article( -... author = jane, -... headline = "Poker has no place on ESPN", -... pub_date = datetime(2006, 6, 16, 11, 00)) ->>> a2 = Article( -... author = joe, -... headline = "Time to reform copyright", -... pub_date = datetime(2006, 6, 16, 13, 00, 11, 345)) ->>> a1.save(); a2.save() ->>> a1.categories = [sports, op_ed] ->>> a2.categories = [music, op_ed] - -# Serialize a queryset to XML ->>> from django.core import serializers ->>> xml = serializers.serialize("xml", Article.objects.all()) - -# The output is valid XML ->>> from xml.dom import minidom ->>> dom = minidom.parseString(xml) - -# Deserializing has a similar interface, except that special DeserializedObject -# instances are returned. This is because data might have changed in the -# database since the data was serialized (we'll simulate that below). ->>> for obj in serializers.deserialize("xml", xml): -... print obj - - - -# Deserializing data with different field values doesn't change anything in the -# database until we call save(): ->>> xml = xml.replace("Poker has no place on ESPN", "Poker has no place on television") ->>> objs = list(serializers.deserialize("xml", xml)) - -# Even those I deserialized, the database hasn't been touched ->>> Article.objects.all() -[, ] - -# But when I save, the data changes as you might except. ->>> objs[0].save() ->>> Article.objects.all() -[, ] - -# Django also ships with a built-in JSON serializers ->>> json = serializers.serialize("json", Category.objects.filter(pk=2)) ->>> json -'[{"pk": 2, "model": "serializers.category", "fields": {"name": "Music"}}]' - -# You can easily create new objects by deserializing data with an empty PK -# (It's easier to demo this with JSON...) ->>> new_author_json = '[{"pk": null, "model": "serializers.author", "fields": {"name": "Bill"}}]' ->>> for obj in serializers.deserialize("json", new_author_json): -... obj.save() ->>> Author.objects.all() -[, , ] - -# All the serializers work the same ->>> json = serializers.serialize("json", Article.objects.all()) ->>> for obj in serializers.deserialize("json", json): -... print obj - - - ->>> json = json.replace("Poker has no place on television", "Just kidding; I love TV poker") ->>> for obj in serializers.deserialize("json", json): -... obj.save() - ->>> Article.objects.all() -[, ] - -# If you use your own primary key field (such as a OneToOneField), -# it doesn't appear in the serialized field list - it replaces the -# pk identifier. ->>> profile = AuthorProfile(author=joe, date_of_birth=datetime(1970,1,1)) ->>> profile.save() - ->>> json = serializers.serialize("json", AuthorProfile.objects.all()) ->>> json -'[{"pk": 1, "model": "serializers.authorprofile", "fields": {"date_of_birth": "1970-01-01"}}]' - ->>> for obj in serializers.deserialize("json", json): -... print obj - - -# Objects ids can be referenced before they are defined in the serialization data -# However, the deserialization process will need to be contained within a transaction ->>> json = '[{"pk": 3, "model": "serializers.article", "fields": {"headline": "Forward references pose no problem", "pub_date": "2006-06-16 15:00:00", "categories": [4, 1], "author": 4}}, {"pk": 4, "model": "serializers.category", "fields": {"name": "Reference"}}, {"pk": 4, "model": "serializers.author", "fields": {"name": "Agnes"}}]' ->>> from django.db import transaction ->>> transaction.enter_transaction_management() ->>> transaction.managed(True) ->>> for obj in serializers.deserialize("json", json): -... obj.save() - ->>> transaction.commit() ->>> transaction.leave_transaction_management() - ->>> article = Article.objects.get(pk=3) ->>> article - ->>> article.categories.all() -[, ] ->>> article.author - - -# Serializer output can be restricted to a subset of fields ->>> print serializers.serialize("json", Article.objects.all(), fields=('headline','pub_date')) -[{"pk": 1, "model": "serializers.article", "fields": {"headline": "Just kidding; I love TV poker", "pub_date": "2006-06-16 11:00:00"}}, {"pk": 2, "model": "serializers.article", "fields": {"headline": "Time to reform copyright", "pub_date": "2006-06-16 13:00:11"}}, {"pk": 3, "model": "serializers.article", "fields": {"headline": "Forward references pose no problem", "pub_date": "2006-06-16 15:00:00"}}] - -# Every string is serialized as a unicode object, also primary key -# which is 'varchar' ->>> ac = Actor(name="Zażółć") ->>> mv = Movie(title="Gęślą jaźń", actor=ac) ->>> ac.save(); mv.save() - -# Let's serialize our movie ->>> print serializers.serialize("json", [mv]) -[{"pk": 1, "model": "serializers.movie", "fields": {"price": "0.00", "actor": "Za\u017c\u00f3\u0142\u0107", "title": "G\u0119\u015bl\u0105 ja\u017a\u0144"}}] - -# Deserialization of movie ->>> list(serializers.deserialize('json', serializers.serialize('json', [mv])))[0].object.title -u'G\u0119\u015bl\u0105 ja\u017a\u0144' - -# None is null after serialization to json -# Primary key is None in case of not saved model ->>> mv2 = Movie(title="Movie 2", actor=ac) ->>> print serializers.serialize("json", [mv2]) -[{"pk": null, "model": "serializers.movie", "fields": {"price": "0.00", "actor": "Za\u017c\u00f3\u0142\u0107", "title": "Movie 2"}}] - -# Deserialization of null returns None for pk ->>> print list(serializers.deserialize('json', serializers.serialize('json', [mv2])))[0].object.id -None - -# Serialization and deserialization of floats: ->>> sc = Score(score=3.4) ->>> print serializers.serialize("json", [sc]) -[{"pk": null, "model": "serializers.score", "fields": {"score": 3.4}}] ->>> print list(serializers.deserialize('json', serializers.serialize('json', [sc])))[0].object.score -3.4 - -# Custom field with non trivial to string convertion value ->>> player = Player() ->>> player.name = "Soslan Djanaev" ->>> player.rank = 1 ->>> player.team = Team("Spartak Moskva") ->>> player.save() - ->>> serialized = serializers.serialize("json", Player.objects.all()) ->>> print serialized -[{"pk": 1, "model": "serializers.player", "fields": {"name": "Soslan Djanaev", "rank": 1, "team": "Spartak Moskva"}}] - ->>> obj = list(serializers.deserialize("json", serialized))[0] ->>> print obj - - -# Regression for #12524 -- dates before 1000AD get prefixed 0's on the year ->>> a = Article.objects.create( -... pk=4, -... author = jane, -... headline = "Nobody remembers the early years", -... pub_date = datetime(1, 2, 3, 4, 5, 6)) - ->>> serialized = serializers.serialize("json", [a]) ->>> print serialized -[{"pk": 4, "model": "serializers.article", "fields": {"headline": "Nobody remembers the early years", "pub_date": "0001-02-03 04:05:06", "categories": [], "author": 2}}] - ->>> obj = list(serializers.deserialize("json", serialized))[0] ->>> print obj.object.pub_date -0001-02-03 04:05:06 - -"""} - -try: - import yaml - __test__['YAML'] = """ -# Create some data: - ->>> articles = Article.objects.all().order_by("id")[:2] ->>> from django.core import serializers - -# test if serial - ->>> serialized = serializers.serialize("yaml", articles) ->>> print serialized -- fields: - author: 2 - categories: [3, 1] - headline: Just kidding; I love TV poker - pub_date: 2006-06-16 11:00:00 - model: serializers.article - pk: 1 -- fields: - author: 1 - categories: [2, 3] - headline: Time to reform copyright - pub_date: 2006-06-16 13:00:11 - model: serializers.article - pk: 2 - - ->>> obs = list(serializers.deserialize("yaml", serialized)) ->>> for i in obs: -... print i - - - -# Custom field with non trivial to string convertion value with YAML serializer - ->>> print serializers.serialize("yaml", Player.objects.all()) -- fields: {name: Soslan Djanaev, rank: 1, team: Spartak Moskva} - model: serializers.player - pk: 1 - - ->>> serialized = serializers.serialize("yaml", Player.objects.all()) ->>> obj = list(serializers.deserialize("yaml", serialized))[0] ->>> print obj - - - -""" -except ImportError: - pass - diff --git a/tests/modeltests/serializers/tests.py b/tests/modeltests/serializers/tests.py new file mode 100644 index 0000000000..9b648a8d4e --- /dev/null +++ b/tests/modeltests/serializers/tests.py @@ -0,0 +1,417 @@ +# -*- coding: utf-8 -*- +from datetime import datetime +from StringIO import StringIO +from xml.dom import minidom + +from django.core import serializers +from django.db import transaction +from django.test import TestCase, TransactionTestCase, Approximate +from django.utils import simplejson + +from models import Category, Author, Article, AuthorProfile, Actor, \ + Movie, Score, Player, Team + +class SerializersTestBase(object): + @staticmethod + def _comparison_value(value): + return value + + def setUp(self): + sports = Category.objects.create(name="Sports") + music = Category.objects.create(name="Music") + op_ed = Category.objects.create(name="Op-Ed") + + self.joe = Author.objects.create(name="Joe") + self.jane = Author.objects.create(name="Jane") + + self.a1 = Article( + author=self.jane, + headline="Poker has no place on ESPN", + pub_date=datetime(2006, 6, 16, 11, 00) + ) + self.a1.save() + self.a1.categories = [sports, op_ed] + + self.a2 = Article( + author=self.joe, + headline="Time to reform copyright", + pub_date=datetime(2006, 6, 16, 13, 00, 11, 345) + ) + self.a2.save() + self.a2.categories = [music, op_ed] + + def test_serialize(self): + """Tests that basic serialization works.""" + serial_str = serializers.serialize(self.serializer_name, + Article.objects.all()) + self.assertTrue(self._validate_output(serial_str)) + + def test_serializer_roundtrip(self): + """Tests that serialized content can be deserialized.""" + serial_str = serializers.serialize(self.serializer_name, + Article.objects.all()) + models = list(serializers.deserialize(self.serializer_name, serial_str)) + self.assertEqual(len(models), 2) + + def test_altering_serialized_output(self): + """ + Tests the ability to create new objects by + modifying serialized content. + """ + old_headline = "Poker has no place on ESPN" + new_headline = "Poker has no place on television" + serial_str = serializers.serialize(self.serializer_name, + Article.objects.all()) + serial_str = serial_str.replace(old_headline, new_headline) + models = list(serializers.deserialize(self.serializer_name, serial_str)) + + # Prior to saving, old headline is in place + self.assertTrue(Article.objects.filter(headline=old_headline)) + self.assertFalse(Article.objects.filter(headline=new_headline)) + + for model in models: + model.save() + + # After saving, new headline is in place + self.assertTrue(Article.objects.filter(headline=new_headline)) + self.assertFalse(Article.objects.filter(headline=old_headline)) + + def test_one_to_one_as_pk(self): + """ + Tests that if you use your own primary key field + (such as a OneToOneField), it doesn't appear in the + serialized field list - it replaces the pk identifier. + """ + profile = AuthorProfile(author=self.joe, + date_of_birth=datetime(1970,1,1)) + profile.save() + serial_str = serializers.serialize(self.serializer_name, + AuthorProfile.objects.all()) + self.assertFalse(self._get_field_values(serial_str, 'author')) + + for obj in serializers.deserialize(self.serializer_name, serial_str): + self.assertEqual(obj.object.pk, self._comparison_value(self.joe.pk)) + + def test_serialize_field_subset(self): + """Tests that output can be restricted to a subset of fields""" + valid_fields = ('headline','pub_date') + invalid_fields = ("author", "categories") + serial_str = serializers.serialize(self.serializer_name, + Article.objects.all(), + fields=valid_fields) + for field_name in invalid_fields: + self.assertFalse(self._get_field_values(serial_str, field_name)) + + for field_name in valid_fields: + self.assertTrue(self._get_field_values(serial_str, field_name)) + + def test_serialize_unicode(self): + """Tests that unicode makes the roundtrip intact""" + actor_name = u"Za\u017c\u00f3\u0142\u0107" + movie_title = u'G\u0119\u015bl\u0105 ja\u017a\u0144' + ac = Actor(name=actor_name) + mv = Movie(title=movie_title, actor=ac) + ac.save() + mv.save() + + serial_str = serializers.serialize(self.serializer_name, [mv]) + self.assertEqual(self._get_field_values(serial_str, "title")[0], movie_title) + self.assertEqual(self._get_field_values(serial_str, "actor")[0], actor_name) + + obj_list = list(serializers.deserialize(self.serializer_name, serial_str)) + mv_obj = obj_list[0].object + self.assertEqual(mv_obj.title, movie_title) + + def test_serialize_with_null_pk(self): + """ + Tests that serialized data with no primary key results + in a model instance with no id + """ + category = Category(name="Reference") + serial_str = serializers.serialize(self.serializer_name, [category]) + pk_value = self._get_pk_values(serial_str)[0] + self.assertFalse(pk_value) + + cat_obj = list(serializers.deserialize(self.serializer_name, + serial_str))[0].object + self.assertEqual(cat_obj.id, None) + + def test_float_serialization(self): + """Tests that float values serialize and deserialize intact""" + sc = Score(score=3.4) + sc.save() + serial_str = serializers.serialize(self.serializer_name, [sc]) + deserial_objs = list(serializers.deserialize(self.serializer_name, + serial_str)) + self.assertEqual(deserial_objs[0].object.score, Approximate(3.4, places=1)) + + def test_custom_field_serialization(self): + """Tests that custom fields serialize and deserialize intact""" + team_str = "Spartak Moskva" + player = Player() + player.name = "Soslan Djanaev" + player.rank = 1 + player.team = Team(team_str) + player.save() + serial_str = serializers.serialize(self.serializer_name, + Player.objects.all()) + team = self._get_field_values(serial_str, "team") + self.assertTrue(team) + self.assertEqual(team[0], team_str) + + deserial_objs = list(serializers.deserialize(self.serializer_name, serial_str)) + self.assertEqual(deserial_objs[0].object.team.to_string(), + player.team.to_string()) + + def test_pre_1000ad_date(self): + """Tests that year values before 1000AD are properly formatted""" + # Regression for #12524 -- dates before 1000AD get prefixed + # 0's on the year + a = Article.objects.create( + author = self.jane, + headline = "Nobody remembers the early years", + pub_date = datetime(1, 2, 3, 4, 5, 6)) + + serial_str = serializers.serialize(self.serializer_name, [a]) + date_values = self._get_field_values(serial_str, "pub_date") + self.assertEquals(date_values[0], "0001-02-03 04:05:06") + + def test_pkless_serialized_strings(self): + """ + Tests that serialized strings without PKs + can be turned into models + """ + deserial_objs = list(serializers.deserialize(self.serializer_name, + self.pkless_str)) + for obj in deserial_objs: + self.assertFalse(obj.object.id) + obj.save() + self.assertEqual(Category.objects.all().count(), 4) + + +class SerializersTransactionTestBase(object): + def test_forward_refs(self): + """ + Tests that objects ids can be referenced before they are + defined in the serialization data. + """ + # The deserialization process needs to be contained + # within a transaction in order to test forward reference + # handling. + transaction.enter_transaction_management() + transaction.managed(True) + objs = serializers.deserialize(self.serializer_name, self.fwd_ref_str) + for obj in objs: + obj.save() + transaction.commit() + transaction.leave_transaction_management() + + for model_cls in (Category, Author, Article): + self.assertEqual(model_cls.objects.all().count(), 1) + art_obj = Article.objects.all()[0] + self.assertEqual(art_obj.categories.all().count(), 1) + self.assertEqual(art_obj.author.name, "Agnes") + + +class XmlSerializerTestCase(SerializersTestBase, TestCase): + serializer_name = "xml" + pkless_str = """ + + + Reference + +""" + + @staticmethod + def _comparison_value(value): + # The XML serializer handles everything as strings, so comparisons + # need to be performed on the stringified value + return unicode(value) + + @staticmethod + def _validate_output(serial_str): + try: + minidom.parseString(serial_str) + except Exception: + return False + else: + return True + + @staticmethod + def _get_pk_values(serial_str): + ret_list = [] + dom = minidom.parseString(serial_str) + fields = dom.getElementsByTagName("object") + for field in fields: + ret_list.append(field.getAttribute("pk")) + return ret_list + + @staticmethod + def _get_field_values(serial_str, field_name): + ret_list = [] + dom = minidom.parseString(serial_str) + fields = dom.getElementsByTagName("field") + for field in fields: + if field.getAttribute("name") == field_name: + temp = [] + for child in field.childNodes: + temp.append(child.nodeValue) + ret_list.append("".join(temp)) + return ret_list + +class XmlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase): + serializer_name = "xml" + fwd_ref_str = """ + + + 1 + Forward references pose no problem + 2006-06-16 15:00:00 + + + + + + Agnes + + + Reference +""" + + +class JsonSerializerTestCase(SerializersTestBase, TestCase): + serializer_name = "json" + pkless_str = """[{"pk": null, "model": "serializers.category", "fields": {"name": "Reference"}}]""" + + @staticmethod + def _validate_output(serial_str): + try: + simplejson.loads(serial_str) + except Exception: + return False + else: + return True + + @staticmethod + def _get_pk_values(serial_str): + ret_list = [] + serial_list = simplejson.loads(serial_str) + for obj_dict in serial_list: + ret_list.append(obj_dict["pk"]) + return ret_list + + @staticmethod + def _get_field_values(serial_str, field_name): + ret_list = [] + serial_list = simplejson.loads(serial_str) + for obj_dict in serial_list: + if field_name in obj_dict["fields"]: + ret_list.append(obj_dict["fields"][field_name]) + return ret_list + +class JsonSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase): + serializer_name = "json" + fwd_ref_str = """[ + { + "pk": 1, + "model": "serializers.article", + "fields": { + "headline": "Forward references pose no problem", + "pub_date": "2006-06-16 15:00:00", + "categories": [1], + "author": 1 + } + }, + { + "pk": 1, + "model": "serializers.category", + "fields": { + "name": "Reference" + } + }, + { + "pk": 1, + "model": "serializers.author", + "fields": { + "name": "Agnes" + } + }]""" + +try: + import yaml +except ImportError: + pass +else: + class YamlSerializerTestCase(SerializersTestBase, TestCase): + serializer_name = "yaml" + fwd_ref_str = """- fields: + headline: Forward references pose no problem + pub_date: 2006-06-16 15:00:00 + categories: [1] + author: 1 + pk: 1 + model: serializers.article +- fields: + name: Reference + pk: 1 + model: serializers.category +- fields: + name: Agnes + pk: 1 + model: serializers.author""" + + pkless_str = """- fields: + name: Reference + pk: null + model: serializers.category""" + + @staticmethod + def _validate_output(serial_str): + try: + yaml.load(StringIO(serial_str)) + except Exception: + return False + else: + return True + + @staticmethod + def _get_pk_values(serial_str): + ret_list = [] + stream = StringIO(serial_str) + for obj_dict in yaml.load(stream): + ret_list.append(obj_dict["pk"]) + return ret_list + + @staticmethod + def _get_field_values(serial_str, field_name): + ret_list = [] + stream = StringIO(serial_str) + for obj_dict in yaml.load(stream): + if "fields" in obj_dict and field_name in obj_dict["fields"]: + field_value = obj_dict["fields"][field_name] + # yaml.load will return non-string objects for some + # of the fields we are interested in, this ensures that + # everything comes back as a string + if isinstance(field_value, basestring): + ret_list.append(field_value) + else: + ret_list.append(str(field_value)) + return ret_list + + class YamlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase): + serializer_name = "yaml" + fwd_ref_str = """- fields: + headline: Forward references pose no problem + pub_date: 2006-06-16 15:00:00 + categories: [1] + author: 1 + pk: 1 + model: serializers.article +- fields: + name: Reference + pk: 1 + model: serializers.category +- fields: + name: Agnes + pk: 1 + model: serializers.author"""