Fixed #29249 -- Made JSON and YAML serializers use Unicode by default.

This commit is contained in:
Hasan Ramezani 2020-04-23 22:14:32 +02:00 committed by Mariusz Felisiak
parent 8970bb4cfd
commit 68fc21b378
6 changed files with 25 additions and 9 deletions

View File

@ -29,6 +29,7 @@ class Serializer(PythonSerializer):
# Prevent trailing spaces # Prevent trailing spaces
self.json_kwargs['separators'] = (',', ': ') self.json_kwargs['separators'] = (',', ': ')
self.json_kwargs.setdefault('cls', DjangoJSONEncoder) self.json_kwargs.setdefault('cls', DjangoJSONEncoder)
self.json_kwargs.setdefault('ensure_ascii', False)
def start_serialization(self): def start_serialization(self):
self._init_options() self._init_options()

View File

@ -57,6 +57,7 @@ class Serializer(PythonSerializer):
super().handle_field(obj, field) super().handle_field(obj, field)
def end_serialization(self): def end_serialization(self):
self.options.setdefault('allow_unicode', True)
yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options) yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options)
def getvalue(self): def getvalue(self):

View File

@ -668,6 +668,11 @@ Miscellaneous
* The undocumented ``version`` parameter to the * The undocumented ``version`` parameter to the
:class:`~django.contrib.gis.db.models.functions.AsKML` function is removed. :class:`~django.contrib.gis.db.models.functions.AsKML` function is removed.
* :ref:`JSON and YAML serializers <serialization-formats>`, used by
:djadmin:`dumpdata`, now dump all data with Unicode by default. If you need
the previous behavior, pass ``ensure_ascii=True`` to JSON serializer, or
``allow_unicode=False`` to YAML serializer.
.. _deprecated-features-3.1: .. _deprecated-features-3.1:
Features deprecated in 3.1 Features deprecated in 3.1

View File

@ -274,6 +274,11 @@ function::
Also note that GeoDjango provides a :doc:`customized GeoJSON serializer Also note that GeoDjango provides a :doc:`customized GeoJSON serializer
</ref/contrib/gis/serializers>`. </ref/contrib/gis/serializers>`.
.. versionchanged:: 3.1
All data is now dumped with Unicode. If you need the previous behavior,
pass ``ensure_ascii=True`` to the ``serializers.serialize()`` function.
``DjangoJSONEncoder`` ``DjangoJSONEncoder``
~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~
@ -315,6 +320,11 @@ again a mapping with the key being name of the field and the value the value::
Referential fields are again represented by the PK or sequence of PKs. Referential fields are again represented by the PK or sequence of PKs.
.. versionchanged:: 3.1
All data is now dumped with Unicode. If you need the previous behavior,
pass ``allow_unicode=False`` to the ``serializers.serialize()`` function.
.. _topics-serialization-natural-keys: .. _topics-serialization-natural-keys:
Natural keys Natural keys

View File

@ -4,7 +4,6 @@ from django.core import serializers
from django.core.serializers.xml_serializer import DTDForbidden from django.core.serializers.xml_serializer import DTDForbidden
from django.test import TestCase, TransactionTestCase from django.test import TestCase, TransactionTestCase
from .models import Author
from .tests import SerializersTestBase, SerializersTransactionTestBase from .tests import SerializersTestBase, SerializersTransactionTestBase
@ -87,13 +86,6 @@ class XmlSerializerTestCase(SerializersTestBase, TestCase):
with self.assertRaises(DTDForbidden): with self.assertRaises(DTDForbidden):
next(serializers.deserialize('xml', xml)) next(serializers.deserialize('xml', xml))
def test_unicode_serialization(self):
unicode_name = 'יוניקוד'
data = serializers.serialize('xml', [Author(name=unicode_name)])
self.assertIn(unicode_name, data)
objs = list(serializers.deserialize('xml', data))
self.assertEqual(objs[0].object.name, unicode_name)
class XmlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase): class XmlSerializerTransactionTestCase(SerializersTransactionTestBase, TransactionTestCase):
serializer_name = "xml" serializer_name = "xml"

View File

@ -202,7 +202,7 @@ class SerializersTestBase:
for field_name in valid_fields: for field_name in valid_fields:
self.assertTrue(self._get_field_values(serial_str, field_name)) self.assertTrue(self._get_field_values(serial_str, field_name))
def test_serialize_unicode(self): def test_serialize_unicode_roundtrip(self):
"""Unicode makes the roundtrip intact""" """Unicode makes the roundtrip intact"""
actor_name = "Za\u017c\u00f3\u0142\u0107" actor_name = "Za\u017c\u00f3\u0142\u0107"
movie_title = 'G\u0119\u015bl\u0105 ja\u017a\u0144' movie_title = 'G\u0119\u015bl\u0105 ja\u017a\u0144'
@ -219,6 +219,13 @@ class SerializersTestBase:
mv_obj = obj_list[0].object mv_obj = obj_list[0].object
self.assertEqual(mv_obj.title, movie_title) self.assertEqual(mv_obj.title, movie_title)
def test_unicode_serialization(self):
unicode_name = 'יוניקוד'
data = serializers.serialize(self.serializer_name, [Author(name=unicode_name)])
self.assertIn(unicode_name, data)
objs = list(serializers.deserialize(self.serializer_name, data))
self.assertEqual(objs[0].object.name, unicode_name)
def test_serialize_progressbar(self): def test_serialize_progressbar(self):
fake_stdout = StringIO() fake_stdout = StringIO()
serializers.serialize( serializers.serialize(