From 2c837233f5de7d5e309833e39782c7a208a03880 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Thu, 12 Dec 2013 15:58:14 +0100 Subject: [PATCH] Fixed #21574 -- Handle bytes consistently in utils.text.normalize_newlines. All input is now coerced to text before being normalized. This changes nothing under Python 2 but it allows bytes to be passed to the function without a TypeError under Python3 (bytes are assumed to be utf-8 encoded text). Thanks to trac user vajrasky for the report. --- django/utils/text.py | 4 +++- tests/utils_tests/test_text.py | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/django/utils/text.py b/django/utils/text.py index ae3dd83694..d172dc91f8 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -250,7 +250,9 @@ get_text_list = allow_lazy(get_text_list, six.text_type) def normalize_newlines(text): - return force_text(re_newlines.sub('\n', text)) + """Normalizes CRLF and CR newlines to just LF.""" + text = force_text(text) + return re_newlines.sub('\n', text) normalize_newlines = allow_lazy(normalize_newlines, six.text_type) diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py index 2f0e9ede6e..86fe8c38ad 100644 --- a/tests/utils_tests/test_text.py +++ b/tests/utils_tests/test_text.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from django.test import SimpleTestCase -from django.utils import text +from django.utils import six, text class TestUtilsText(SimpleTestCase): @@ -114,6 +114,12 @@ class TestUtilsText(SimpleTestCase): self.assertEqual(text.normalize_newlines("abcdefghi"), "abcdefghi") self.assertEqual(text.normalize_newlines(""), "") + def test_normalize_newlines_bytes(self): + """normalize_newlines should be able to handle bytes too""" + normalized = text.normalize_newlines(b"abc\ndef\rghi\r\n") + self.assertEqual(normalized, "abc\ndef\nghi\n") + self.assertIsInstance(normalized, six.text_type) + def test_slugify(self): items = ( ('Hello, World!', 'hello-world'),