Fixed #23271 -- Fixed makemessages crash/test failure for some locales.

django/core/management/commands/makemessages.py

@@ -36,14 +36,17 @@ def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="u
     """
     Makes sure text obtained from stdout of gettext utilities is Unicode.
     """
-    stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
-    if os.name == 'nt' and six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING:
-        # This looks weird because it's undoing what
-        # subprocess.Popen(universal_newlines=True).communicate()
-        # does when capturing PO files contents from stdout of gettext command
-        # line programs. No need to do anything on Python 2 because it's
-        # already a byte-string there (#23271).
-        stdout = stdout.encode(DEFAULT_LOCALE_ENCODING).decode(stdout_encoding)
+    # This both decodes utf-8 and cleans line endings. Simply using
+    # popen_wrapper(universal_newlines=True) doesn't properly handle the
+    # encoding. This goes back to popen's flaky support for encoding:
+    # https://bugs.python.org/issue6135. This is a solution for #23271, #21928.
+    # No need to do anything on Python 2 because it's already a byte-string there.
+    manual_io_wrapper = six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING
+
+    stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type,
+                                                universal_newlines=not manual_io_wrapper)
+    if manual_io_wrapper:
+        stdout = io.TextIOWrapper(io.BytesIO(stdout), encoding=stdout_encoding).read()
     if six.PY2:
         stdout = stdout.decode(stdout_encoding)
     return stdout, stderr, status_code

django/core/management/utils.py

@@ -10,7 +10,7 @@ from django.utils.encoding import DEFAULT_LOCALE_ENCODING, force_text
 from .base import CommandError
 
 
-def popen_wrapper(args, os_err_exc_type=CommandError):
+def popen_wrapper(args, os_err_exc_type=CommandError, universal_newlines=True):
     """
     Friendly wrapper around Popen.
 
@@ -18,7 +18,7 @@ def popen_wrapper(args, os_err_exc_type=CommandError):
     """
     try:
         p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
-                close_fds=os.name != 'nt', universal_newlines=True)
+                close_fds=os.name != 'nt', universal_newlines=universal_newlines)
     except OSError as e:
         strerror = force_text(e.strerror, DEFAULT_LOCALE_ENCODING,
                               strings_only=True)

docs/releases/1.8.1.txt

@@ -78,6 +78,8 @@ Bugfixes
 * Removed flushing of the test database with :djadminopt:`--keepdb`, which
   prevented apps with data migrations from using the option (:ticket:`24729`).
 
+* Fixed ``makemessages`` crash in some locales (:ticket:`23271`).
+
 Optimizations
 =============