From 23b234a9d9ea26acb9bb12ba14231ce8844d2e53 Mon Sep 17 00:00:00 2001
From: Tai Lee <tai.lee@3030.com.au>
Date: Mon, 6 May 2013 13:32:07 +1000
Subject: [PATCH] [1.5.x] Fixed #20354 -- `makemessages` no longer crashes with
 `UnicodeDecodeError`

Handle the `UnicodeDecodeError` exception, send a warning to `stdout` with the
file name and location, and continue processing other files.
Backport of 99a6f0e77 from master.
---
 django/core/management/commands/makemessages.py   |  7 +++++--
 tests/regressiontests/i18n/commands/extraction.py | 15 ++++++++++++++-
 .../regressiontests/i18n/commands/not_utf8.sample |  1 +
 3 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 tests/regressiontests/i18n/commands/not_utf8.sample

diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
index 606cbe0b85..f5762dfd56 100644
--- a/django/core/management/commands/makemessages.py
+++ b/django/core/management/commands/makemessages.py
@@ -324,8 +324,11 @@ def make_messages(locale=None, domain='django', verbosity=1, all=False,
 
         for dirpath, file in find_files(".", ignore_patterns, verbosity,
                 stdout, symlinks=symlinks):
-            process_file(file, dirpath, potfile, domain, verbosity, extensions,
-                    wrap, location, stdout)
+            try:
+                process_file(file, dirpath, potfile, domain, verbosity, extensions,
+                        wrap, location, stdout)
+            except UnicodeDecodeError:
+                stdout.write("UnicodeDecodeError: skipped file %s in %s" % (file, dirpath))
 
         if os.path.exists(potfile):
             write_po_file(pofile, potfile, domain, locale, verbosity, stdout,
diff --git a/tests/regressiontests/i18n/commands/extraction.py b/tests/regressiontests/i18n/commands/extraction.py
index aa5efe1967..47d58c4d63 100644
--- a/tests/regressiontests/i18n/commands/extraction.py
+++ b/tests/regressiontests/i18n/commands/extraction.py
@@ -27,6 +27,10 @@ class ExtractorTests(TestCase):
             return
         shutil.rmtree(dname)
 
+    def rmfile(self, filepath):
+        if os.path.exists(filepath):
+            os.remove(filepath)
+
     def tearDown(self):
         os.chdir(self.test_dir)
         try:
@@ -120,12 +124,21 @@ class BasicExtractorTests(ExtractorTests):
         # Check that the temporary file was cleaned up
         self.assertFalse(os.path.exists('./templates/template_with_error.html.py'))
 
+    def test_unicode_decode_error(self):
+        os.chdir(self.test_dir)
+        shutil.copyfile('./not_utf8.sample', './not_utf8.txt')
+        self.addCleanup(self.rmfile, os.path.join(self.test_dir, 'not_utf8.txt'))
+        stdout = StringIO()
+        management.call_command('makemessages', locale=LOCALE, stdout=stdout)
+        self.assertIn("UnicodeDecodeError: skipped file not_utf8.txt in .",
+                      force_text(stdout.getvalue()))
+
     def test_extraction_warning(self):
         os.chdir(self.test_dir)
         shutil.copyfile('./code.sample', './code_sample.py')
+        self.addCleanup(self.rmfile, os.path.join(self.test_dir, 'code_sample.py'))
         stdout = StringIO()
         management.call_command('makemessages', locale=LOCALE, stdout=stdout)
-        os.remove('./code_sample.py')
         self.assertIn("code_sample.py:4", force_text(stdout.getvalue()))
 
     def test_template_message_context_extractor(self):
diff --git a/tests/regressiontests/i18n/commands/not_utf8.sample b/tests/regressiontests/i18n/commands/not_utf8.sample
new file mode 100644
index 0000000000..6449f52803
--- /dev/null
+++ b/tests/regressiontests/i18n/commands/not_utf8.sample
@@ -0,0 +1 @@
+Copyright (c) 2009 Øyvind Sean Kinsey, oyvind@kinsey.no
\ No newline at end of file