From b12c7391430400d5e4b71207b31150fdc6cb07cb Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Sun, 10 Oct 2010 16:38:28 +0000 Subject: [PATCH] Fixed #6073 -- Made compilemessages 18n management command reject PO files with BOM. git-svn-id: http://code.djangoproject.com/svn/django/trunk@14125 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- .../management/commands/compilemessages.py | 19 +++++++--- docs/topics/i18n/localization.txt | 6 ++++ .../makemessages/compilation.py | 35 +++++++++++++++++++ .../locale/es_AR/LC_MESSAGES/django.po | 20 +++++++++++ tests/regressiontests/makemessages/tests.py | 3 ++ 5 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 tests/regressiontests/makemessages/compilation.py create mode 100644 tests/regressiontests/makemessages/locale/es_AR/LC_MESSAGES/django.po diff --git a/django/core/management/commands/compilemessages.py b/django/core/management/commands/compilemessages.py index 7d600e0fe9..b5eaeb1f52 100644 --- a/django/core/management/commands/compilemessages.py +++ b/django/core/management/commands/compilemessages.py @@ -1,9 +1,17 @@ +import codecs import os import sys from optparse import make_option from django.core.management.base import BaseCommand, CommandError -def compile_messages(locale=None): +def has_bom(fn): + f = open(fn, 'r') + sample = f.read(4) + return sample[:3] == '\xef\xbb\xbf' or \ + sample.startswith(codecs.BOM_UTF16_LE) or \ + sample.startswith(codecs.BOM_UTF16_BE) + +def compile_messages(stderr, locale=None): basedirs = [os.path.join('conf', 'locale'), 'locale'] if os.environ.get('DJANGO_SETTINGS_MODULE'): from django.conf import settings @@ -21,8 +29,11 @@ def compile_messages(locale=None): for dirpath, dirnames, filenames in os.walk(basedir): for f in filenames: if f.endswith('.po'): - sys.stderr.write('processing file %s in %s\n' % (f, dirpath)) - pf = os.path.splitext(os.path.join(dirpath, f))[0] + stderr.write('processing file %s in %s\n' % (f, dirpath)) + fn = os.path.join(dirpath, f) + if has_bom(fn): + raise CommandError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % fn) + pf = os.path.splitext(fn)[0] # Store the names of the .mo and .po files in an environment # variable, rather than doing a string replacement into the # command, so that we can take advantage of shell quoting, to @@ -49,4 +60,4 @@ class Command(BaseCommand): def handle(self, **options): locale = options.get('locale') - compile_messages(locale) + compile_messages(self.stderr, locale=locale) diff --git a/docs/topics/i18n/localization.txt b/docs/topics/i18n/localization.txt index 8ba1e1ecdc..38d74e68e3 100644 --- a/docs/topics/i18n/localization.txt +++ b/docs/topics/i18n/localization.txt @@ -188,6 +188,12 @@ That's it. Your translations are ready for use. ``django-admin compilemessages`` works see :ref:`gettext_on_windows` for more information. +.. admonition:: .po files: Encoding and BOM usage. + + Django only supports ``.po`` files encoded in UTF-8 and without any BOM + (Byte Order Mark) so if your text editor adds such marks to the beginning of + files by default then you will need to reconfigure it. + .. _creating-message-files-from-js-code: Creating message files from JavaScript source code diff --git a/tests/regressiontests/makemessages/compilation.py b/tests/regressiontests/makemessages/compilation.py new file mode 100644 index 0000000000..3ca2d57138 --- /dev/null +++ b/tests/regressiontests/makemessages/compilation.py @@ -0,0 +1,35 @@ +import os +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +from django.core.management import CommandError +from django.core.management.commands.compilemessages import compile_messages +from django.test import TestCase + +LOCALE='es_AR' + + +class MessageCompilationTests(TestCase): + + MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE + + def setUp(self): + self._cwd = os.getcwd() + self.test_dir = os.path.abspath(os.path.dirname(__file__)) + + def tearDown(self): + os.chdir(self._cwd) + + +class PoFileTests(MessageCompilationTests): + + def test_bom_rejection(self): + os.chdir(self.test_dir) + # We don't use the django.core.management intrastructure (call_command() + # et al) because CommandError's cause exit(1) there. We test the + # underlying compile_messages function instead + out = StringIO() + self.assertRaises(CommandError, compile_messages, out, locale=LOCALE) + self.failIf(os.path.exists(self.MO_FILE)) diff --git a/tests/regressiontests/makemessages/locale/es_AR/LC_MESSAGES/django.po b/tests/regressiontests/makemessages/locale/es_AR/LC_MESSAGES/django.po new file mode 100644 index 0000000000..7f74a4ef4a --- /dev/null +++ b/tests/regressiontests/makemessages/locale/es_AR/LC_MESSAGES/django.po @@ -0,0 +1,20 @@ +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR +# +msgid "" +msgstr "" +"Project-Id-Version: Django\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2010-01-05 20:25-0300\n" +"PO-Revision-Date: 2010-02-22 00:14-0300\n" +"Last-Translator: Translator \n" +"Language-Team: Django-I18N \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: path/file.py:1 +msgid "This file has a UTF-8 BOM an should be rejected by the Django makemessages command." +msgstr "" diff --git a/tests/regressiontests/makemessages/tests.py b/tests/regressiontests/makemessages/tests.py index 5798e671b0..c3be2178e4 100644 --- a/tests/regressiontests/makemessages/tests.py +++ b/tests/regressiontests/makemessages/tests.py @@ -38,3 +38,6 @@ if xgettext_cmd: if xversion >= (0, 15): from extraction import * del p + +if find_command('msgfmt'): + from compilation import *