Fixed #11240 -- Made makemessages i18n command escape % symbols in literals passed to the trans tag.

This avoids problems with unintended automatic detection, marking and
validation of Python string formatting specifiers performed by
xgettext(1)/msgfmt(1) that stem from the fact that under the hood makemessages
converts templates to Python code before passing them to xgettext.

This also makes it consistent with its behavior on literals passed to the
blocktrans tag.

Thanks Jannis and claude for reviewing and feedback.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@17190 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Ramiro Morales 2011-12-11 00:07:06 +00:00
parent 073d987a84
commit ff7556c4ec
6 changed files with 253 additions and 80 deletions

View File

@ -439,6 +439,7 @@ block_re = re.compile(r"""^\s*blocktrans(\s+.*context\s+(?:"[^"]*?")|(?:'[^']*?'
endblock_re = re.compile(r"""^\s*endblocktrans$""")
plural_re = re.compile(r"""^\s*plural$""")
constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")
one_percent_re = re.compile(r"""(?<!%)%(?!%)""")
def templatize(src, origin=None):
@ -529,6 +530,7 @@ def templatize(src, origin=None):
g = g.strip('"')
elif g[0] == "'":
g = g.strip("'")
g = one_percent_re.sub('%%', g)
if imatch.group(2):
# A context is provided
context_match = context_re.match(imatch.group(2))

View File

@ -7,17 +7,15 @@ except ImportError:
from django.core.management import CommandError
from django.core.management.commands.compilemessages import compile_messages
from django.test import TestCase
from django.test.utils import override_settings
from django.utils import translation
LOCALE='es_AR'
test_dir = os.path.abspath(os.path.dirname(__file__))
class MessageCompilationTests(TestCase):
MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE
def setUp(self):
self._cwd = os.getcwd()
self.test_dir = os.path.abspath(os.path.dirname(__file__))
def tearDown(self):
os.chdir(self._cwd)
@ -25,11 +23,60 @@ class MessageCompilationTests(TestCase):
class PoFileTests(MessageCompilationTests):
LOCALE='es_AR'
MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE
def test_bom_rejection(self):
os.chdir(self.test_dir)
# We don't use the django.core.management intrastructure (call_command()
os.chdir(test_dir)
# We don't use the django.core.management infrastructure (call_command()
# et al) because CommandError's cause exit(1) there. We test the
# underlying compile_messages function instead
out = StringIO()
self.assertRaises(CommandError, compile_messages, out, locale=LOCALE)
self.assertRaises(CommandError, compile_messages, out, locale=self.LOCALE)
self.assertFalse(os.path.exists(self.MO_FILE))
class PoFileContentsTests(MessageCompilationTests):
# Ticket #11240
LOCALE='fr'
MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE
def setUp(self):
super(PoFileContentsTests, self).setUp()
self.addCleanup(os.unlink, os.path.join(test_dir, self.MO_FILE))
def test_percent_symbol_in_po_file(self):
os.chdir(test_dir)
# We don't use the django.core.management infrastructure (call_command()
# et al) because CommandError's cause exit(1) there. We test the
# underlying compile_messages function instead
out = StringIO()
compile_messages(out, locale=self.LOCALE)
self.assertTrue(os.path.exists(self.MO_FILE))
class PercentRenderingTests(MessageCompilationTests):
# Ticket #11240 -- Testing rendering doesn't belong here but we are trying
# to keep tests for all the stack together
LOCALE='it'
MO_FILE='locale/%s/LC_MESSAGES/django.mo' % LOCALE
@override_settings(LOCALE_PATHS=(os.path.join(test_dir, 'locale'),))
def test_percent_symbol_escaping(self):
from django.template import Template, Context
os.chdir(test_dir)
# We don't use the django.core.management infrastructure (call_command()
# et al) because CommandError's cause exit(1) there. We test the
# underlying compile_messages function instead
out = StringIO()
compile_messages(out, locale=self.LOCALE)
with translation.override(self.LOCALE):
t = Template('{% load i18n %}{% trans "Looks like a str fmt spec %% o but shouldn\'t be interpreted as such" %}')
rendered = t.render(Context({}))
self.assertEqual(rendered, 'IT translation contains %% for the above string')
t = Template('{% load i18n %}{% trans "Completed 50%% of all the tasks" %}')
rendered = t.render(Context({}))
self.assertEqual(rendered, 'IT translation of Completed 50%% of all the tasks')

View File

@ -31,10 +31,13 @@ class ExtractorTests(TestCase):
os.chdir(self._cwd)
def assertMsgId(self, msgid, s, use_quotes=True):
q = '"'
if use_quotes:
msgid = '"%s"' % msgid
q = "'"
needle = 'msgid %s' % msgid
msgid = re.escape(msgid)
return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE))
return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
def assertNotMsgId(self, msgid, s, use_quotes=True):
if use_quotes:
@ -49,35 +52,57 @@ class BasicExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertTrue('#. Translators: This comment should be extracted' in po_contents)
self.assertTrue('This comment should not be extracted' not in po_contents)
# Comments in templates
self.assertTrue('#. Translators: Django template comment for translators' in po_contents)
self.assertTrue("#. Translators: Django comment block for translators\n#. string's meaning unveiled" in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertTrue('#. Translators: This comment should be extracted' in po_contents)
self.assertTrue('This comment should not be extracted' not in po_contents)
# Comments in templates
self.assertTrue('#. Translators: Django template comment for translators' in po_contents)
self.assertTrue("#. Translators: Django comment block for translators\n#. string's meaning unveiled" in po_contents)
self.assertTrue('#. Translators: One-line translator comment #1' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #1\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #1' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #1\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #2' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #2\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #2' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #2\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #3' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #3\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #3' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #3\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #4' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #4\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #4' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #4\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #5 -- with non ASCII characters: áéíóúö' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #5 -- with non ASCII characters: áéíóúö\n#. continued here.' in po_contents)
self.assertTrue('#. Translators: One-line translator comment #5 -- with non ASCII characters: áéíóúö' in po_contents)
self.assertTrue('#. Translators: Two-line translator comment #5 -- with non ASCII characters: áéíóúö\n#. continued here.' in po_contents)
def test_templatize(self):
def test_templatize_trans_tag(self):
# ticket #11240
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('I think that 100%% is more that 50%% of anything.', po_contents)
self.assertMsgId('I think that 100%% is more that 50%% of %(obj)s.', po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('Literal with a percent symbol at the end %%', po_contents)
self.assertMsgId('Literal with a percent %% symbol in the middle', po_contents)
self.assertMsgId('Completed 50%% of all the tasks', po_contents)
self.assertMsgId('Completed 99%% of all the tasks', po_contents)
self.assertMsgId("Shouldn't double escape this sequence: %% (two percent signs)", po_contents)
self.assertMsgId("Shouldn't double escape this sequence %% either", po_contents)
self.assertMsgId("Looks like a str fmt spec %%s but shouldn't be interpreted as such", po_contents)
self.assertMsgId("Looks like a str fmt spec %% o but shouldn't be interpreted as such", po_contents)
def test_templatize_blocktrans_tag(self):
# ticket #11966
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('I think that 100%% is more that 50%% of anything.', po_contents)
self.assertMsgId('I think that 100%% is more that 50%% of %(obj)s.', po_contents)
def test_extraction_error(self):
os.chdir(self.test_dir)
@ -103,26 +128,28 @@ class BasicExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
# {% trans %}
self.assertTrue('msgctxt "Special trans context #1"' in po_contents)
self.assertTrue("Translatable literal #7a" in po_contents)
self.assertTrue('msgctxt "Special trans context #2"' in po_contents)
self.assertTrue("Translatable literal #7b" in po_contents)
self.assertTrue('msgctxt "Special trans context #3"' in po_contents)
self.assertTrue("Translatable literal #7c" in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
# {% trans %}
self.assertTrue('msgctxt "Special trans context #1"' in po_contents)
self.assertTrue("Translatable literal #7a" in po_contents)
self.assertTrue('msgctxt "Special trans context #2"' in po_contents)
self.assertTrue("Translatable literal #7b" in po_contents)
self.assertTrue('msgctxt "Special trans context #3"' in po_contents)
self.assertTrue("Translatable literal #7c" in po_contents)
# {% blocktrans %}
self.assertTrue('msgctxt "Special blocktrans context #1"' in po_contents)
self.assertTrue("Translatable literal #8a" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #2"' in po_contents)
self.assertTrue("Translatable literal #8b-singular" in po_contents)
self.assertTrue("Translatable literal #8b-plural" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #3"' in po_contents)
self.assertTrue("Translatable literal #8c-singular" in po_contents)
self.assertTrue("Translatable literal #8c-plural" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #4"' in po_contents)
self.assertTrue("Translatable literal #8d" in po_contents)
# {% blocktrans %}
self.assertTrue('msgctxt "Special blocktrans context #1"' in po_contents)
self.assertTrue("Translatable literal #8a" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #2"' in po_contents)
self.assertTrue("Translatable literal #8b-singular" in po_contents)
self.assertTrue("Translatable literal #8b-plural" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #3"' in po_contents)
self.assertTrue("Translatable literal #8c-singular" in po_contents)
self.assertTrue("Translatable literal #8c-plural" in po_contents)
self.assertTrue('msgctxt "Special blocktrans context #4"' in po_contents)
self.assertTrue("Translatable literal #8d" in po_contents)
class JavascriptExtractorTests(ExtractorTests):
@ -132,20 +159,22 @@ class JavascriptExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', domain='djangojs', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertMsgId('This one as well.', po_contents)
self.assertMsgId(r'He said, \"hello\".', po_contents)
self.assertMsgId("okkkk", po_contents)
self.assertMsgId("TEXT", po_contents)
self.assertMsgId("It's at http://example.com", po_contents)
self.assertMsgId("String", po_contents)
self.assertMsgId("/* but this one will be too */ 'cause there is no way of telling...", po_contents)
self.assertMsgId("foo", po_contents)
self.assertMsgId("bar", po_contents)
self.assertMsgId("baz", po_contents)
self.assertMsgId("quz", po_contents)
self.assertMsgId("foobar", po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertMsgId('This one as well.', po_contents)
self.assertMsgId(r'He said, \"hello\".', po_contents)
self.assertMsgId("okkkk", po_contents)
self.assertMsgId("TEXT", po_contents)
self.assertMsgId("It's at http://example.com", po_contents)
self.assertMsgId("String", po_contents)
self.assertMsgId("/* but this one will be too */ 'cause there is no way of telling...", po_contents)
self.assertMsgId("foo", po_contents)
self.assertMsgId("bar", po_contents)
self.assertMsgId("baz", po_contents)
self.assertMsgId("quz", po_contents)
self.assertMsgId("foobar", po_contents)
class IgnoredExtractorTests(ExtractorTests):
@ -154,9 +183,11 @@ class IgnoredExtractorTests(ExtractorTests):
pattern1 = os.path.join('ignore_dir', '*')
management.call_command('makemessages', locale=LOCALE, verbosity=0, ignore_patterns=[pattern1])
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertNotMsgId('This should be ignored.', po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertNotMsgId('This should be ignored.', po_contents)
class SymlinkExtractorTests(ExtractorTests):
@ -184,9 +215,11 @@ class SymlinkExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertTrue('templates_symlinked/test.html' in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('This literal should be included.', po_contents)
self.assertTrue('templates_symlinked/test.html' in po_contents)
class CopyPluralFormsExtractorTests(ExtractorTests):
@ -195,8 +228,10 @@ class CopyPluralFormsExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertTrue('Plural-Forms: nplurals=2; plural=(n != 1)' in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertTrue('Plural-Forms: nplurals=2; plural=(n != 1)' in po_contents)
class NoWrapExtractorTests(ExtractorTests):
@ -205,15 +240,19 @@ class NoWrapExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=True)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('This literal should also be included wrapped or not wrapped depending on the use of the --no-wrap option.', po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('This literal should also be included wrapped or not wrapped depending on the use of the --no-wrap option.', po_contents)
def test_no_wrap_disabled(self):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0, no_wrap=False)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertMsgId('""\n"This literal should also be included wrapped or not wrapped depending on the "\n"use of the --no-wrap option."', po_contents, use_quotes=False)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertMsgId('""\n"This literal should also be included wrapped or not wrapped depending on the "\n"use of the --no-wrap option."', po_contents, use_quotes=False)
class NoLocationExtractorTests(ExtractorTests):
@ -222,12 +261,16 @@ class NoLocationExtractorTests(ExtractorTests):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=True)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertFalse('#: templates/test.html:55' in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertFalse('#: templates/test.html:55' in po_contents)
def test_no_location_disabled(self):
os.chdir(self.test_dir)
management.call_command('makemessages', locale=LOCALE, verbosity=0, no_location=False)
self.assertTrue(os.path.exists(self.PO_FILE))
po_contents = open(self.PO_FILE, 'r').read()
self.assertTrue('#: templates/test.html:55' in po_contents)
#po_contents = open(self.PO_FILE, 'r').read()
with open(self.PO_FILE, 'r') as fp:
po_contents = fp.read()
self.assertTrue('#: templates/test.html:55' in po_contents)

View File

@ -0,0 +1,71 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2011-12-04 04:59-0600\n"
"PO-Revision-Date: 2011-12-10 19:12-0300\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: fr\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
#. Translators: Django template comment for translators
#: templates/test.html:9
#, python-format
msgid "I think that 100%% is more that 50%% of anything."
msgstr ""
#: templates/test.html:10
#, python-format
msgid "I think that 100%% is more that 50%% of %(obj)s."
msgstr ""
#: templates/test.html:70
#, python-format
msgid "Literal with a percent symbol at the end %%"
msgstr ""
#: templates/test.html:71
#, python-format
msgid "Literal with a percent %% symbol in the middle"
msgstr ""
#: templates/test.html:72
#, python-format
msgid "Completed 50%% of all the tasks"
msgstr ""
#: templates/test.html:73
#, python-format
msgctxt "ctx0"
msgid "Completed 99%% of all the tasks"
msgstr ""
#: templates/test.html:74
#, python-format
msgid "Shouldn't double escape this sequence: %% (two percent signs)"
msgstr ""
#: templates/test.html:75
#, python-format
msgctxt "ctx1"
msgid "Shouldn't double escape this sequence %% either"
msgstr ""
#: templates/test.html:76
#, python-format
msgid "Looks like a str fmt spec %%s but shouldn't be interpreted as such"
msgstr "Translation of the above string"
#: templates/test.html:77
#, python-format
msgid "Looks like a str fmt spec %% o but shouldn't be interpreted as such"
msgstr "Translation contains %% for the above string"

View File

@ -65,4 +65,13 @@ continued here.{% endcomment %}
{% blocktrans context "Special blocktrans context #1" %}Translatable literal #8a{% endblocktrans %}
{% blocktrans count 2 context "Special blocktrans context #2" %}Translatable literal #8b-singular{% plural %}Translatable literal #8b-plural{% endblocktrans %}
{% blocktrans context "Special blocktrans context #3" count 2 %}Translatable literal #8c-singular{% plural %}Translatable literal #8c-plural{% endblocktrans %}
{% blocktrans with a=1 context "Special blocktrans context #4" %}Translatable literal #8d {{ a }}{% endblocktrans %}
{% blocktrans with a=1 context "Special blocktrans context #4" %}Translatable literal #8d {{ a }}{% endblocktrans %}
{% trans "Literal with a percent symbol at the end %" %}
{% trans "Literal with a percent % symbol in the middle" %}
{% trans "Completed 50% of all the tasks" %}
{% trans "Completed 99% of all the tasks" context "ctx0" %}
{% trans "Shouldn't double escape this sequence: %% (two percent signs)" %}
{% trans "Shouldn't double escape this sequence %% either" context "ctx1" %}
{% trans "Looks like a str fmt spec %s but shouldn't be interpreted as such" %}
{% trans "Looks like a str fmt spec % o but shouldn't be interpreted as such" %}

View File

@ -31,7 +31,8 @@ if can_run_extraction_tests:
CopyPluralFormsExtractorTests, NoWrapExtractorTests,
NoLocationExtractorTests)
if can_run_compilation_tests:
from .commands.compilation import MessageCompilationTests, PoFileTests
from .commands.compilation import (PoFileTests, PoFileContentsTests,
PercentRenderingTests)
from .contenttypes.tests import ContentTypeTests
from .forms import I18nForm, SelectDateForm, SelectDateWidget, CompanyForm
from .models import Company, TestModel