From 50a8ab7cd1e611e6422a148becaec02218577d67 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 30 Nov 2013 10:53:08 +0100 Subject: [PATCH] Enabled makemessages to support several translation directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Rémy Hubscher, Ramiro Morales, Unai Zalakain and Tim Graham for the reviews. Also fixes #16084. --- .../core/management/commands/makemessages.py | 156 +++++++++++------- docs/man/django-admin.1 | 3 +- docs/ref/django-admin.txt | 4 +- docs/releases/1.7.txt | 5 + docs/topics/i18n/translation.txt | 28 ++-- tests/i18n/project_dir/__init__.py | 4 + .../project_dir/app_no_locale/__init__.py | 0 .../i18n/project_dir/app_no_locale/models.py | 3 + .../project_dir/app_with_locale/__init__.py | 0 .../app_with_locale/locale/.gitkeep | 0 .../project_dir/app_with_locale/models.py | 3 + .../i18n/project_dir/project_locale/.gitkeep | 0 tests/i18n/test_extraction.py | 46 ++++++ 13 files changed, 172 insertions(+), 80 deletions(-) create mode 100644 tests/i18n/project_dir/__init__.py create mode 100644 tests/i18n/project_dir/app_no_locale/__init__.py create mode 100644 tests/i18n/project_dir/app_no_locale/models.py create mode 100644 tests/i18n/project_dir/app_with_locale/__init__.py create mode 100644 tests/i18n/project_dir/app_with_locale/locale/.gitkeep create mode 100644 tests/i18n/project_dir/app_with_locale/models.py create mode 100644 tests/i18n/project_dir/project_locale/.gitkeep diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 25ca250c4a..4994aaf6ba 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -29,25 +29,28 @@ def check_programs(*programs): @total_ordering class TranslatableFile(object): - def __init__(self, dirpath, file_name): + def __init__(self, dirpath, file_name, locale_dir): self.file = file_name self.dirpath = dirpath + self.locale_dir = locale_dir def __repr__(self): return "" % os.sep.join([self.dirpath, self.file]) def __eq__(self, other): - return self.dirpath == other.dirpath and self.file == other.file + return self.path == other.path def __lt__(self, other): - if self.dirpath == other.dirpath: - return self.file < other.file - return self.dirpath < other.dirpath + return self.path < other.path - def process(self, command, potfile, domain, keep_pot=False): + @property + def path(self): + return os.path.join(self.dirpath, self.file) + + def process(self, command, domain): """ - Extract translatable literals from self.file for :param domain: - creating or updating the :param potfile: POT file. + Extract translatable literals from self.file for :param domain:, + creating or updating the POT file. Uses the xgettext GNU gettext utility. """ @@ -127,8 +130,6 @@ class TranslatableFile(object): if status != STATUS_OK: if is_templatized: os.unlink(work_file) - if not keep_pot and os.path.exists(potfile): - os.unlink(potfile) raise CommandError( "errors happened while running xgettext on %s\n%s" % (self.file, errors)) @@ -136,6 +137,8 @@ class TranslatableFile(object): # Print warnings command.stdout.write(errors) if msgs: + # Write/append messages to pot file + potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain)) if is_templatized: # Remove '.py' suffix if os.name == 'nt': @@ -147,6 +150,7 @@ class TranslatableFile(object): new = '#: ' + orig_file[2:] msgs = msgs.replace(old, new) write_pot_file(potfile, msgs) + if is_templatized: os.unlink(work_file) @@ -242,64 +246,94 @@ class Command(NoArgsCommand): % get_text_list(list(self.extensions), 'and')) self.invoked_for_django = False + self.locale_paths = [] + self.default_locale_path = None if os.path.isdir(os.path.join('conf', 'locale')): - localedir = os.path.abspath(os.path.join('conf', 'locale')) + self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))] + self.default_locale_path = self.locale_paths[0] self.invoked_for_django = True # Ignoring all contrib apps self.ignore_patterns += ['contrib/*'] - elif os.path.isdir('locale'): - localedir = os.path.abspath('locale') else: - raise CommandError("This script should be run from the Django Git " - "tree or your project or app tree. If you did indeed run it " - "from the Git checkout or your project or application, " - "maybe you are just missing the conf/locale (in the django " - "tree) or locale (for project and application) directory? It " - "is not created automatically, you have to create it by hand " - "if you want to enable i18n for your project or application.") + self.locale_paths.extend(list(settings.LOCALE_PATHS)) + # Allow to run makemessages inside an app dir + if os.path.isdir('locale'): + self.locale_paths.append(os.path.abspath('locale')) + if self.locale_paths: + self.default_locale_path = self.locale_paths[0] + if not os.path.exists(self.default_locale_path): + os.makedirs(self.default_locale_path) - check_programs('xgettext') - - potfile = self.build_pot_file(localedir) - - # Build po files for each selected locale + # Build locale list locales = [] if locale is not None: locales = locale elif process_all: - locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir)) + locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path)) locales = [os.path.basename(l) for l in locale_dirs] - if locales: check_programs('msguniq', 'msgmerge', 'msgattrib') + check_programs('xgettext') + try: + potfiles = self.build_potfiles() + + # Build po files for each selected locale for locale in locales: if self.verbosity > 0: self.stdout.write("processing locale %s\n" % locale) - self.write_po_file(potfile, locale) + for potfile in potfiles: + self.write_po_file(potfile, locale) finally: - if not self.keep_pot and os.path.exists(potfile): - os.unlink(potfile) + if not self.keep_pot: + self.remove_potfiles() - def build_pot_file(self, localedir): + def build_potfiles(self): + """ + Build pot files and apply msguniq to them. + """ file_list = self.find_files(".") - - potfile = os.path.join(localedir, '%s.pot' % str(self.domain)) - if os.path.exists(potfile): - # Remove a previous undeleted potfile, if any - os.unlink(potfile) - + self.remove_potfiles() for f in file_list: try: - f.process(self, potfile, self.domain, self.keep_pot) + f.process(self, self.domain) except UnicodeDecodeError: self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath)) - return potfile + + potfiles = [] + for path in self.locale_paths: + potfile = os.path.join(path, '%s.pot' % str(self.domain)) + if not os.path.exists(potfile): + continue + args = ['msguniq', '--to-code=utf-8'] + if self.wrap: + args.append(self.wrap) + if self.location: + args.append(self.location) + args.append(potfile) + msgs, errors, status = popen_wrapper(args) + if errors: + if status != STATUS_OK: + raise CommandError( + "errors happened while running msguniq\n%s" % errors) + elif self.verbosity > 0: + self.stdout.write(errors) + with open(potfile, 'w') as fp: + fp.write(msgs) + potfiles.append(potfile) + return potfiles + + def remove_potfiles(self): + for path in self.locale_paths: + pot_path = os.path.join(path, '%s.pot' % str(self.domain)) + if os.path.exists(pot_path): + os.unlink(pot_path) def find_files(self, root): """ - Helper method to get all files in the given root. + Helper method to get all files in the given root. Also check that there + is a matching locale dir for each file. """ def is_ignored(path, ignore_patterns): @@ -319,12 +353,26 @@ class Command(NoArgsCommand): dirnames.remove(dirname) if self.verbosity > 1: self.stdout.write('ignoring directory %s\n' % dirname) + elif dirname == 'locale': + dirnames.remove(dirname) + self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname)) for filename in filenames: - if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), self.ignore_patterns): + file_path = os.path.normpath(os.path.join(dirpath, filename)) + if is_ignored(file_path, self.ignore_patterns): if self.verbosity > 1: self.stdout.write('ignoring file %s in %s\n' % (filename, dirpath)) else: - all_files.append(TranslatableFile(dirpath, filename)) + locale_dir = None + for path in self.locale_paths: + if os.path.abspath(dirpath).startswith(os.path.dirname(path)): + locale_dir = path + break + if not locale_dir: + locale_dir = self.default_locale_path + if not locale_dir: + raise CommandError( + "Unable to find a locale path to store translations for file %s" % file_path) + all_files.append(TranslatableFile(dirpath, filename, locale_dir)) return sorted(all_files) def write_po_file(self, potfile, locale): @@ -332,30 +380,14 @@ class Command(NoArgsCommand): Creates or updates the PO file for self.domain and :param locale:. Uses contents of the existing :param potfile:. - Uses mguniq, msgmerge, and msgattrib GNU gettext utilities. + Uses msgmerge, and msgattrib GNU gettext utilities. """ - args = ['msguniq', '--to-code=utf-8'] - if self.wrap: - args.append(self.wrap) - if self.location: - args.append(self.location) - args.append(potfile) - msgs, errors, status = popen_wrapper(args) - if errors: - if status != STATUS_OK: - raise CommandError( - "errors happened while running msguniq\n%s" % errors) - elif self.verbosity > 0: - self.stdout.write(errors) - basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES') if not os.path.isdir(basedir): os.makedirs(basedir) pofile = os.path.join(basedir, '%s.po' % str(self.domain)) if os.path.exists(pofile): - with open(potfile, 'w') as fp: - fp.write(msgs) args = ['msgmerge', '-q'] if self.wrap: args.append(self.wrap) @@ -369,8 +401,10 @@ class Command(NoArgsCommand): "errors happened while running msgmerge\n%s" % errors) elif self.verbosity > 0: self.stdout.write(errors) - elif not self.invoked_for_django: - msgs = self.copy_plural_forms(msgs, locale) + else: + msgs = open(potfile, 'r').read() + if not self.invoked_for_django: + msgs = self.copy_plural_forms(msgs, locale) msgs = msgs.replace( "#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "") with open(pofile, 'w') as fp: diff --git a/docs/man/django-admin.1 b/docs/man/django-admin.1 index f1b568daf5..c72c3520b5 100644 --- a/docs/man/django-admin.1 +++ b/docs/man/django-admin.1 @@ -192,7 +192,8 @@ Ignore files or directories matching this glob-style pattern. Use multiple times to ignore more (makemessages command). .TP .I \-\-no\-default\-ignore -Don't ignore the common private glob-style patterns 'CVS', '.*' and '*~' (makemessages command). +Don't ignore the common private glob-style patterns 'CVS', '.*', '*~' and '*.pyc' +(makemessages command). .TP .I \-\-no\-wrap Don't break long message lines into several lines (makemessages command). diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 02c6cd5851..69555dcb5c 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -557,7 +557,7 @@ Example usage:: Use the ``--ignore`` or ``-i`` option to ignore files or directories matching the given :mod:`glob`-style pattern. Use multiple times to ignore more. -These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'`` +These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``, ``'*.pyc'`` Example usage:: @@ -584,7 +584,7 @@ for technically skilled translators to understand each message's context. .. versionadded:: 1.6 Use the ``--keep-pot`` option to prevent Django from deleting the temporary -.pot file it generates before creating the .po file. This is useful for +.pot files it generates before creating the .po file. This is useful for debugging errors which may prevent the final language files from being created. makemigrations [] diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 0767659540..1041edb37b 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -375,6 +375,11 @@ Internationalization in the corresponding entry in the PO file, which makes the translation process easier. +* When you run :djadmin:`makemessages` from the root directory of your project, + any extracted strings will now be automatically distributed to the proper + app or project message file. See :ref:`how-to-create-language-files` for + details. + Management Commands ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 8a484702e7..66b9a99e21 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1256,6 +1256,17 @@ is configured correctly). It creates (or updates) a message file in the directory ``locale/LANG/LC_MESSAGES``. In the ``de`` example, the file will be ``locale/de/LC_MESSAGES/django.po``. +.. versionchanged:: 1.7 + + When you run ``makemessages`` from the root directory of your project, the + extracted strings will be automatically distributed to the proper message + files. That is, a string extracted from a file of an app containing a + ``locale`` directory will go in a message file under that directory. + A string extracted from a file of an app without any ``locale`` directory + will either go in a message file under the directory listed first in + :setting:`LOCALE_PATHS` or will generate an error if :setting:`LOCALE_PATHS` + is empty. + By default :djadmin:`django-admin.py makemessages ` examines every file that has the ``.html`` or ``.txt`` file extension. In case you want to override that default, use the ``--extension`` or ``-e`` option to specify the @@ -1730,24 +1741,9 @@ All message file repositories are structured the same way. They are: * ``$PYTHONPATH/django/conf/locale//LC_MESSAGES/django.(po|mo)`` To create message files, you use the :djadmin:`django-admin.py makemessages ` -tool. You only need to be in the same directory where the ``locale/`` directory -is located. And you use :djadmin:`django-admin.py compilemessages ` +tool. And you use :djadmin:`django-admin.py compilemessages ` to produce the binary ``.mo`` files that are used by ``gettext``. You can also run :djadmin:`django-admin.py compilemessages --settings=path.to.settings ` to make the compiler process all the directories in your :setting:`LOCALE_PATHS` setting. - -Finally, you should give some thought to the structure of your translation -files. If your applications need to be delivered to other users and will be used -in other projects, you might want to use app-specific translations. But using -app-specific translations and project-specific translations could produce weird -problems with :djadmin:`makemessages`: it will traverse all directories below -the current path and so might put message IDs into a unified, common message -file for the current project that are already in application message files. - -The easiest way out is to store applications that are not part of the project -(and so carry their own translations) outside the project tree. That way, -:djadmin:`django-admin.py makemessages `, when ran on a project -level will only extract strings that are connected to your explicit project and -not strings that are distributed independently. diff --git a/tests/i18n/project_dir/__init__.py b/tests/i18n/project_dir/__init__.py new file mode 100644 index 0000000000..b32b258e37 --- /dev/null +++ b/tests/i18n/project_dir/__init__.py @@ -0,0 +1,4 @@ +# Sample project used by test_extraction.CustomLayoutExtractionTests +from django.utils.translation import ugettext as _ + +string = _("This is a project-level string") diff --git a/tests/i18n/project_dir/app_no_locale/__init__.py b/tests/i18n/project_dir/app_no_locale/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_no_locale/models.py b/tests/i18n/project_dir/app_no_locale/models.py new file mode 100644 index 0000000000..06dfbaa5d4 --- /dev/null +++ b/tests/i18n/project_dir/app_no_locale/models.py @@ -0,0 +1,3 @@ +from django.utils.translation import ugettext as _ + +string = _("This app has no locale directory") diff --git a/tests/i18n/project_dir/app_with_locale/__init__.py b/tests/i18n/project_dir/app_with_locale/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_with_locale/locale/.gitkeep b/tests/i18n/project_dir/app_with_locale/locale/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_with_locale/models.py b/tests/i18n/project_dir/app_with_locale/models.py new file mode 100644 index 0000000000..ab35d5002a --- /dev/null +++ b/tests/i18n/project_dir/app_with_locale/models.py @@ -0,0 +1,3 @@ +from django.utils.translation import ugettext as _ + +string = _("This app has a locale directory") diff --git a/tests/i18n/project_dir/project_locale/.gitkeep b/tests/i18n/project_dir/project_locale/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py index d2af8b0cc8..1637e29f67 100644 --- a/tests/i18n/test_extraction.py +++ b/tests/i18n/test_extraction.py @@ -8,9 +8,11 @@ import shutil from unittest import SkipTest, skipUnless import warnings +from django.conf import settings from django.core import management from django.core.management.utils import find_command from django.test import SimpleTestCase +from django.test.utils import override_settings from django.utils.encoding import force_text from django.utils._os import upath from django.utils import six @@ -497,3 +499,47 @@ class MultipleLocaleExtractionTests(ExtractorTests): management.call_command('makemessages', locale=['pt', 'de'], verbosity=0) self.assertTrue(os.path.exists(self.PO_FILE_PT)) self.assertTrue(os.path.exists(self.PO_FILE_DE)) + + +class CustomLayoutExtractionTests(ExtractorTests): + def setUp(self): + self._cwd = os.getcwd() + self.test_dir = os.path.join(os.path.dirname(upath(__file__)), 'project_dir') + + def test_no_locale_raises(self): + os.chdir(self.test_dir) + with six.assertRaisesRegex(self, management.CommandError, + "Unable to find a locale path to store translations for file"): + management.call_command('makemessages', locale=LOCALE, verbosity=0) + + @override_settings( + LOCALE_PATHS=(os.path.join( + os.path.dirname(upath(__file__)), 'project_dir', 'project_locale'),) + ) + def test_project_locale_paths(self): + """ + Test that: + * translations for an app containing a locale folder are stored in that folder + * translations outside of that app are in LOCALE_PATHS[0] + """ + os.chdir(self.test_dir) + self.addCleanup(shutil.rmtree, + os.path.join(settings.LOCALE_PATHS[0], LOCALE), True) + self.addCleanup(shutil.rmtree, + os.path.join(self.test_dir, 'app_with_locale', 'locale', LOCALE), True) + + management.call_command('makemessages', locale=[LOCALE], verbosity=0) + project_de_locale = os.path.join( + self.test_dir, 'project_locale', 'de', 'LC_MESSAGES', 'django.po') + app_de_locale = os.path.join( + self.test_dir, 'app_with_locale', 'locale', 'de', 'LC_MESSAGES', 'django.po') + self.assertTrue(os.path.exists(project_de_locale)) + self.assertTrue(os.path.exists(app_de_locale)) + + with open(project_de_locale, 'r') as fp: + po_contents = force_text(fp.read()) + self.assertMsgId('This app has no locale directory', po_contents) + self.assertMsgId('This is a project-level string', po_contents) + with open(app_de_locale, 'r') as fp: + po_contents = force_text(fp.read()) + self.assertMsgId('This app has a locale directory', po_contents)