Fixed #23005 -- Allowed specifying special fallback languages

This fixes the Chinese language issues described in #23005 but
also provides for other fallback exceptions by updating the
LANG_INFO structure.
Thanks caxekis at gmail.com for the report and Tim Graham for the
review.

django/conf/locale/__init__.py

@@ -1,8 +1,14 @@
 # -*- encoding: utf-8 -*-
 from __future__ import unicode_literals
 
-# About name_local: capitalize it as if your language name was appearing
-# inside a sentence in your language.
+"""
+LANG_INFO is a dictionary structure to provide meta information about languages.
+
+About name_local: capitalize it as if your language name was appearing
+inside a sentence in your language.
+The 'fallback' key can be used to specify a special fallback logic which doesn't
+follow the traditional 'fr-ca' -> 'fr' fallback logic.
+"""
 
 LANG_INFO = {
     'af': {
@@ -486,6 +492,7 @@ LANG_INFO = {
         'name_local': 'Tiếng Việt',
     },
     'zh-cn': {
+        'fallback': ['zh-hans'],
         'bidi': False,
         'code': 'zh-cn',
         'name': 'Simplified Chinese',
@@ -503,10 +510,23 @@ LANG_INFO = {
         'name': 'Traditional Chinese',
         'name_local': '繁體中文',
     },
+    'zh-hk': {
+        'fallback': ['zh-hant'],
+    },
+    'zh-mo': {
+        'fallback': ['zh-hant'],
+    },
+    'zh-my': {
+        'fallback': ['zh-hans'],
+    },
+    'zh-sg': {
+        'fallback': ['zh-hans'],
+    },
     'zh-tw': {
+        'fallback': ['zh-hant'],
         'bidi': False,
         'code': 'zh-tw',
         'name': 'Traditional Chinese',
         'name_local': '繁體中文',
-    }
+    },
 }

django/utils/translation/__init__.py

@@ -212,7 +212,10 @@ string_concat = lazy(_string_concat, six.text_type)
 def get_language_info(lang_code):
     from django.conf.locale import LANG_INFO
     try:
-        return LANG_INFO[lang_code]
+        lang_info = LANG_INFO[lang_code]
+        if 'fallback' in lang_info and 'name' not in lang_info:
+            return get_language_info(lang_info['fallback'][0])
+        return lang_info
     except KeyError:
         if '-' not in lang_code:
             raise KeyError("Unknown language code %s." % lang_code)

django/utils/translation/trans_real.py

@@ -11,6 +11,7 @@ import warnings
 
 from django.apps import apps
 from django.conf import settings
+from django.conf.locale import LANG_INFO
 from django.core.exceptions import AppRegistryNotReady
 from django.dispatch import receiver
 from django.test.signals import setting_changed
@@ -22,7 +23,6 @@ from django.utils import six, lru_cache
 from django.utils.six import StringIO
 from django.utils.translation import TranslatorCommentWarning, trim_whitespace, LANGUAGE_SESSION_KEY
 
-
 # Translations are cached in a dictionary for every language.
 # The active translations are stored by threadid to make them thread local.
 _translations = {}
@@ -51,13 +51,11 @@ language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE)
 language_code_prefix_re = re.compile(r'^/([\w-]+)(/|$)')
 
 # some browsers use deprecated locales. refs #18419
-_BROWSERS_DEPRECATED_LOCALES = {
+_DJANGO_DEPRECATED_LOCALES = {
     'zh-cn': 'zh-hans',
     'zh-tw': 'zh-hant',
 }
 
-_DJANGO_DEPRECATED_LOCALES = _BROWSERS_DEPRECATED_LOCALES
-
 
 @receiver(setting_changed)
 def reset_cache(**kwargs):
@@ -429,13 +427,16 @@ def get_supported_language_variant(lang_code, strict=False):
     if _supported is None:
         _supported = OrderedDict(settings.LANGUAGES)
     if lang_code:
-        # some browsers use deprecated language codes -- #18419
-        replacement = _BROWSERS_DEPRECATED_LOCALES.get(lang_code)
-        if lang_code not in _supported and replacement in _supported:
-            return replacement
-        # if fr-ca is not supported, try fr.
+        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
+        possible_lang_codes = [lang_code]
+        try:
+            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
+        except KeyError:
+            pass
         generic_lang_code = lang_code.split('-')[0]
-        for code in (lang_code, generic_lang_code):
+        possible_lang_codes.append(generic_lang_code)
+
+        for code in possible_lang_codes:
             if code in _supported and check_for_language(code):
                 return code
         if not strict:

tests/i18n/tests.py

@@ -983,6 +983,16 @@ class MiscTests(TestCase):
         r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-tw,en'}
         self.assertEqual(g(r), 'zh-tw')
 
+    def test_special_fallback_language(self):
+        """
+        Some languages may have special fallbacks that don't follow the simple
+        'fr-ca' -> 'fr' logic (notably Chinese codes).
+        """
+        r = self.rf.get('/')
+        r.COOKIES = {}
+        r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-my,en'}
+        self.assertEqual(get_language_from_request(r), 'zh-hans')
+
     def test_parse_language_cookie(self):
         """
         Now test that we parse language preferences stored in a cookie correctly.
@@ -1156,6 +1166,16 @@ class TestLanguageInfo(TestCase):
     def test_unknown_language_code_and_country_code(self):
         six.assertRaisesRegex(self, KeyError, r"Unknown language code xx-xx and xx\.", get_language_info, 'xx-xx')
 
+    def test_fallback_language_code(self):
+        """
+        get_language_info return the first fallback language info if the lang_info
+        struct does not contain the 'name' key.
+        """
+        li = get_language_info('zh-my')
+        self.assertEqual(li['code'], 'zh-hans')
+        li = get_language_info('zh-cn')
+        self.assertEqual(li['code'], 'zh-cn')
+
 
 class MultipleLocaleActivationTests(TestCase):
     """