Improved input sanitizing with thousand separators

For languages with non-breaking space as thousand separator,
standard space input should also be allowed, as few people know
how to enter non-breaking space on keyboards. Refs #17217.
Thanks Alexey Boriskin for the report and initial patch.

django/utils/formats.py

@@ -1,5 +1,6 @@
 import decimal
 import datetime
+import unicodedata
 
 from django.conf import settings
 from django.utils import dateformat, numberformat, datetime_safe
@@ -192,16 +193,17 @@ def sanitize_separators(value):
     Sanitizes a value according to the current decimal and
     thousand separator setting. Used with form field input.
     """
-    if settings.USE_L10N:
+    if settings.USE_L10N and isinstance(value, six.string_types):
-        decimal_separator = get_format('DECIMAL_SEPARATOR')
-        if isinstance(value, six.string_types):
         parts = []
+        decimal_separator = get_format('DECIMAL_SEPARATOR')
         if decimal_separator in value:
             value, decimals = value.split(decimal_separator, 1)
             parts.append(decimals)
         if settings.USE_THOUSAND_SEPARATOR:
-                parts.append(value.replace(get_format('THOUSAND_SEPARATOR'), ''))
+            thousand_sep = get_format('THOUSAND_SEPARATOR')
-            else:
+            for replacement in set([
+                    thousand_sep, unicodedata.normalize('NFKD', thousand_sep)]):
+                value = value.replace(replacement, '')
         parts.append(value)
         value = '.'.join(reversed(parts))
     return value

tests/regressiontests/i18n/tests.py

@@ -15,7 +15,7 @@ from django.test.utils import override_settings
 from django.utils import translation
 from django.utils.formats import (get_format, date_format, time_format,
     localize, localize_input, iter_format_modules, get_format_modules,
-    number_format)
+    number_format, sanitize_separators)
 from django.utils.importlib import import_module
 from django.utils.numberformat import format as nformat
 from django.utils._os import upath
@@ -669,6 +669,24 @@ class FormattingTests(TestCase):
                 # Checking for the localized "products_delivered" field
                 self.assertInHTML('<input type="text" name="products_delivered" value="12.000" id="id_products_delivered" />', form6.as_ul())
 
+    def test_sanitize_separators(self):
+        """
+        Tests django.utils.formats.sanitize_separators.
+        """
+        # Non-strings are untouched
+        self.assertEqual(sanitize_separators(123), 123)
+
+        with translation.override('ru', deactivate=True):
+            # Russian locale has non-breaking space (\xa0) as thousand separator
+            # Check that usual space is accepted too when sanitizing inputs
+            with self.settings(USE_THOUSAND_SEPARATOR=True):
+                self.assertEqual(sanitize_separators('1\xa0234\xa0567'), '1234567')
+                self.assertEqual(sanitize_separators('77\xa0777,777'), '77777.777')
+                self.assertEqual(sanitize_separators('12 345'), '12345')
+                self.assertEqual(sanitize_separators('77 777,777'), '77777.777')
+            with self.settings(USE_THOUSAND_SEPARATOR=True, USE_L10N=False):
+                self.assertEqual(sanitize_separators('12\xa0345'), '12\xa0345')
+
     def test_iter_format_modules(self):
         """
         Tests the iter_format_modules function.