From 8d76443aba863b75ad3b1392ca7e1d59bad84dc4 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Wed, 24 Apr 2019 04:30:34 -0700 Subject: [PATCH] Fixed #30399 -- Changed django.utils.html.escape()/urlize() to use html.escape()/unescape(). --- django/utils/html.py | 26 ++++--------------- docs/intro/tutorial05.txt | 2 +- docs/ref/templates/builtins.txt | 2 +- docs/ref/templates/language.txt | 2 +- docs/ref/utils.txt | 5 ++++ docs/releases/3.0.txt | 4 +++ tests/admin_docs/test_views.py | 2 +- tests/auth_tests/test_forms.py | 2 +- tests/forms_tests/tests/test_forms.py | 10 +++---- tests/forms_tests/widget_tests/base.py | 5 +++- .../widget_tests/test_clearablefileinput.py | 2 +- tests/model_forms/tests.py | 14 +++++----- .../filter_tests/test_addslashes.py | 2 +- .../filter_tests/test_make_list.py | 2 +- .../template_tests/filter_tests/test_title.py | 2 +- .../filter_tests/test_urlize.py | 6 ++--- tests/template_tests/syntax_tests/test_url.py | 2 +- tests/utils_tests/test_html.py | 4 ++- tests/view_tests/tests/test_csrf.py | 8 +++--- tests/view_tests/tests/test_debug.py | 14 +++++----- 20 files changed, 57 insertions(+), 59 deletions(-) diff --git a/django/utils/html.py b/django/utils/html.py index 9c519978f5..b26cbd16b8 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -1,5 +1,6 @@ """HTML utilities suitable for global use.""" +import html import json import re from html.parser import HTMLParser @@ -24,14 +25,6 @@ word_split_re = re.compile(r'''([\s<>"']+)''') simple_url_re = re.compile(r'^https?://\[?\w', re.IGNORECASE) simple_url_2_re = re.compile(r'^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)($|/.*)$', re.IGNORECASE) -_html_escapes = { - ord('&'): '&', - ord('<'): '<', - ord('>'): '>', - ord('"'): '"', - ord("'"): ''', -} - @keep_lazy(str, SafeString) def escape(text): @@ -43,7 +36,7 @@ def escape(text): This may result in double-escaping. If this is a concern, use conditional_escape() instead. """ - return mark_safe(str(text).translate(_html_escapes)) + return mark_safe(html.escape(str(text))) _js_escapes = { @@ -259,15 +252,6 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): return x return '%s…' % x[:max(0, limit - 1)] - def unescape(text): - """ - If input URL is HTML-escaped, unescape it so that it can be safely fed - to smart_urlquote. For example: - http://example.com?x=1&y=<2> => http://example.com?x=1&y=<2> - """ - return text.replace('&', '&').replace('<', '<').replace( - '>', '>').replace('"', '"').replace(''', "'") - def trim_punctuation(lead, middle, trail): """ Trim trailing and wrapping punctuation from `middle`. Return the items @@ -292,7 +276,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): # Trim trailing punctuation (after trimming wrapping punctuation, # as encoded entities contain ';'). Unescape entites to avoid # breaking them by removing ';'. - middle_unescaped = unescape(middle) + middle_unescaped = html.unescape(middle) stripped = middle_unescaped.rstrip(TRAILING_PUNCTUATION_CHARS) if middle_unescaped != stripped: trail = middle[len(stripped):] + trail @@ -329,9 +313,9 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): url = None nofollow_attr = ' rel="nofollow"' if nofollow else '' if simple_url_re.match(middle): - url = smart_urlquote(unescape(middle)) + url = smart_urlquote(html.unescape(middle)) elif simple_url_2_re.match(middle): - url = smart_urlquote('http://%s' % unescape(middle)) + url = smart_urlquote('http://%s' % html.unescape(middle)) elif ':' not in middle and is_email_simple(middle): local, domain = middle.rsplit('@', 1) try: diff --git a/docs/intro/tutorial05.txt b/docs/intro/tutorial05.txt index 42891a6ecd..c59a0db155 100644 --- a/docs/intro/tutorial05.txt +++ b/docs/intro/tutorial05.txt @@ -387,7 +387,7 @@ With that ready, we can ask the client to do some work for us:: >>> response.status_code 200 >>> response.content - b'\n \n\n' + b'\n \n\n' >>> response.context['latest_question_list'] ]> diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 9509377a24..cd5476fd6f 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1603,7 +1603,7 @@ Escapes a string's HTML. Specifically, it makes these replacements: * ``<`` is converted to ``<`` * ``>`` is converted to ``>`` -* ``'`` (single quote) is converted to ``'`` +* ``'`` (single quote) is converted to ``'`` * ``"`` (double quote) is converted to ``"`` * ``&`` is converted to ``&`` diff --git a/docs/ref/templates/language.txt b/docs/ref/templates/language.txt index 1287f012c2..33687fc49d 100644 --- a/docs/ref/templates/language.txt +++ b/docs/ref/templates/language.txt @@ -492,7 +492,7 @@ escaped: * ``<`` is converted to ``<`` * ``>`` is converted to ``>`` -* ``'`` (single quote) is converted to ``'`` +* ``'`` (single quote) is converted to ``'`` * ``"`` (double quote) is converted to ``"`` * ``&`` is converted to ``&`` diff --git a/docs/ref/utils.txt b/docs/ref/utils.txt index 5f03dc66e4..b36282048f 100644 --- a/docs/ref/utils.txt +++ b/docs/ref/utils.txt @@ -584,6 +584,11 @@ escaping HTML. for use in HTML. The input is first coerced to a string and the output has :func:`~django.utils.safestring.mark_safe` applied. + .. versionchanged:: 3.0 + + In older versions, ``'`` is converted to its decimal code ``'`` + instead of the equivalent hex code ``'``. + .. function:: conditional_escape(text) Similar to ``escape()``, except that it doesn't operate on pre-escaped diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index b279f9a451..2ee02206a1 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -348,6 +348,10 @@ Miscellaneous the session and :func:`django.contrib.auth.logout` no longer preserves the session's language after logout. +* :func:`django.utils.html.escape` now uses :func:`html.escape` to escape HTML. + This converts ``'`` to ``'`` instead of the previous equivalent decimal + code ``'``. + .. _deprecated-features-3.0: Features deprecated in 3.0 diff --git a/tests/admin_docs/test_views.py b/tests/admin_docs/test_views.py index bcadff7d8a..03d38bb2fd 100644 --- a/tests/admin_docs/test_views.py +++ b/tests/admin_docs/test_views.py @@ -199,7 +199,7 @@ class TestModelDetailView(TestDataMixin, AdminDocsTestCase): """ Methods with keyword arguments should have their arguments displayed. """ - self.assertContains(self.response, "suffix='ltd'") + self.assertContains(self.response, 'suffix='ltd'') def test_methods_with_multiple_arguments_display_arguments(self): """ diff --git a/tests/auth_tests/test_forms.py b/tests/auth_tests/test_forms.py index e12cf0161f..e36931501e 100644 --- a/tests/auth_tests/test_forms.py +++ b/tests/auth_tests/test_forms.py @@ -236,7 +236,7 @@ class UserCreationFormTest(TestDataMixin, TestCase): form = UserCreationForm() self.assertEqual( form.fields['password1'].help_text, - '' + '' ) @override_settings(AUTH_PASSWORD_VALIDATORS=[ diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py index edce5e801f..18fb4a94de 100644 --- a/tests/forms_tests/tests/test_forms.py +++ b/tests/forms_tests/tests/test_forms.py @@ -995,7 +995,7 @@ Java self.assertHTMLEqual( f.as_table(), """<em>Special</em> Field: - + Special Field: @@ -1008,10 +1008,10 @@ Java self.assertHTMLEqual( f.as_table(), """<em>Special</em> Field: - + +value="Should escape < & > and <script>alert('xss')</script>" required> Special Field: """ @@ -2632,7 +2632,7 @@ Password: t.render(Context({'form': UserRegistration(auto_id=False)})), """

Username:
-Good luck picking a username that doesn't already exist.

+Good luck picking a username that doesn't already exist.

Password1:

Password2:

diff --git a/tests/forms_tests/widget_tests/base.py b/tests/forms_tests/widget_tests/base.py index 7222910479..339d78bc71 100644 --- a/tests/forms_tests/widget_tests/base.py +++ b/tests/forms_tests/widget_tests/base.py @@ -22,7 +22,10 @@ class WidgetTest(SimpleTestCase): if self.jinja2_renderer: output = widget.render(name, value, attrs=attrs, renderer=self.jinja2_renderer, **kwargs) # Django escapes quotes with '"' while Jinja2 uses '"'. - assertEqual(output.replace('"', '"'), html) + output = output.replace('"', '"') + # Django escapes single quotes with ''' while Jinja2 uses '''. + output = output.replace(''', ''') + assertEqual(output, html) output = widget.render(name, value, attrs=attrs, renderer=self.django_renderer, **kwargs) assertEqual(output, html) diff --git a/tests/forms_tests/widget_tests/test_clearablefileinput.py b/tests/forms_tests/widget_tests/test_clearablefileinput.py index 2ba376db8a..ff6a1c74e7 100644 --- a/tests/forms_tests/widget_tests/test_clearablefileinput.py +++ b/tests/forms_tests/widget_tests/test_clearablefileinput.py @@ -46,7 +46,7 @@ class ClearableFileInputTest(WidgetTest): self.check_html(ClearableFileInput(), 'my
file', StrangeFieldFile(), html=( """ Currently: - something<div onclick="alert('oops')">.jpg + something<div onclick="alert('oops')">.jpg
Change: diff --git a/tests/model_forms/tests.py b/tests/model_forms/tests.py index e4f24c31ee..3893eb334c 100644 --- a/tests/model_forms/tests.py +++ b/tests/model_forms/tests.py @@ -1197,7 +1197,7 @@ class ModelFormBasicTests(TestCase):
  • Article:
  • Categories:
  • Status:
  • Categories:
  • Status: - +
    • """ % (self.c1.pk, self.c2.pk, self.c3.pk)) @@ -1361,7 +1361,7 @@ class ModelFormBasicTests(TestCase): Article: Categories: Status:
  • Categories:
  • Status:
  • Categories:
  • Status:
  • Categories:
    • diff --git a/tests/template_tests/filter_tests/test_addslashes.py b/tests/template_tests/filter_tests/test_addslashes.py index 9c6abd8684..ede25cfb14 100644 --- a/tests/template_tests/filter_tests/test_addslashes.py +++ b/tests/template_tests/filter_tests/test_addslashes.py @@ -15,7 +15,7 @@ class AddslashesTests(SimpleTestCase): @setup({'addslashes02': '{{ a|addslashes }} {{ b|addslashes }}'}) def test_addslashes02(self): output = self.engine.render_to_string('addslashes02', {"a": "'", "b": mark_safe("'")}) - self.assertEqual(output, r"<a>\' \'") + self.assertEqual(output, r"<a>\' \'") class FunctionTests(SimpleTestCase): diff --git a/tests/template_tests/filter_tests/test_make_list.py b/tests/template_tests/filter_tests/test_make_list.py index 17c4cac480..a3e8535ecb 100644 --- a/tests/template_tests/filter_tests/test_make_list.py +++ b/tests/template_tests/filter_tests/test_make_list.py @@ -19,7 +19,7 @@ class MakeListTests(SimpleTestCase): @setup({'make_list02': '{{ a|make_list }}'}) def test_make_list02(self): output = self.engine.render_to_string('make_list02', {"a": mark_safe("&")}) - self.assertEqual(output, "['&']") + self.assertEqual(output, '['&']') @setup({'make_list03': '{% autoescape off %}{{ a|make_list|stringformat:"s"|safe }}{% endautoescape %}'}) def test_make_list03(self): diff --git a/tests/template_tests/filter_tests/test_title.py b/tests/template_tests/filter_tests/test_title.py index 08a5fb0de4..2eaf4a0219 100644 --- a/tests/template_tests/filter_tests/test_title.py +++ b/tests/template_tests/filter_tests/test_title.py @@ -9,7 +9,7 @@ class TitleTests(SimpleTestCase): @setup({'title1': '{{ a|title }}'}) def test_title1(self): output = self.engine.render_to_string('title1', {'a': 'JOE\'S CRAB SHACK'}) - self.assertEqual(output, 'Joe's Crab Shack') + self.assertEqual(output, 'Joe's Crab Shack') @setup({'title2': '{{ a|title }}'}) def test_title2(self): diff --git a/tests/template_tests/filter_tests/test_urlize.py b/tests/template_tests/filter_tests/test_urlize.py index 649a965203..3843cc3e0a 100644 --- a/tests/template_tests/filter_tests/test_urlize.py +++ b/tests/template_tests/filter_tests/test_urlize.py @@ -52,7 +52,7 @@ class UrlizeTests(SimpleTestCase): @setup({'urlize06': '{{ a|urlize }}'}) def test_urlize06(self): output = self.engine.render_to_string('urlize06', {'a': ""}) - self.assertEqual(output, '<script>alert('foo')</script>') + self.assertEqual(output, '<script>alert('foo')</script>') # mailto: testing for urlize @setup({'urlize07': '{{ a|urlize }}'}) @@ -113,7 +113,7 @@ class FunctionTests(SimpleTestCase): ) self.assertEqual( urlize('www.server.com\'abc'), - 'www.server.com'abc', + 'www.server.com'abc', ) self.assertEqual( urlize('www.server.com', ('<', '>')), ('[]', ('[', ']')), ('""', ('"', '"')), - ("''", (''', ''')), + ("''", (''', ''')), ) for wrapping_in, (start_out, end_out) in wrapping_chars: with self.subTest(wrapping_in=wrapping_in): diff --git a/tests/template_tests/syntax_tests/test_url.py b/tests/template_tests/syntax_tests/test_url.py index a6cc2d50a0..bdbc61454d 100644 --- a/tests/template_tests/syntax_tests/test_url.py +++ b/tests/template_tests/syntax_tests/test_url.py @@ -78,7 +78,7 @@ class UrlTagTests(SimpleTestCase): @setup({'url12': '{% url "client_action" id=client.id action="!$&\'()*+,;=~:@," %}'}) def test_url12(self): output = self.engine.render_to_string('url12', {'client': {'id': 1}}) - self.assertEqual(output, '/client/1/!$&'()*+,;=~:@,/') + self.assertEqual(output, '/client/1/!$&'()*+,;=~:@,/') @setup({'url13': '{% url "client_action" id=client.id action=arg|join:"-" %}'}) def test_url13(self): diff --git a/tests/utils_tests/test_html.py b/tests/utils_tests/test_html.py index d87927cdfe..02825f5e1e 100644 --- a/tests/utils_tests/test_html.py +++ b/tests/utils_tests/test_html.py @@ -27,7 +27,7 @@ class TestUtilsHtml(SimpleTestCase): ('<', '<'), ('>', '>'), ('"', '"'), - ("'", '''), + ("'", '''), ) # Substitution patterns for testing the above items. patterns = ("%s", "asdf%sfdsa", "%s1", "1%sb") @@ -70,6 +70,8 @@ class TestUtilsHtml(SimpleTestCase): items = ( ('

    See: 'é is an apostrophe followed by e acute

    ', 'See: 'é is an apostrophe followed by e acute'), + ('

    See: 'é is an apostrophe followed by e acute

    ', + 'See: 'é is an apostrophe followed by e acute'), ('a', 'a'), ('a', 'a'), ('e', 'e'), diff --git a/tests/view_tests/tests/test_csrf.py b/tests/view_tests/tests/test_csrf.py index 4c20cb897d..77f2373347 100644 --- a/tests/view_tests/tests/test_csrf.py +++ b/tests/view_tests/tests/test_csrf.py @@ -44,22 +44,22 @@ class CsrfViewTests(SimpleTestCase): self.assertContains( response, 'You are seeing this message because this HTTPS site requires a ' - ''Referer header' to be sent by your Web browser, but ' + ''Referer header' to be sent by your Web browser, but ' 'none was sent.', status_code=403, ) self.assertContains( response, - 'If you have configured your browser to disable 'Referer' ' + 'If you have configured your browser to disable 'Referer' ' 'headers, please re-enable them, at least for this site, or for ' - 'HTTPS connections, or for 'same-origin' requests.', + 'HTTPS connections, or for 'same-origin' requests.', status_code=403, ) self.assertContains( response, 'If you are using the <meta name="referrer" ' 'content="no-referrer"> tag or including the ' - ''Referrer-Policy: no-referrer' header, please remove them.', + ''Referrer-Policy: no-referrer' header, please remove them.', status_code=403, ) diff --git a/tests/view_tests/tests/test_debug.py b/tests/view_tests/tests/test_debug.py index a61e4b24ff..abe41dc375 100644 --- a/tests/view_tests/tests/test_debug.py +++ b/tests/view_tests/tests/test_debug.py @@ -304,7 +304,7 @@ class ExceptionReporterTests(SimpleTestCase): reporter = ExceptionReporter(request, exc_type, exc_value, tb) html = reporter.get_traceback_html() self.assertInHTML('

    ValueError at /test_view/

    ', html) - self.assertIn('
    Can't find my keys
    ', html) + self.assertIn('
    Can't find my keys
    ', html) self.assertIn('Request Method:', html) self.assertIn('Request URL:', html) self.assertIn('

    USER

    ', html) @@ -325,7 +325,7 @@ class ExceptionReporterTests(SimpleTestCase): reporter = ExceptionReporter(None, exc_type, exc_value, tb) html = reporter.get_traceback_html() self.assertInHTML('

    ValueError

    ', html) - self.assertIn('
    Can't find my keys
    ', html) + self.assertIn('
    Can't find my keys
    ', html) self.assertNotIn('Request Method:', html) self.assertNotIn('Request URL:', html) self.assertNotIn('

    USER

    ', html) @@ -463,7 +463,7 @@ class ExceptionReporterTests(SimpleTestCase): reporter = ExceptionReporter(request, None, "I'm a little teapot", None) html = reporter.get_traceback_html() self.assertInHTML('

    Report at /test_view/

    ', html) - self.assertIn('
    I'm a little teapot
    ', html) + self.assertIn('
    I'm a little teapot
    ', html) self.assertIn('Request Method:', html) self.assertIn('Request URL:', html) self.assertNotIn('Exception Type:', html) @@ -476,7 +476,7 @@ class ExceptionReporterTests(SimpleTestCase): reporter = ExceptionReporter(None, None, "I'm a little teapot", None) html = reporter.get_traceback_html() self.assertInHTML('

    Report

    ', html) - self.assertIn('
    I'm a little teapot
    ', html) + self.assertIn('
    I'm a little teapot
    ', html) self.assertNotIn('Request Method:', html) self.assertNotIn('Request URL:', html) self.assertNotIn('Exception Type:', html) @@ -508,7 +508,7 @@ class ExceptionReporterTests(SimpleTestCase): except Exception: exc_type, exc_value, tb = sys.exc_info() html = ExceptionReporter(None, exc_type, exc_value, tb).get_traceback_html() - self.assertIn('
    '<p>Local variable</p>'
    ', html) + self.assertIn('
    '<p>Local variable</p>'
    ', html) def test_unprintable_values_handling(self): "Unprintable values should not make the output generation choke." @@ -607,7 +607,7 @@ class ExceptionReporterTests(SimpleTestCase): An exception report can be generated for requests with 'items' in request GET, POST, FILES, or COOKIES QueryDicts. """ - value = 'items
    'Oops'
    ' + value = 'items
    'Oops'
    ' # GET request = self.rf.get('/test_view/?items=Oops') reporter = ExceptionReporter(request, None, None, None) @@ -634,7 +634,7 @@ class ExceptionReporterTests(SimpleTestCase): request = rf.get('/test_view/') reporter = ExceptionReporter(request, None, None, None) html = reporter.get_traceback_html() - self.assertInHTML('items
    'Oops'
    ', html) + self.assertInHTML('items
    'Oops'
    ', html) def test_exception_fetching_user(self): """