p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #30399 -- Changed django.utils.html.escape()/urlize() to use html.escape()/unescape().

This commit is contained in:
Jon Dufresne 2019-04-24 04:30:34 -07:00 committed by Carlton Gibson
parent 28d5262fa3
commit 8d76443aba
20 changed files with 57 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
django/utils/html.py
View File

@ -1,5 +1,6 @@
"""HTML utilities suitable for global use."""
import html
import json
import re
from html.parser import HTMLParser
@ -24,14 +25,6 @@ word_split_re = re.compile(r'''([\s<>"']+)''')
simple_url_re = re.compile(r'^https?://\[?\w', re.IGNORECASE)
simple_url_2_re = re.compile(r'^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)($|/.*)$', re.IGNORECASE)
_html_escapes = {
ord('&'): '&amp;',
ord('<'): '&lt;',
ord('>'): '&gt;',
ord('"'): '&quot;',
ord("'"): '&#39;',
}
@keep_lazy(str, SafeString)
def escape(text):
@ -43,7 +36,7 @@ def escape(text):
This may result in double-escaping. If this is a concern, use
conditional_escape() instead.
"""
return mark_safe(str(text).translate(_html_escapes))
return mark_safe(html.escape(str(text)))
_js_escapes = {
@ -259,15 +252,6 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
return x
return '%s…' % x[:max(0, limit - 1)]
def unescape(text):
"""
If input URL is HTML-escaped, unescape it so that it can be safely fed
to smart_urlquote. For example:
http://example.com?x=1&amp;y=&lt;2&gt; => http://example.com?x=1&y=<2>
"""
return text.replace('&amp;', '&').replace('&lt;', '<').replace(
'&gt;', '>').replace('&quot;', '"').replace('&#39;', "'")
def trim_punctuation(lead, middle, trail):
"""
Trim trailing and wrapping punctuation from `middle`. Return the items
@ -292,7 +276,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
# Trim trailing punctuation (after trimming wrapping punctuation,
# as encoded entities contain ';'). Unescape entites to avoid
# breaking them by removing ';'.
middle_unescaped = unescape(middle)
middle_unescaped = html.unescape(middle)
stripped = middle_unescaped.rstrip(TRAILING_PUNCTUATION_CHARS)
if middle_unescaped != stripped:
trail = middle[len(stripped):] + trail
@ -329,9 +313,9 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
url = None
nofollow_attr = ' rel="nofollow"' if nofollow else ''
if simple_url_re.match(middle):
url = smart_urlquote(unescape(middle))
url = smart_urlquote(html.unescape(middle))
elif simple_url_2_re.match(middle):
url = smart_urlquote('http://%s' % unescape(middle))
url = smart_urlquote('http://%s' % html.unescape(middle))
elif ':' not in middle and is_email_simple(middle):
local, domain = middle.rsplit('@', 1)
try:

2
docs/intro/tutorial05.txt
View File

@ -387,7 +387,7 @@ With that ready, we can ask the client to do some work for us::
>>> response.status_code
200
>>> response.content
b'\n <ul>\n \n <li><a href="/polls/1/">What&#39;s up?</a></li>\n \n </ul>\n\n'
b'\n <ul>\n \n <li><a href="/polls/1/">What&#x27;s up?</a></li>\n \n </ul>\n\n'
>>> response.context['latest_question_list']
<QuerySet [<Question: What's up?>]>

2
docs/ref/templates/builtins.txt
View File

@ -1603,7 +1603,7 @@ Escapes a string's HTML. Specifically, it makes these replacements:
* ``<`` is converted to ``&lt;``
* ``>`` is converted to ``&gt;``
* ``'`` (single quote) is converted to ``&#39;``
* ``'`` (single quote) is converted to ``&#x27;``
* ``"`` (double quote) is converted to ``&quot;``
* ``&`` is converted to ``&amp;``

2
docs/ref/templates/language.txt
View File

@ -492,7 +492,7 @@ escaped:
* ``<`` is converted to ``&lt;``
* ``>`` is converted to ``&gt;``
* ``'`` (single quote) is converted to ``&#39;``
* ``'`` (single quote) is converted to ``&#x27;``
* ``"`` (double quote) is converted to ``&quot;``
* ``&`` is converted to ``&amp;``

5
docs/ref/utils.txt
View File

@ -584,6 +584,11 @@ escaping HTML.
for use in HTML. The input is first coerced to a string and the output has
:func:`~django.utils.safestring.mark_safe` applied.
.. versionchanged:: 3.0
In older versions, ``'`` is converted to its decimal code ``&#39;``
instead of the equivalent hex code ``&#x27;``.
.. function:: conditional_escape(text)
Similar to ``escape()``, except that it doesn't operate on pre-escaped

4
docs/releases/3.0.txt
View File

@ -348,6 +348,10 @@ Miscellaneous
the session and :func:`django.contrib.auth.logout` no longer preserves the
session's language after logout.
* :func:`django.utils.html.escape` now uses :func:`html.escape` to escape HTML.
This converts ``'`` to ``&#x27;`` instead of the previous equivalent decimal
code ``&#39;``.
.. _deprecated-features-3.0:
Features deprecated in 3.0

2
tests/admin_docs/test_views.py
View File

@ -199,7 +199,7 @@ class TestModelDetailView(TestDataMixin, AdminDocsTestCase):
"""
Methods with keyword arguments should have their arguments displayed.
"""
self.assertContains(self.response, "<td>suffix=&#39;ltd&#39;</td>")
self.assertContains(self.response, '<td>suffix=&#x27;ltd&#x27;</td>')
def test_methods_with_multiple_arguments_display_arguments(self):
"""

2
tests/auth_tests/test_forms.py
View File

@ -236,7 +236,7 @@ class UserCreationFormTest(TestDataMixin, TestCase):
form = UserCreationForm()
self.assertEqual(
form.fields['password1'].help_text,
'<ul><li>Your password can&#39;t be too similar to your other personal information.</li></ul>'
'<ul><li>Your password can&#x27;t be too similar to your other personal information.</li></ul>'
)
@override_settings(AUTH_PASSWORD_VALIDATORS=[

10
tests/forms_tests/tests/test_forms.py
View File

@ -995,7 +995,7 @@ Java</label></li>
self.assertHTMLEqual(
f.as_table(),
"""<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
<ul class="errorlist"><li>Something&#39;s wrong with &#39;Nothing to escape&#39;</li></ul>
<ul class="errorlist"><li>Something&#x27;s wrong with &#x27;Nothing to escape&#x27;</li></ul>
<input type="text" name="special_name" value="Nothing to escape" required></td></tr>
<tr><th><em>Special</em> Field:</th><td>
<ul class="errorlist"><li>'<b>Nothing to escape</b>' is a safe string</li></ul>
@ -1008,10 +1008,10 @@ Java</label></li>
self.assertHTMLEqual(
f.as_table(),
"""<tr><th>&lt;em&gt;Special&lt;/em&gt; Field:</th><td>
<ul class="errorlist"><li>Something&#39;s wrong with &#39;Should escape &lt; &amp; &gt; and
&lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;&#39;</li></ul>
<ul class="errorlist"><li>Something&#x27;s wrong with &#x27;Should escape &lt; &amp; &gt; and
&lt;script&gt;alert(&#x27;xss&#x27;)&lt;/script&gt;&#x27;</li></ul>
<input type="text" name="special_name"
value="Should escape &lt; &amp; &gt; and &lt;script&gt;alert(&#39;xss&#39;)&lt;/script&gt;" required></td></tr>
value="Should escape &lt; &amp; &gt; and &lt;script&gt;alert(&#x27;xss&#x27;)&lt;/script&gt;" required></td></tr>
<tr><th><em>Special</em> Field:</th><td>
<ul class="errorlist"><li>'<b><i>Do not escape</i></b>' is a safe string</li></ul>
<input type="text" name="special_safe_name" value="&lt;i&gt;Do not escape&lt;/i&gt;" required></td></tr>"""
@ -2632,7 +2632,7 @@ Password: <input type="password" name="password" required>
t.render(Context({'form': UserRegistration(auto_id=False)})),
"""<form>
<p>Username: <input type="text" name="username" maxlength="10" required><br>
Good luck picking a username that doesn&#39;t already exist.</p>
Good luck picking a username that doesn&#x27;t already exist.</p>
<p>Password1: <input type="password" name="password1" required></p>
<p>Password2: <input type="password" name="password2" required></p>
<input type="submit" required>

5
tests/forms_tests/widget_tests/base.py
View File

@ -22,7 +22,10 @@ class WidgetTest(SimpleTestCase):
if self.jinja2_renderer:
output = widget.render(name, value, attrs=attrs, renderer=self.jinja2_renderer, **kwargs)
# Django escapes quotes with '&quot;' while Jinja2 uses '&#34;'.
assertEqual(output.replace('&#34;', '&quot;'), html)
output = output.replace('&#34;', '&quot;')
# Django escapes single quotes with '&#x27;' while Jinja2 uses '&#39;'.
output = output.replace('&#39;', '&#x27;')
assertEqual(output, html)
output = widget.render(name, value, attrs=attrs, renderer=self.django_renderer, **kwargs)
assertEqual(output, html)

2
tests/forms_tests/widget_tests/test_clearablefileinput.py
View File

@ -46,7 +46,7 @@ class ClearableFileInputTest(WidgetTest):
self.check_html(ClearableFileInput(), 'my<div>file', StrangeFieldFile(), html=(
"""
Currently: <a href="something?chapter=1&amp;sect=2&amp;copy=3&amp;lang=en">
something&lt;div onclick=&quot;alert(&#39;oops&#39;)&quot;&gt;.jpg</a>
something&lt;div onclick=&quot;alert(&#x27;oops&#x27;)&quot;&gt;.jpg</a>
<input type="checkbox" name="my&lt;div&gt;file-clear" id="my&lt;div&gt;file-clear_id">
<label for="my&lt;div&gt;file-clear_id">Clear</label><br>
Change: <input type="file" name="my&lt;div&gt;file">

14
tests/model_forms/tests.py
View File

@ -1197,7 +1197,7 @@ class ModelFormBasicTests(TestCase):
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple name="categories">
<option value="%s" selected>Entertainment</option>
<option value="%s" selected>It&#39;s a test</option>
<option value="%s" selected>It&#x27;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
@ -1239,7 +1239,7 @@ class ModelFormBasicTests(TestCase):
<li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
<li>Categories: <select multiple name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">It&#x27;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
@ -1290,7 +1290,7 @@ class ModelFormBasicTests(TestCase):
<li><label for="id_categories">Categories:</label>
<select multiple name="categories" id="id_categories">
<option value="%d" selected>Entertainment</option>
<option value="%d" selected>It&39;s a test</option>
<option value="%d" selected>It&#x27;s a test</option>
<option value="%d">Third test</option>
</select></li>"""
% (self.c1.pk, self.c2.pk, self.c3.pk))
@ -1361,7 +1361,7 @@ class ModelFormBasicTests(TestCase):
<tr><th>Article:</th><td><textarea rows="10" cols="40" name="article" required></textarea></td></tr>
<tr><th>Categories:</th><td><select multiple name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">It&#x27;s a test</option>
<option value="%s">Third test</option>
</select></td></tr>
<tr><th>Status:</th><td><select name="status">
@ -1391,7 +1391,7 @@ class ModelFormBasicTests(TestCase):
<li>Article: <textarea rows="10" cols="40" name="article" required>Hello.</textarea></li>
<li>Categories: <select multiple name="categories">
<option value="%s" selected>Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">It&#x27;s a test</option>
<option value="%s">Third test</option>
</select></li>
<li>Status: <select name="status">
@ -1535,7 +1535,7 @@ class ModelFormBasicTests(TestCase):
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">It&#x27;s a test</option>
<option value="%s">Third test</option>
</select> </li>
<li>Status: <select name="status">
@ -1561,7 +1561,7 @@ class ModelFormBasicTests(TestCase):
<li>Article: <textarea rows="10" cols="40" name="article" required></textarea></li>
<li>Categories: <select multiple name="categories">
<option value="%s">Entertainment</option>
<option value="%s">It&#39;s a test</option>
<option value="%s">It&#x27;s a test</option>
<option value="%s">Third test</option>
<option value="%s">Fourth</option>
</select></li>

2
tests/template_tests/filter_tests/test_addslashes.py
View File

@ -15,7 +15,7 @@ class AddslashesTests(SimpleTestCase):
@setup({'addslashes02': '{{ a|addslashes }} {{ b|addslashes }}'})
def test_addslashes02(self):
output = self.engine.render_to_string('addslashes02', {"a": "<a>'", "b": mark_safe("<a>'")})
self.assertEqual(output, r"&lt;a&gt;\&#39; <a>\'")
self.assertEqual(output, r"&lt;a&gt;\&#x27; <a>\'")
class FunctionTests(SimpleTestCase):

2
tests/template_tests/filter_tests/test_make_list.py
View File

@ -19,7 +19,7 @@ class MakeListTests(SimpleTestCase):
@setup({'make_list02': '{{ a|make_list }}'})
def test_make_list02(self):
output = self.engine.render_to_string('make_list02', {"a": mark_safe("&")})
self.assertEqual(output, "[&#39;&amp;&#39;]")
self.assertEqual(output, '[&#x27;&amp;&#x27;]')
@setup({'make_list03': '{% autoescape off %}{{ a|make_list|stringformat:"s"|safe }}{% endautoescape %}'})
def test_make_list03(self):

2
tests/template_tests/filter_tests/test_title.py
View File

@ -9,7 +9,7 @@ class TitleTests(SimpleTestCase):
@setup({'title1': '{{ a|title }}'})
def test_title1(self):
output = self.engine.render_to_string('title1', {'a': 'JOE\'S CRAB SHACK'})
self.assertEqual(output, 'Joe&#39;s Crab Shack')
self.assertEqual(output, 'Joe&#x27;s Crab Shack')
@setup({'title2': '{{ a|title }}'})
def test_title2(self):

6
tests/template_tests/filter_tests/test_urlize.py
View File

@ -52,7 +52,7 @@ class UrlizeTests(SimpleTestCase):
@setup({'urlize06': '{{ a|urlize }}'})
def test_urlize06(self):
output = self.engine.render_to_string('urlize06', {'a': "<script>alert('foo')</script>"})
self.assertEqual(output, '&lt;script&gt;alert(&#39;foo&#39;)&lt;/script&gt;')
self.assertEqual(output, '&lt;script&gt;alert(&#x27;foo&#x27;)&lt;/script&gt;')
# mailto: testing for urlize
@setup({'urlize07': '{{ a|urlize }}'})
@ -113,7 +113,7 @@ class FunctionTests(SimpleTestCase):
)
self.assertEqual(
urlize('www.server.com\'abc'),
'<a href="http://www.server.com" rel="nofollow">www.server.com</a>&#39;abc',
'<a href="http://www.server.com" rel="nofollow">www.server.com</a>&#x27;abc',
)
self.assertEqual(
urlize('www.server.com<abc'),
@ -284,7 +284,7 @@ class FunctionTests(SimpleTestCase):
('<>', ('&lt;', '&gt;')),
('[]', ('[', ']')),
('""', ('&quot;', '&quot;')),
("''", ('&#39;', '&#39;')),
("''", ('&#x27;', '&#x27;')),
)
for wrapping_in, (start_out, end_out) in wrapping_chars:
with self.subTest(wrapping_in=wrapping_in):

2
tests/template_tests/syntax_tests/test_url.py
View File

@ -78,7 +78,7 @@ class UrlTagTests(SimpleTestCase):
@setup({'url12': '{% url "client_action" id=client.id action="!$&\'()*+,;=~:@," %}'})
def test_url12(self):
output = self.engine.render_to_string('url12', {'client': {'id': 1}})
self.assertEqual(output, '/client/1/!$&amp;&#39;()*+,;=~:@,/')
self.assertEqual(output, '/client/1/!$&amp;&#x27;()*+,;=~:@,/')
@setup({'url13': '{% url "client_action" id=client.id action=arg|join:"-" %}'})
def test_url13(self):

4
tests/utils_tests/test_html.py
View File

@ -27,7 +27,7 @@ class TestUtilsHtml(SimpleTestCase):
('<', '&lt;'),
('>', '&gt;'),
('"', '&quot;'),
("'", '&#39;'),
("'", '&#x27;'),
)
# Substitution patterns for testing the above items.
patterns = ("%s", "asdf%sfdsa", "%s1", "1%sb")
@ -70,6 +70,8 @@ class TestUtilsHtml(SimpleTestCase):
items = (
('<p>See: &#39;&eacute; is an apostrophe followed by e acute</p>',
'See: &#39;&eacute; is an apostrophe followed by e acute'),
('<p>See: &#x27;&eacute; is an apostrophe followed by e acute</p>',
'See: &#x27;&eacute; is an apostrophe followed by e acute'),
('<adf>a', 'a'),
('</adf>a', 'a'),
('<asdf><asdf>e', 'e'),

8
tests/view_tests/tests/test_csrf.py
View File

@ -44,22 +44,22 @@ class CsrfViewTests(SimpleTestCase):
self.assertContains(
response,
'You are seeing this message because this HTTPS site requires a '
'&#39;Referer header&#39; to be sent by your Web browser, but '
'&#x27;Referer header&#x27; to be sent by your Web browser, but '
'none was sent.',
status_code=403,
)
self.assertContains(
response,
'If you have configured your browser to disable &#39;Referer&#39; '
'If you have configured your browser to disable &#x27;Referer&#x27; '
'headers, please re-enable them, at least for this site, or for '
'HTTPS connections, or for &#39;same-origin&#39; requests.',
'HTTPS connections, or for &#x27;same-origin&#x27; requests.',
status_code=403,
)
self.assertContains(
response,
'If you are using the &lt;meta name=&quot;referrer&quot; '
'content=&quot;no-referrer&quot;&gt; tag or including the '
'&#39;Referrer-Policy: no-referrer&#39; header, please remove them.',
'&#x27;Referrer-Policy: no-referrer&#x27; header, please remove them.',
status_code=403,
)

14
tests/view_tests/tests/test_debug.py
View File

@ -304,7 +304,7 @@ class ExceptionReporterTests(SimpleTestCase):
reporter = ExceptionReporter(request, exc_type, exc_value, tb)
html = reporter.get_traceback_html()
self.assertInHTML('<h1>ValueError at /test_view/</h1>', html)
self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
self.assertIn('<pre class="exception_value">Can&#x27;t find my keys</pre>', html)
self.assertIn('<th>Request Method:</th>', html)
self.assertIn('<th>Request URL:</th>', html)
self.assertIn('<h3 id="user-info">USER</h3>', html)
@ -325,7 +325,7 @@ class ExceptionReporterTests(SimpleTestCase):
reporter = ExceptionReporter(None, exc_type, exc_value, tb)
html = reporter.get_traceback_html()
self.assertInHTML('<h1>ValueError</h1>', html)
self.assertIn('<pre class="exception_value">Can&#39;t find my keys</pre>', html)
self.assertIn('<pre class="exception_value">Can&#x27;t find my keys</pre>', html)
self.assertNotIn('<th>Request Method:</th>', html)
self.assertNotIn('<th>Request URL:</th>', html)
self.assertNotIn('<h3 id="user-info">USER</h3>', html)
@ -463,7 +463,7 @@ class ExceptionReporterTests(SimpleTestCase):
reporter = ExceptionReporter(request, None, "I'm a little teapot", None)
html = reporter.get_traceback_html()
self.assertInHTML('<h1>Report at /test_view/</h1>', html)
self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
self.assertIn('<pre class="exception_value">I&#x27;m a little teapot</pre>', html)
self.assertIn('<th>Request Method:</th>', html)
self.assertIn('<th>Request URL:</th>', html)
self.assertNotIn('<th>Exception Type:</th>', html)
@ -476,7 +476,7 @@ class ExceptionReporterTests(SimpleTestCase):
reporter = ExceptionReporter(None, None, "I'm a little teapot", None)
html = reporter.get_traceback_html()
self.assertInHTML('<h1>Report</h1>', html)
self.assertIn('<pre class="exception_value">I&#39;m a little teapot</pre>', html)
self.assertIn('<pre class="exception_value">I&#x27;m a little teapot</pre>', html)
self.assertNotIn('<th>Request Method:</th>', html)
self.assertNotIn('<th>Request URL:</th>', html)
self.assertNotIn('<th>Exception Type:</th>', html)
@ -508,7 +508,7 @@ class ExceptionReporterTests(SimpleTestCase):
except Exception:
exc_type, exc_value, tb = sys.exc_info()
html = ExceptionReporter(None, exc_type, exc_value, tb).get_traceback_html()
self.assertIn('<td class="code"><pre>&#39;&lt;p&gt;Local variable&lt;/p&gt;&#39;</pre></td>', html)
self.assertIn('<td class="code"><pre>&#x27;&lt;p&gt;Local variable&lt;/p&gt;&#x27;</pre></td>', html)
def test_unprintable_values_handling(self):
"Unprintable values should not make the output generation choke."
@ -607,7 +607,7 @@ class ExceptionReporterTests(SimpleTestCase):
An exception report can be generated for requests with 'items' in
request GET, POST, FILES, or COOKIES QueryDicts.
"""
value = '<td>items</td><td class="code"><pre>&#39;Oops&#39;</pre></td>'
value = '<td>items</td><td class="code"><pre>&#x27;Oops&#x27;</pre></td>'
# GET
request = self.rf.get('/test_view/?items=Oops')
reporter = ExceptionReporter(request, None, None, None)
@ -634,7 +634,7 @@ class ExceptionReporterTests(SimpleTestCase):
request = rf.get('/test_view/')
reporter = ExceptionReporter(request, None, None, None)
html = reporter.get_traceback_html()
self.assertInHTML('<td>items</td><td class="code"><pre>&#39;Oops&#39;</pre></td>', html)
self.assertInHTML('<td>items</td><td class="code"><pre>&#x27;Oops&#x27;</pre></td>', html)
def test_exception_fetching_user(self):
"""