Refs #23919 -- Replaced usage of django.utils.http utilities with Python equivalents

Thanks Tim Graham for the review.

django/contrib/admin/options.py

@@ -3,6 +3,7 @@ import json
 import operator
 from collections import OrderedDict
 from functools import partial, reduce, update_wrapper
+from urllib.parse import quote as urlquote
 
 from django import forms
 from django.conf import settings
@@ -39,7 +40,7 @@ from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.utils.encoding import force_text
 from django.utils.html import format_html
-from django.utils.http import urlencode, urlquote
+from django.utils.http import urlencode
 from django.utils.safestring import mark_safe
 from django.utils.text import capfirst, format_lazy, get_text_list
 from django.utils.translation import ugettext as _, ungettext

django/core/cache/utils.py

@@ -1,7 +1,7 @@
 import hashlib
+from urllib.parse import quote
 
 from django.utils.encoding import force_bytes
-from django.utils.http import urlquote
 
 TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s'
 
@@ -9,6 +9,6 @@ TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s'
 def make_template_fragment_key(fragment_name, vary_on=None):
     if vary_on is None:
         vary_on = ()
-    key = ':'.join(urlquote(var) for var in vary_on)
+    key = ':'.join(quote(str(var)) for var in vary_on)
     args = hashlib.md5(force_bytes(key))
     return TEMPLATE_FRAGMENT_KEY_TEMPLATE % (fragment_name, args.hexdigest())

django/template/defaultfilters.py

@@ -5,6 +5,7 @@ from decimal import ROUND_HALF_UP, Context, Decimal, InvalidOperation
 from functools import wraps
 from operator import itemgetter
 from pprint import pformat
+from urllib.parse import quote
 
 from django.utils import formats
 from django.utils.dateformat import format, time_format
@@ -13,7 +14,6 @@ from django.utils.html import (
     avoid_wrapping, conditional_escape, escape, escapejs, linebreaks,
     strip_tags, urlize as _urlize,
 )
-from django.utils.http import urlquote
 from django.utils.safestring import SafeData, mark_safe
 from django.utils.text import (
     Truncator, normalize_newlines, phone2numeric, slugify as _slugify, wrap,
@@ -318,14 +318,14 @@ def urlencode(value, safe=None):
     Escapes a value for use in a URL.
 
     Takes an optional ``safe`` parameter used to determine the characters which
-    should not be escaped by Django's ``urlquote`` method. If not provided, the
+    should not be escaped by Python's quote() function. If not provided, the
     default safe characters will be used (but an empty string can be provided
     when *all* characters should be escaped).
     """
     kwargs = {}
     if safe is not None:
         kwargs['safe'] = safe
-    return urlquote(value, **kwargs)
+    return quote(value, **kwargs)
 
 
 @register.filter(is_safe=True, needs_autoescape=True)

django/urls/resolvers.py

@@ -9,6 +9,7 @@ import functools
 import re
 import threading
 from importlib import import_module
+from urllib.parse import quote
 
 from django.conf import settings
 from django.core.checks import Warning
@@ -17,7 +18,7 @@ from django.core.exceptions import ImproperlyConfigured
 from django.utils.datastructures import MultiValueDict
 from django.utils.encoding import force_text
 from django.utils.functional import cached_property
-from django.utils.http import RFC3986_SUBDELIMS, urlquote
+from django.utils.http import RFC3986_SUBDELIMS
 from django.utils.regex_helper import normalize
 from django.utils.translation import get_language
 
@@ -455,7 +456,7 @@ class RegexURLResolver(LocaleRegexProvider):
                 candidate_pat = _prefix.replace('%', '%%') + result
                 if re.search('^%s%s' % (re.escape(_prefix), pattern), candidate_pat % candidate_subs):
                     # safe characters from `pchar` definition of RFC 3986
-                    url = urlquote(candidate_pat % candidate_subs, safe=RFC3986_SUBDELIMS + str('/~:@'))
+                    url = quote(candidate_pat % candidate_subs, safe=RFC3986_SUBDELIMS + '/~:@')
                     # Don't allow construction of scheme relative urls.
                     if url.startswith('//'):
                         url = '/%%2F%s' % url[2:]

django/utils/http.py

@@ -14,7 +14,7 @@ from urllib.parse import (
 from django.core.exceptions import TooManyFieldsSent
 from django.utils.datastructures import MultiValueDict
 from django.utils.deprecation import RemovedInDjango21Warning
-from django.utils.encoding import force_bytes, force_str, force_text
+from django.utils.encoding import force_bytes
 from django.utils.functional import keep_lazy_text
 
 # based on RFC 7232, Appendix C
@@ -47,58 +47,53 @@ FIELDS_MATCH = re.compile('[&;]')
 @keep_lazy_text
 def urlquote(url, safe='/'):
     """
-    A version of Python's urllib.quote() function that can operate on unicode
+    A legacy compatibility wrapper to Python's urllib.parse.quote() function.
-    strings. The url is first UTF-8 encoded before quoting. The returned string
+    (was used for unicode handling on Python 2)
-    can safely be used as part of an argument to a subsequent iri_to_uri() call
-    without double-quoting occurring.
     """
-    return force_text(quote(force_str(url), force_str(safe)))
+    return quote(url, safe)
 
 
 @keep_lazy_text
 def urlquote_plus(url, safe=''):
     """
-    A version of Python's urllib.quote_plus() function that can operate on
+    A legacy compatibility wrapper to Python's urllib.parse.quote_plus()
-    unicode strings. The url is first UTF-8 encoded before quoting. The
+    function. (was used for unicode handling on Python 2)
-    returned string can safely be used as part of an argument to a subsequent
-    iri_to_uri() call without double-quoting occurring.
     """
-    return force_text(quote_plus(force_str(url), force_str(safe)))
+    return quote_plus(url, safe)
 
 
 @keep_lazy_text
 def urlunquote(quoted_url):
     """
-    A wrapper for Python's urllib.unquote() function that can operate on
+    A legacy compatibility wrapper to Python's urllib.parse.unquote() function.
-    the result of django.utils.http.urlquote().
+    (was used for unicode handling on Python 2)
     """
-    return force_text(unquote(force_str(quoted_url)))
+    return unquote(quoted_url)
 
 
 @keep_lazy_text
 def urlunquote_plus(quoted_url):
     """
-    A wrapper for Python's urllib.unquote_plus() function that can operate on
+    A legacy compatibility wrapper to Python's urllib.parse.unquote_plus()
-    the result of django.utils.http.urlquote_plus().
+    function. (was used for unicode handling on Python 2)
     """
-    return force_text(unquote_plus(force_str(quoted_url)))
+    return unquote_plus(quoted_url)
 
 
-def urlencode(query, doseq=0):
+def urlencode(query, doseq=False):
     """
-    A version of Python's urllib.urlencode() function that can operate on
+    A version of Python's urllib.parse.urlencode() function that can operate on
-    unicode strings. The parameters are first cast to UTF-8 encoded strings and
+    MultiValueDict and non-string values.
-    then encoded as per normal.
     """
     if isinstance(query, MultiValueDict):
         query = query.lists()
     elif hasattr(query, 'items'):
         query = query.items()
     return original_urlencode(
-        [(force_str(k),
+        [(k, [str(i) for i in v] if isinstance(v, (list, tuple)) else str(v))
-         [force_str(i) for i in v] if isinstance(v, (list, tuple)) else force_str(v))
+         for k, v in query],
-            for k, v in query],
+        doseq
-        doseq)
+    )
 
 
 def cookie_date(epoch_seconds=None):

django/views/i18n.py

@@ -1,6 +1,7 @@
 import itertools
 import json
 import os
+from urllib.parse import unquote
 
 from django import http
 from django.apps import apps
@@ -9,7 +10,7 @@ from django.template import Context, Engine
 from django.urls import translate_url
 from django.utils.encoding import force_text
 from django.utils.formats import get_format
-from django.utils.http import is_safe_url, urlunquote
+from django.utils.http import is_safe_url
 from django.utils.translation import (
     LANGUAGE_SESSION_KEY, check_for_language, get_language,
 )
@@ -35,7 +36,7 @@ def set_language(request):
             not is_safe_url(url=next, allowed_hosts={request.get_host()}, require_https=request.is_secure())):
         next = request.META.get('HTTP_REFERER')
         if next:
-            next = urlunquote(next)  # HTTP_REFERER may be encoded.
+            next = unquote(next)  # HTTP_REFERER may be encoded.
         if not is_safe_url(url=next, allowed_hosts={request.get_host()}, require_https=request.is_secure()):
             next = '/'
     response = http.HttpResponseRedirect(next) if next else http.HttpResponse(status=204)

docs/ref/unicode.txt

@@ -156,8 +156,8 @@ Django provides some assistance.
 * The function :func:`django.utils.encoding.iri_to_uri()` implements the
   conversion from IRI to URI as required by the specification (:rfc:`3987#section-3.1`).
 
-* The functions :func:`django.utils.http.urlquote()` and
+* The functions ``django.utils.http.urlquote()`` and
-  :func:`django.utils.http.urlquote_plus()` are versions of Python's standard
+  ``django.utils.http.urlquote_plus()`` are versions of Python's standard
   ``urllib.quote()`` and ``urllib.quote_plus()`` that work with non-ASCII
   characters. (The data is converted to UTF-8 prior to encoding.)
 

docs/ref/urlresolvers.txt

@@ -70,9 +70,8 @@ use for reversing. By default, the root URLconf for the current thread is used.
         >>> reverse('cities', args=['Orléans'])
         '.../Orl%C3%A9ans/'
 
-    Applying further encoding (such as :meth:`~django.utils.http.urlquote` or
+    Applying further encoding (such as :func:`urllib.parse.quote`) to the output
-    ``urllib.quote``) to the output of ``reverse()`` may produce undesirable
+    of ``reverse()`` may produce undesirable results.
-    results.
 
 ``reverse_lazy()``
 ==================

docs/ref/utils.txt

@@ -684,27 +684,10 @@ escaping HTML.
 .. module:: django.utils.http
    :synopsis: HTTP helper functions. (URL encoding, cookie handling, ...)
 
-.. function:: urlquote(url, safe='/')
-
-    A version of Python's ``urllib.quote()`` function that can operate on
-    unicode strings. The url is first UTF-8 encoded before quoting. The
-    returned string can safely be used as part of an argument to a subsequent
-    ``iri_to_uri()`` call without double-quoting occurring. Employs lazy
-    execution.
-
-.. function:: urlquote_plus(url, safe='')
-
-    A version of Python's urllib.quote_plus() function that can operate on
-    unicode strings. The url is first UTF-8 encoded before quoting. The
-    returned string can safely be used as part of an argument to a subsequent
-    ``iri_to_uri()`` call without double-quoting occurring. Employs lazy
-    execution.
-
 .. function:: urlencode(query, doseq=0)
 
-    A version of Python's urllib.urlencode() function that can operate on
+    A version of Python's :func:`urllib.parse.urlencode` function that can
-    unicode strings. The parameters are first cast to UTF-8 encoded strings
+    operate on ``MultiValueDict`` and non-string values.
-    and then encoded as per normal.
 
 .. function:: cookie_date(epoch_seconds=None)
 

docs/releases/1.6.txt

@@ -585,7 +585,7 @@ be at the end of a line. If they are not, the comments are ignored and
 Quoting in ``reverse()``
 ------------------------
 
-When reversing URLs, Django didn't apply :func:`~django.utils.http.urlquote`
+When reversing URLs, Django didn't apply ``django.utils.http.urlquote``
 to arguments before interpolating them in URL patterns. This bug is fixed in
 Django 1.6. If you worked around this bug by applying URL quoting before
 passing arguments to ``reverse()``, this may result in double-quoting. If this

tests/auth_tests/test_views.py

@@ -3,7 +3,7 @@ import itertools
 import os
 import re
 from importlib import import_module
-from urllib.parse import ParseResult, urlparse
+from urllib.parse import ParseResult, quote, urlparse
 
 from django.apps import apps
 from django.conf import settings
@@ -28,7 +28,6 @@ from django.test.utils import patch_logger
 from django.urls import NoReverseMatch, reverse, reverse_lazy
 from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.encoding import force_text
-from django.utils.http import urlquote
 from django.utils.translation import LANGUAGE_SESSION_KEY
 
 from .client import PasswordResetConfirmClient
@@ -546,7 +545,7 @@ class LoginTest(AuthViewsTestCase):
             nasty_url = '%(url)s?%(next)s=%(bad_url)s' % {
                 'url': login_url,
                 'next': REDIRECT_FIELD_NAME,
-                'bad_url': urlquote(bad_url),
+                'bad_url': quote(bad_url),
             }
             response = self.client.post(nasty_url, {
                 'username': 'testclient',
@@ -568,7 +567,7 @@ class LoginTest(AuthViewsTestCase):
             safe_url = '%(url)s?%(next)s=%(good_url)s' % {
                 'url': login_url,
                 'next': REDIRECT_FIELD_NAME,
-                'good_url': urlquote(good_url),
+                'good_url': quote(good_url),
             }
             response = self.client.post(safe_url, {
                 'username': 'testclient',
@@ -583,7 +582,7 @@ class LoginTest(AuthViewsTestCase):
         not_secured_url = '%(url)s?%(next)s=%(next_url)s' % {
             'url': login_url,
             'next': REDIRECT_FIELD_NAME,
-            'next_url': urlquote(non_https_next_url),
+            'next_url': quote(non_https_next_url),
         }
         post_data = {
             'username': 'testclient',
@@ -701,13 +700,13 @@ class LoginURLSettings(AuthViewsTestCase):
 
     @override_settings(LOGIN_URL='http://remote.example.com/login')
     def test_remote_login_url(self):
-        quoted_next = urlquote('http://testserver/login_required/')
+        quoted_next = quote('http://testserver/login_required/')
         expected = 'http://remote.example.com/login?next=%s' % quoted_next
         self.assertLoginURLEquals(expected)
 
     @override_settings(LOGIN_URL='https:///login/')
     def test_https_login_url(self):
-        quoted_next = urlquote('http://testserver/login_required/')
+        quoted_next = quote('http://testserver/login_required/')
         expected = 'https:///login/?next=%s' % quoted_next
         self.assertLoginURLEquals(expected)
 
@@ -717,7 +716,7 @@ class LoginURLSettings(AuthViewsTestCase):
 
     @override_settings(LOGIN_URL='http://remote.example.com/login/?next=/default/')
     def test_remote_login_url_with_next_querystring(self):
-        quoted_next = urlquote('http://testserver/login_required/')
+        quoted_next = quote('http://testserver/login_required/')
         expected = 'http://remote.example.com/login/?next=%s' % quoted_next
         self.assertLoginURLEquals(expected)
 
@@ -973,7 +972,7 @@ class LogoutTest(AuthViewsTestCase):
             nasty_url = '%(url)s?%(next)s=%(bad_url)s' % {
                 'url': logout_url,
                 'next': REDIRECT_FIELD_NAME,
-                'bad_url': urlquote(bad_url),
+                'bad_url': quote(bad_url),
             }
             self.login()
             response = self.client.get(nasty_url)
@@ -994,7 +993,7 @@ class LogoutTest(AuthViewsTestCase):
             safe_url = '%(url)s?%(next)s=%(good_url)s' % {
                 'url': logout_url,
                 'next': REDIRECT_FIELD_NAME,
-                'good_url': urlquote(good_url),
+                'good_url': quote(good_url),
             }
             self.login()
             response = self.client.get(safe_url)
@@ -1008,7 +1007,7 @@ class LogoutTest(AuthViewsTestCase):
         url = '%(url)s?%(next)s=%(next_url)s' % {
             'url': logout_url,
             'next': REDIRECT_FIELD_NAME,
-            'next_url': urlquote(non_https_next_url),
+            'next_url': quote(non_https_next_url),
         }
         self.login()
         response = self.client.get(url, secure=True)

tests/contenttypes_tests/models.py

@@ -1,10 +1,11 @@
+from urllib.parse import quote
+
 from django.contrib.contenttypes.fields import (
     GenericForeignKey, GenericRelation,
 )
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.sites.models import SiteManager
 from django.db import models
-from django.utils.http import urlquote
 
 
 class Site(models.Model):
@@ -72,7 +73,7 @@ class FooWithUrl(FooWithoutUrl):
     """
 
     def get_absolute_url(self):
-        return "/users/%s/" % urlquote(self.name)
+        return "/users/%s/" % quote(self.name)
 
 
 class FooWithBrokenAbsoluteUrl(FooWithoutUrl):
@@ -126,4 +127,4 @@ class ModelWithNullFKToSite(models.Model):
         return self.title
 
     def get_absolute_url(self):
-        return '/title/%s/' % urlquote(self.title)
+        return '/title/%s/' % quote(self.title)

tests/file_uploads/tests.py

@@ -7,13 +7,13 @@ import sys
 import tempfile as sys_tempfile
 import unittest
 from io import BytesIO, StringIO
+from urllib.parse import quote
 
 from django.core.files import temp as tempfile
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.http.multipartparser import MultiPartParser, parse_header
 from django.test import SimpleTestCase, TestCase, client, override_settings
 from django.utils.encoding import force_bytes
-from django.utils.http import urlquote
 
 from . import uploadhandler
 from .models import FileModel
@@ -127,7 +127,7 @@ class FileUploadTests(TestCase):
         payload = client.FakePayload()
         payload.write('\r\n'.join([
             '--' + client.BOUNDARY,
-            'Content-Disposition: form-data; name="file_unicode"; filename*=UTF-8\'\'%s' % urlquote(UNICODE_FILENAME),
+            'Content-Disposition: form-data; name="file_unicode"; filename*=UTF-8\'\'%s' % quote(UNICODE_FILENAME),
             'Content-Type: application/octet-stream',
             '',
             'You got pwnd.\r\n',
@@ -153,7 +153,7 @@ class FileUploadTests(TestCase):
         payload.write(
             '\r\n'.join([
                 '--' + client.BOUNDARY,
-                'Content-Disposition: form-data; name*=UTF-8\'\'file_unicode; filename*=UTF-8\'\'%s' % urlquote(
+                'Content-Disposition: form-data; name*=UTF-8\'\'file_unicode; filename*=UTF-8\'\'%s' % quote(
                     UNICODE_FILENAME
                 ),
                 'Content-Type: application/octet-stream',

tests/requests/tests.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from http import cookies
 from io import BytesIO
 from itertools import chain
-from urllib.parse import urlencode as original_urlencode
+from urllib.parse import urlencode
 
 from django.core.exceptions import SuspiciousOperation
 from django.core.handlers.wsgi import LimitedStream, WSGIRequest
@@ -14,7 +14,7 @@ from django.http.request import split_domain_port
 from django.test import RequestFactory, SimpleTestCase, override_settings
 from django.test.client import FakePayload
 from django.test.utils import freeze_time
-from django.utils.http import cookie_date, urlencode
+from django.utils.http import cookie_date
 from django.utils.timezone import utc
 
 
@@ -379,7 +379,7 @@ class RequestsTests(SimpleTestCase):
         """
         Test a POST with non-utf-8 payload encoding.
         """
-        payload = FakePayload(original_urlencode({'key': 'España'.encode('latin-1')}))
+        payload = FakePayload(urlencode({'key': 'España'.encode('latin-1')}))
         request = WSGIRequest({
             'REQUEST_METHOD': 'POST',
             'CONTENT_LENGTH': len(payload),

tests/servers/tests.py

@@ -5,10 +5,10 @@ import errno
 import os
 import socket
 from urllib.error import HTTPError
+from urllib.parse import urlencode
 from urllib.request import urlopen
 
 from django.test import LiveServerTestCase, override_settings
-from django.utils.http import urlencode
 
 from .models import Person
 

tests/utils_tests/test_encoding.py

@@ -1,12 +1,12 @@
 import datetime
 import unittest
+from urllib.parse import quote_plus
 
 from django.utils.encoding import (
     escape_uri_path, filepath_to_uri, force_bytes, force_text, iri_to_uri,
     smart_text, uri_to_iri,
 )
 from django.utils.functional import SimpleLazyObject
-from django.utils.http import urlquote_plus
 
 
 class TestEncodingUtils(unittest.TestCase):
@@ -72,7 +72,7 @@ class TestRFC3987IEncodingUtils(unittest.TestCase):
             # Valid UTF-8 sequences are encoded.
             ('red%09rosé#red', 'red%09ros%C3%A9#red'),
             ('/blog/for/Jürgen Münster/', '/blog/for/J%C3%BCrgen%20M%C3%BCnster/'),
-            ('locations/%s' % urlquote_plus('Paris & Orléans'), 'locations/Paris+%26+Orl%C3%A9ans'),
+            ('locations/%s' % quote_plus('Paris & Orléans'), 'locations/Paris+%26+Orl%C3%A9ans'),
 
             # Reserved chars remain unescaped.
             ('%&', '%&'),