Refs #23919 -- Removed default 'utf-8' argument for str.encode()/decode().

This commit is contained in:
Tim Graham 2017-02-07 12:05:47 -05:00
parent 21f13ff5b3
commit 500532c95d
32 changed files with 61 additions and 65 deletions

View File

@ -173,7 +173,7 @@ class CommonPasswordValidator:
def __init__(self, password_list_path=DEFAULT_PASSWORD_LIST_PATH):
try:
with gzip.open(password_list_path) as f:
common_passwords_lines = f.read().decode('utf-8').splitlines()
common_passwords_lines = f.read().decode().splitlines()
except IOError:
with open(password_list_path) as f:
common_passwords_lines = f.readlines()

View File

@ -381,7 +381,7 @@ class ManifestFilesMixin(HashedFilesMixin):
def read_manifest(self):
try:
with self.open(self.manifest_name) as manifest:
return manifest.read().decode('utf-8')
return manifest.read().decode()
except IOError:
return None
@ -411,7 +411,7 @@ class ManifestFilesMixin(HashedFilesMixin):
payload = {'paths': self.hashed_files, 'version': self.manifest_version}
if self.exists(self.manifest_name):
self.delete(self.manifest_name)
contents = json.dumps(payload).encode('utf-8')
contents = json.dumps(payload).encode()
self._save(self.manifest_name, ContentFile(contents))
def stored_name(self, name):

View File

@ -11,8 +11,6 @@ from django.urls import set_script_prefix
from django.utils.encoding import force_text, repercent_broken_unicode
from django.utils.functional import cached_property
ISO_8859_1, UTF_8 = 'iso-8859-1', 'utf-8'
_slashes_re = re.compile(br'/+')
@ -168,7 +166,7 @@ def get_path_info(environ):
"""
path_info = get_bytes_from_wsgi(environ, 'PATH_INFO', '/')
return repercent_broken_unicode(path_info).decode(UTF_8)
return repercent_broken_unicode(path_info).decode()
def get_script_name(environ):
@ -201,7 +199,7 @@ def get_script_name(environ):
else:
script_name = get_bytes_from_wsgi(environ, 'SCRIPT_NAME', '')
return script_name.decode(UTF_8)
return script_name.decode()
def get_bytes_from_wsgi(environ, key, default):
@ -214,7 +212,7 @@ def get_bytes_from_wsgi(environ, key, default):
# Non-ASCII values in the WSGI environ are arbitrarily decoded with
# ISO-8859-1. This is wrong for Django websites where UTF-8 is the default.
# Re-encode to recover the original bytestring.
return value.encode(ISO_8859_1)
return value.encode('iso-8859-1')
def get_str_from_wsgi(environ, key, default):
@ -224,4 +222,4 @@ def get_str_from_wsgi(environ, key, default):
key and default should be str objects.
"""
value = get_bytes_from_wsgi(environ, key, default)
return value.decode(UTF_8, errors='replace')
return value.decode(errors='replace')

View File

@ -172,7 +172,7 @@ class SafeMIMEText(MIMEMixin, MIMEText):
def set_payload(self, payload, charset=None):
if charset == 'utf-8':
has_long_lines = any(
len(l.encode('utf-8')) > RFC5322_EMAIL_LINE_LENGTH_LIMIT
len(l.encode()) > RFC5322_EMAIL_LINE_LENGTH_LIMIT
for l in payload.splitlines()
)
# Quoted-Printable encoding has the side effect of shortening long
@ -322,7 +322,7 @@ class EmailMessage:
if basetype == 'text':
if isinstance(content, bytes):
try:
content = content.decode('utf-8')
content = content.decode()
except UnicodeDecodeError:
# If mimetype suggests the file is text but it's
# actually binary, read() raises a UnicodeDecodeError.

View File

@ -70,7 +70,7 @@ def Deserializer(stream_or_string, **options):
if not isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
stream_or_string = stream_or_string.decode()
try:
objects = json.loads(stream_or_string)
for obj in PythonDeserializer(objects, **options):

View File

@ -68,7 +68,7 @@ def Deserializer(stream_or_string, **options):
Deserialize a stream or string of YAML data.
"""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
stream_or_string = stream_or_string.decode()
if isinstance(stream_or_string, str):
stream = StringIO(stream_or_string)
else:

View File

@ -224,7 +224,7 @@ class DatabaseOperations(BaseDatabaseOperations):
# http://initd.org/psycopg/docs/cursor.html#cursor.query
# The query attribute is a Psycopg extension to the DB API 2.0.
if cursor.query is not None:
return cursor.query.decode('utf-8')
return cursor.query.decode()
return None
def return_insert_id(self):

View File

@ -31,7 +31,7 @@ def decoder(conv_func):
This function converts the received value to a regular string before
passing it to the receiver function.
"""
return lambda s: conv_func(s.decode('utf-8'))
return lambda s: conv_func(s.decode())
Database.register_converter("bool", decoder(lambda s: s == '1'))

View File

@ -10,7 +10,7 @@ from urllib.parse import urljoin, urlparse, urlsplit
from django.conf import settings
from django.core.handlers.base import BaseHandler
from django.core.handlers.wsgi import ISO_8859_1, UTF_8, WSGIRequest
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import (
got_request_exception, request_finished, request_started,
)
@ -320,11 +320,11 @@ class RequestFactory:
# If there are parameters, add them
if parsed.params:
path += ";" + parsed.params
path = uri_to_iri(path).encode(UTF_8)
path = uri_to_iri(path).encode()
# Replace the behavior where non-ASCII values in the WSGI environ are
# arbitrarily decoded with ISO-8859-1.
# Refs comment in `get_bytes_from_wsgi()`.
return path.decode(ISO_8859_1)
return path.decode('iso-8859-1')
def get(self, path, data=None, secure=False, **extra):
"Construct a GET request."

View File

@ -63,11 +63,9 @@ def get_random_string(length=12,
# is better than absolute predictability.
random.seed(
hashlib.sha256(
("%s%s%s" % (
random.getstate(),
time.time(),
settings.SECRET_KEY)).encode('utf-8')
).digest())
('%s%s%s' % (random.getstate(), time.time(), settings.SECRET_KEY)).encode()
).digest()
)
return ''.join(random.choice(allowed_chars) for i in range(length))

View File

@ -165,7 +165,7 @@ def uri_to_iri(uri):
return uri
uri = force_bytes(uri)
iri = unquote_to_bytes(uri)
return repercent_broken_unicode(iri).decode('utf-8')
return repercent_broken_unicode(iri).decode()
def escape_uri_path(path):
@ -192,7 +192,7 @@ def repercent_broken_unicode(path):
strictly legal UTF-8 octet sequence.
"""
try:
path.decode('utf-8')
path.decode()
except UnicodeDecodeError as e:
repercent = quote(path[e.start:e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
path = repercent_broken_unicode(

View File

@ -111,7 +111,7 @@ def lazy(func, *resultclasses):
return bytes(func(*self.__args, **self.__kw))
def __bytes_cast_encoded(self):
return func(*self.__args, **self.__kw).encode('utf-8')
return func(*self.__args, **self.__kw).encode()
def __cast(self):
if self._delegate_bytes:

View File

@ -886,13 +886,13 @@ class BaseCacheTests:
get_cache_data = fetch_middleware.process_request(request)
self.assertIsNotNone(get_cache_data)
self.assertEqual(get_cache_data.content, content.encode('utf-8'))
self.assertEqual(get_cache_data.content, content.encode())
self.assertEqual(get_cache_data.cookies, response.cookies)
update_middleware.process_response(request, get_cache_data)
get_cache_data = fetch_middleware.process_request(request)
self.assertIsNotNone(get_cache_data)
self.assertEqual(get_cache_data.content, content.encode('utf-8'))
self.assertEqual(get_cache_data.content, content.encode())
self.assertEqual(get_cache_data.cookies, response.cookies)
def test_add_fail_on_pickleerror(self):

View File

@ -97,7 +97,7 @@ def file_upload_echo_content(request):
"""
def read_and_close(f):
with f:
return f.read().decode('utf-8')
return f.read().decode()
r = {k: read_and_close(f) for k, f in request.FILES.items()}
return JsonResponse(r)

View File

@ -135,7 +135,7 @@ class FileTests(unittest.TestCase):
def test_io_wrapper(self):
content = "vive l'été\n"
with tempfile.TemporaryFile() as temp, File(temp, name='something.txt') as test_file:
test_file.write(content.encode('utf-8'))
test_file.write(content.encode())
test_file.seek(0)
wrapper = TextIOWrapper(test_file, 'utf-8', newline='\n')
self.assertEqual(wrapper.read(), content)
@ -144,7 +144,7 @@ class FileTests(unittest.TestCase):
self.assertEqual(wrapper.read(), content * 2)
test_file = wrapper.detach()
test_file.seek(0)
self.assertEqual(test_file.read(), (content * 2).encode('utf-8'))
self.assertEqual(test_file.read(), (content * 2).encode())
class NoNameFileTestCase(unittest.TestCase):

View File

@ -33,7 +33,7 @@ class FileFieldTest(SimpleTestCase):
f.clean(SimpleUploadedFile('name', b''))
self.assertEqual(SimpleUploadedFile, type(f.clean(SimpleUploadedFile('name', b'Some File Content'))))
self.assertIsInstance(
f.clean(SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))),
f.clean(SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode())),
SimpleUploadedFile
)
self.assertIsInstance(

View File

@ -2421,7 +2421,7 @@ Password: <input type="password" name="password" required />
)
self.assertTrue(f.is_valid())
file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))
file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode())
f = FileForm(data={}, files={'file1': file1}, auto_id=False)
self.assertHTMLEqual(
f.as_table(),

View File

@ -171,7 +171,7 @@ class ModelFormCallableModelDefault(TestCase):
class FormsModelTestCase(TestCase):
def test_unicode_filename(self):
# FileModel with unicode filename and data #########################
file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode('utf-8'))
file1 = SimpleUploadedFile('我隻氣墊船裝滿晒鱔.txt', 'मेरी मँडराने वाली नाव सर्पमीनों से भरी ह'.encode())
f = FileForm(data={}, files={'file1': file1}, auto_id=False)
self.assertTrue(f.is_valid())
self.assertIn('file1', f.cleaned_data)

View File

@ -317,7 +317,7 @@ class HttpResponseTests(unittest.TestCase):
with self.assertRaises(UnicodeError):
r.__setitem__('føø', 'bar')
with self.assertRaises(UnicodeError):
r.__setitem__('føø'.encode('utf-8'), 'bar')
r.__setitem__('føø'.encode(), 'bar')
def test_long_line(self):
# Bug #20889: long lines trigger newlines to be added to headers
@ -371,7 +371,7 @@ class HttpResponseTests(unittest.TestCase):
# test odd inputs
r = HttpResponse()
r.content = ['1', '2', 3, '\u079e']
# '\xde\x9e' == unichr(1950).encode('utf-8')
# '\xde\x9e' == unichr(1950).encode()
self.assertEqual(r.content, b'123\xde\x9e')
# .content can safely be accessed multiple times.
@ -573,7 +573,7 @@ class StreamingHttpResponseTests(SimpleTestCase):
# iterating over strings still yields bytestring chunks.
r.streaming_content = iter(['hello', 'café'])
chunks = list(r)
# '\xc3\xa9' == unichr(233).encode('utf-8')
# '\xc3\xa9' == unichr(233).encode()
self.assertEqual(chunks, [b'hello', b'caf\xc3\xa9'])
for chunk in chunks:
self.assertIsInstance(chunk, bytes)

View File

@ -447,7 +447,7 @@ args=(sys.stdout,)
format=%(message)s
"""
self.temp_file = NamedTemporaryFile()
self.temp_file.write(logging_conf.encode('utf-8'))
self.temp_file.write(logging_conf.encode())
self.temp_file.flush()
sdict = {'LOGGING_CONFIG': '"logging.config.fileConfig"',
'LOGGING': 'r"%s"' % self.temp_file.name}

View File

@ -1147,7 +1147,7 @@ class FakeSMTPServer(smtpd.SMTPServer, threading.Thread):
self.sink_lock = threading.Lock()
def process_message(self, peer, mailfrom, rcpttos, data):
data = data.encode('utf-8')
data = data.encode()
m = message_from_bytes(data)
maddr = parseaddr(m.get('from'))[1]
@ -1419,7 +1419,7 @@ class SMTPBackendTests(BaseEmailBackendTests, SMTPBackendTestsBase):
self.assertTrue(msg)
msg = msg.decode('utf-8')
msg = msg.decode()
# The message only contains CRLF and not combinations of CRLF, LF, and CR.
msg = msg.replace('\r\n', '')
self.assertNotIn('\r', msg)

View File

@ -785,7 +785,7 @@ class GZipMiddlewareTest(SimpleTestCase):
r = GZipMiddleware().process_response(self.req, self.stream_resp_unicode)
self.assertEqual(
self.decompress(b''.join(r)),
b''.join(x.encode('utf-8') for x in self.sequence_unicode)
b''.join(x.encode() for x in self.sequence_unicode)
)
self.assertEqual(r.get('Content-Encoding'), 'gzip')
self.assertFalse(r.has_header('Content-Length'))

View File

@ -144,7 +144,7 @@ class SerializersTestBase:
if isinstance(stream, StringIO):
self.assertEqual(string_data, stream.getvalue())
else:
self.assertEqual(string_data, stream.content.decode('utf-8'))
self.assertEqual(string_data, stream.content.decode())
def test_serialize_specific_fields(self):
obj = ComplexModel(field1='first', field2='second', field3='third')

View File

@ -81,7 +81,7 @@ class WSGIRequestHandlerTestCase(SimpleTestCase):
'%s:%s' % (k, v) for k, v in environ.items()
if k.startswith('HTTP_')
)
yield (','.join(http_environ_items)).encode('utf-8')
yield (','.join(http_environ_items)).encode()
rfile = BytesIO()
rfile.write(b"GET / HTTP/1.0\r\n")

View File

@ -14,7 +14,7 @@ class TestSigner(SimpleTestCase):
for s in (
b'hello',
b'3098247:529:087:',
'\u2019'.encode('utf-8'),
'\u2019'.encode(),
):
self.assertEqual(
signer.signature(s),

View File

@ -18,4 +18,4 @@ class GenericViewsSitemapTests(SitemapTestsBase):
%s
</urlset>
""" % expected
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)

View File

@ -25,7 +25,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
@override_settings(TEMPLATES=[{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
@ -40,7 +40,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_simple_sitemap_section(self):
"A simple sitemap section can be rendered"
@ -50,7 +50,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_simple_sitemap(self):
"A simple sitemap can be rendered"
@ -60,7 +60,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
@override_settings(TEMPLATES=[{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
@ -75,7 +75,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_sitemap_last_modified(self):
"Last-Modified header is set correctly"
@ -162,7 +162,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<url><loc>http://testserver/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % date.today()
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
@skipUnless(apps.is_installed('django.contrib.sites'),
"django.contrib.sites app not installed.")
@ -207,7 +207,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<sitemap><loc>%s/cached/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_x_robots_sitemap(self):
response = self.client.get('/simple/index.xml')
@ -229,7 +229,7 @@ class HTTPSitemapTests(SitemapTestsBase):
<url><loc>{0}/en/i18n/testmodel/{1}/</loc><changefreq>never</changefreq><priority>0.5</priority></url><url><loc>{0}/pt/i18n/testmodel/{1}/</loc><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""".format(self.base_url, self.i18n_model.pk)
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_sitemap_without_entries(self):
response = self.client.get('/sitemap-without-entries/sitemap.xml')
@ -237,4 +237,4 @@ class HTTPSitemapTests(SitemapTestsBase):
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
</urlset>"""
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)

View File

@ -17,7 +17,7 @@ class HTTPSSitemapTests(SitemapTestsBase):
<sitemap><loc>%s/secure/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_secure_sitemap_section(self):
"A secure sitemap section can be rendered"
@ -27,7 +27,7 @@ class HTTPSSitemapTests(SitemapTestsBase):
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url, date.today())
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
@override_settings(SECURE_PROXY_SSL_HEADER=False)
@ -42,7 +42,7 @@ class HTTPSDetectionSitemapTests(SitemapTestsBase):
<sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
</sitemapindex>
""" % self.base_url.replace('http://', 'https://')
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)
def test_sitemap_section_with_https_request(self):
"A sitemap section requested in HTTPS is rendered with HTTPS links"
@ -52,4 +52,4 @@ class HTTPSDetectionSitemapTests(SitemapTestsBase):
<url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
</urlset>
""" % (self.base_url.replace('http://', 'https://'), date.today())
self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
self.assertXMLEqual(response.content.decode(), expected_content)

View File

@ -131,14 +131,14 @@ class AssertContainsTests(SimpleTestCase):
# Regression test for #10183
r = self.client.get('/check_unicode/')
self.assertContains(r, 'さかき')
self.assertContains(r, b'\xe5\xb3\xa0'.decode('utf-8'))
self.assertContains(r, b'\xe5\xb3\xa0'.decode())
def test_unicode_not_contains(self):
"Unicode characters can be searched for, and not found in template context"
# Regression test for #10183
r = self.client.get('/check_unicode/')
self.assertNotContains(r, 'はたけ')
self.assertNotContains(r, b'\xe3\x81\xaf\xe3\x81\x9f\xe3\x81\x91'.decode('utf-8'))
self.assertNotContains(r, b'\xe3\x81\xaf\xe3\x81\x9f\xe3\x81\x91'.decode())
def test_binary_contains(self):
r = self.client.get('/check_binary/')
@ -1277,7 +1277,7 @@ class UnicodePayloadTests(SimpleTestCase):
# Regression test for #10571
json = '{"dog": "собака"}'
response = self.client.post("/parse_unicode_json/", json, content_type="application/json; charset=utf-8")
self.assertEqual(response.content, json.encode('utf-8'))
self.assertEqual(response.content, json.encode())
def test_unicode_payload_utf16(self):
"A non-ASCII unicode data encoded as UTF-16 can be POSTed"

View File

@ -42,7 +42,7 @@ class TestEncodingUtils(SimpleTestCase):
"""
error_msg = "This is an exception, voilà"
exc = ValueError(error_msg)
self.assertEqual(force_bytes(exc), error_msg.encode('utf-8'))
self.assertEqual(force_bytes(exc), error_msg.encode())
self.assertEqual(force_bytes(exc, encoding='ascii', errors='ignore'), b'This is an exception, voil')
def test_force_bytes_strings_only(self):

View File

@ -214,7 +214,7 @@ class TestUtilsText(SimpleTestCase):
def test_compress_sequence(self):
data = [{'key': i} for i in range(10)]
seq = list(json.JSONEncoder().iterencode(data))
seq = [s.encode('utf-8') for s in seq]
seq = [s.encode() for s in seq]
actual_length = len(b''.join(seq))
out = text.compress_sequence(seq)
compressed_length = len(b''.join(out))

View File

@ -214,7 +214,7 @@ class JsI18NTests(SimpleTestCase):
"""
with override('de'):
response = self.client.get('/jsoni18n/')
data = json.loads(response.content.decode('utf-8'))
data = json.loads(response.content.decode())
self.assertIn('catalog', data)
self.assertIn('formats', data)
self.assertIn('plural', data)
@ -243,7 +243,7 @@ class JsI18NTests(SimpleTestCase):
"""
with self.settings(LANGUAGE_CODE='es'), override('en-us'):
response = self.client.get('/jsoni18n/')
data = json.loads(response.content.decode('utf-8'))
data = json.loads(response.content.decode())
self.assertIn('catalog', data)
self.assertIn('formats', data)
self.assertIn('plural', data)