diff --git a/django/http/__init__.py b/django/http/__init__.py index ed4a23928f..62415da3c6 100644 --- a/django/http/__init__.py +++ b/django/http/__init__.py @@ -113,7 +113,7 @@ def build_request_repr(request, path_override=None, GET_override=None, get = (pformat(GET_override) if GET_override is not None else pformat(request.GET)) - except: + except Exception: get = '' if request._post_parse_error: post = '' @@ -122,19 +122,19 @@ def build_request_repr(request, path_override=None, GET_override=None, post = (pformat(POST_override) if POST_override is not None else pformat(request.POST)) - except: + except Exception: post = '' try: cookies = (pformat(COOKIES_override) if COOKIES_override is not None else pformat(request.COOKIES)) - except: + except Exception: cookies = '' try: meta = (pformat(META_override) if META_override is not None else pformat(request.META)) - except: + except Exception: meta = '' path = path_override if path_override is not None else request.path return smart_str('<%s\npath:%s,\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % @@ -177,14 +177,14 @@ class HttpRequest(object): # Reconstruct the host using the algorithm from PEP 333. host = self.META['SERVER_NAME'] server_port = str(self.META['SERVER_PORT']) - if server_port != (self.is_secure() and '443' or '80'): + if server_port != ('443' if self.is_secure() else '80'): host = '%s:%s' % (host, server_port) return host def get_full_path(self): # RFC 3986 requires query string arguments to be in the ASCII range. # Rather than crash if this doesn't happen, we encode defensively. - return '%s%s' % (self.path, self.META.get('QUERY_STRING', '') and ('?' + iri_to_uri(self.META.get('QUERY_STRING', ''))) or '') + return '%s%s' % (self.path, ('?' + iri_to_uri(self.META.get('QUERY_STRING', ''))) if self.META.get('QUERY_STRING', '') else '') def get_signed_cookie(self, key, default=RAISE_ERROR, salt='', max_age=None): """ @@ -218,7 +218,7 @@ class HttpRequest(object): if not location: location = self.get_full_path() if not absolute_http_url_re.match(location): - current_uri = '%s://%s%s' % (self.is_secure() and 'https' or 'http', + current_uri = '%s://%s%s' % ('https' if self.is_secure() else 'http', self.get_host(), self.path) location = urljoin(current_uri, location) return iri_to_uri(location) @@ -281,7 +281,7 @@ class HttpRequest(object): """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" self.upload_handlers = ImmutableList( self.upload_handlers, - warning = "You cannot alter upload handlers after the upload has been processed." + warning="You cannot alter upload handlers after the upload has been processed." ) parser = MultiPartParser(META, post_data, self.upload_handlers, self.encoding) return parser.parse() @@ -294,7 +294,7 @@ class HttpRequest(object): try: self._body = self.read() except IOError as e: - six.reraise(UnreadablePostError, UnreadablePostError(*tuple(e.args)), sys.exc_info()[2]) + six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2]) self._stream = BytesIO(self._body) return self._body @@ -360,6 +360,7 @@ class HttpRequest(object): if not buf: break yield buf + __iter__ = xreadlines def readlines(self): @@ -384,9 +385,16 @@ class QueryDict(MultiValueDict): if not encoding: encoding = settings.DEFAULT_CHARSET self.encoding = encoding - for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True - self.appendlist(force_text(key, encoding, errors='replace'), - force_text(value, encoding, errors='replace')) + if six.PY3: + for key, value in parse_qsl(query_string or '', + keep_blank_values=True, + encoding=encoding): + self.appendlist(key, value) + else: + for key, value in parse_qsl(query_string or '', + keep_blank_values=True): + self.appendlist(force_text(key, encoding, errors='replace'), + force_text(value, encoding, errors='replace')) self._mutable = mutable def _get_encoding(self): @@ -405,8 +413,8 @@ class QueryDict(MultiValueDict): def __setitem__(self, key, value): self._assert_mutable() - key = str_to_unicode(key, self.encoding) - value = str_to_unicode(value, self.encoding) + key = bytes_to_text(key, self.encoding) + value = bytes_to_text(value, self.encoding) super(QueryDict, self).__setitem__(key, value) def __delitem__(self, key): @@ -428,8 +436,8 @@ class QueryDict(MultiValueDict): def setlist(self, key, list_): self._assert_mutable() - key = str_to_unicode(key, self.encoding) - list_ = [str_to_unicode(elt, self.encoding) for elt in list_] + key = bytes_to_text(key, self.encoding) + list_ = [bytes_to_text(elt, self.encoding) for elt in list_] super(QueryDict, self).setlist(key, list_) def setlistdefault(self, key, default_list=None): @@ -438,8 +446,8 @@ class QueryDict(MultiValueDict): def appendlist(self, key, value): self._assert_mutable() - key = str_to_unicode(key, self.encoding) - value = str_to_unicode(value, self.encoding) + key = bytes_to_text(key, self.encoding) + value = bytes_to_text(value, self.encoding) super(QueryDict, self).appendlist(key, value) def pop(self, key, *args): @@ -456,8 +464,8 @@ class QueryDict(MultiValueDict): def setdefault(self, key, default=None): self._assert_mutable() - key = str_to_unicode(key, self.encoding) - default = str_to_unicode(default, self.encoding) + key = bytes_to_text(key, self.encoding) + default = bytes_to_text(default, self.encoding) return super(QueryDict, self).setdefault(key, default) def copy(self): @@ -531,6 +539,7 @@ class HttpResponse(object): if not content_type: content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, self._charset) + # content is a bytestring. See _get_content / _set_content. self.content = content self.cookies = SimpleCookie() if status: @@ -538,30 +547,40 @@ class HttpResponse(object): self['Content-Type'] = content_type - def __str__(self): - """Full HTTP message, including headers.""" - return '\n'.join(['%s: %s' % (key, value) - for key, value in self._headers.values()]) \ - + '\n\n' + self.content + def serialize(self): + """Full HTTP message, including headers, as a bytestring.""" + headers = [ + b'%s: %s' % (key.encode('us-ascii'), value.encode('us-ascii')) + for key, value in self._headers.values() + ] + return b'\r\n'.join(headers) + b'\r\n\r\n' + self.content + + if six.PY3: + __bytes__ = serialize + else: + __str__ = serialize def _convert_to_ascii(self, *values): """Converts all values to ascii strings.""" for value in values: - if isinstance(value, six.text_type): - try: - if not six.PY3: - value = value.encode('us-ascii') - else: - # In Python 3, use a string in headers, - # but ensure in only contains ASCII characters. - value.encode('us-ascii') - except UnicodeError as e: - e.reason += ', HTTP response headers must be in US-ASCII format' - raise - else: + if not isinstance(value, six.string_types): value = str(value) + try: + if six.PY3: + # Ensure string only contains ASCII + value.encode('us-ascii') + else: + if isinstance(value, str): + # Ensure string only contains ASCII + value.decode('us-ascii') + else: + # Convert unicode to an ASCII string + value = value.encode('us-ascii') + except UnicodeError as e: + e.reason += ', HTTP response headers must be in US-ASCII format' + raise if '\n' in value or '\r' in value: - raise BadHeaderError("Header values can't contain newlines (got %r)" % (value)) + raise BadHeaderError("Header values can't contain newlines (got %r)" % value) yield value def __setitem__(self, header, value): @@ -652,11 +671,12 @@ class HttpResponse(object): def _get_content(self): if self.has_header('Content-Encoding'): - return b''.join([str(e) for e in self._container]) + # XXX this doesn't work under Python 3 when e is an integer (#18764) + return b''.join([bytes(e) for e in self._container]) return b''.join([smart_bytes(e, self._charset) for e in self._container]) def _set_content(self, value): - if hasattr(value, '__iter__') and not isinstance(value, (bytes, six.text_type)): + if hasattr(value, '__iter__') and not isinstance(value, six.string_types): self._container = value self._base_content_is_iter = True else: @@ -673,7 +693,7 @@ class HttpResponse(object): chunk = next(self._iterator) if isinstance(chunk, six.text_type): chunk = chunk.encode(self._charset) - return str(chunk) + return bytes(chunk) next = __next__ # Python 2 compatibility @@ -743,8 +763,8 @@ def get_host(request): # It's neither necessary nor appropriate to use # django.utils.encoding.smart_text for parsing URLs and form inputs. Thus, -# this slightly more restricted function. -def str_to_unicode(s, encoding): +# this slightly more restricted function, used by QueryDict. +def bytes_to_text(s, encoding): """ Converts basestring objects to unicode, using the given encoding. Illegally encoded input characters are replaced with Unicode "unknown" codepoint diff --git a/tests/regressiontests/httpwrappers/tests.py b/tests/regressiontests/httpwrappers/tests.py index ad0ce4197e..332f25dfe9 100644 --- a/tests/regressiontests/httpwrappers/tests.py +++ b/tests/regressiontests/httpwrappers/tests.py @@ -9,16 +9,17 @@ from django.http import (QueryDict, HttpResponse, HttpResponseRedirect, HttpResponsePermanentRedirect, SimpleCookie, BadHeaderError, parse_cookie) +from django.utils import six from django.utils import unittest class QueryDictTests(unittest.TestCase): def test_missing_key(self): - q = QueryDict('') + q = QueryDict(str('')) self.assertRaises(KeyError, q.__getitem__, 'foo') def test_immutability(self): - q = QueryDict('') + q = QueryDict(str('')) self.assertRaises(AttributeError, q.__setitem__, 'something', 'bar') self.assertRaises(AttributeError, q.setlist, 'foo', ['bar']) self.assertRaises(AttributeError, q.appendlist, 'foo', ['bar']) @@ -28,26 +29,26 @@ class QueryDictTests(unittest.TestCase): self.assertRaises(AttributeError, q.clear) def test_immutable_get_with_default(self): - q = QueryDict('') + q = QueryDict(str('')) self.assertEqual(q.get('foo', 'default'), 'default') def test_immutable_basic_operations(self): - q = QueryDict('') + q = QueryDict(str('')) self.assertEqual(q.getlist('foo'), []) - self.assertEqual(q.has_key('foo'), False) + if not six.PY3: + self.assertEqual(q.has_key('foo'), False) self.assertEqual('foo' in q, False) - self.assertEqual(q.items(), []) - self.assertEqual(q.lists(), []) - self.assertEqual(q.items(), []) - self.assertEqual(q.keys(), []) - self.assertEqual(q.values(), []) + self.assertEqual(list(six.iteritems(q)), []) + self.assertEqual(list(six.iterlists(q)), []) + self.assertEqual(list(six.iterkeys(q)), []) + self.assertEqual(list(six.itervalues(q)), []) self.assertEqual(len(q), 0) self.assertEqual(q.urlencode(), '') def test_single_key_value(self): """Test QueryDict with one key/value pair""" - q = QueryDict('foo=bar') + q = QueryDict(str('foo=bar')) self.assertEqual(q['foo'], 'bar') self.assertRaises(KeyError, q.__getitem__, 'bar') self.assertRaises(AttributeError, q.__setitem__, 'something', 'bar') @@ -60,15 +61,17 @@ class QueryDictTests(unittest.TestCase): self.assertRaises(AttributeError, q.setlist, 'foo', ['bar']) self.assertRaises(AttributeError, q.appendlist, 'foo', ['bar']) - self.assertTrue(q.has_key('foo')) + if not six.PY3: + self.assertTrue(q.has_key('foo')) self.assertTrue('foo' in q) - self.assertFalse(q.has_key('bar')) + if not six.PY3: + self.assertFalse(q.has_key('bar')) self.assertFalse('bar' in q) - self.assertEqual(q.items(), [('foo', 'bar')]) - self.assertEqual(q.lists(), [('foo', ['bar'])]) - self.assertEqual(q.keys(), ['foo']) - self.assertEqual(q.values(), ['bar']) + self.assertEqual(list(six.iteritems(q)), [('foo', 'bar')]) + self.assertEqual(list(six.iterlists(q)), [('foo', ['bar'])]) + self.assertEqual(list(six.iterkeys(q)), ['foo']) + self.assertEqual(list(six.itervalues(q)), ['bar']) self.assertEqual(len(q), 1) self.assertRaises(AttributeError, q.update, {'foo': 'bar'}) @@ -80,30 +83,30 @@ class QueryDictTests(unittest.TestCase): self.assertEqual(q.urlencode(), 'foo=bar') def test_urlencode(self): - q = QueryDict('', mutable=True) + q = QueryDict(str(''), mutable=True) q['next'] = '/a&b/' self.assertEqual(q.urlencode(), 'next=%2Fa%26b%2F') self.assertEqual(q.urlencode(safe='/'), 'next=/a%26b/') - q = QueryDict('', mutable=True) + q = QueryDict(str(''), mutable=True) q['next'] = '/t\xebst&key/' self.assertEqual(q.urlencode(), 'next=%2Ft%C3%ABst%26key%2F') self.assertEqual(q.urlencode(safe='/'), 'next=/t%C3%ABst%26key/') def test_mutable_copy(self): """A copy of a QueryDict is mutable.""" - q = QueryDict('').copy() + q = QueryDict(str('')).copy() self.assertRaises(KeyError, q.__getitem__, "foo") q['name'] = 'john' self.assertEqual(q['name'], 'john') def test_mutable_delete(self): - q = QueryDict('').copy() + q = QueryDict(str('')).copy() q['name'] = 'john' del q['name'] self.assertFalse('name' in q) def test_basic_mutable_operations(self): - q = QueryDict('').copy() + q = QueryDict(str('')).copy() q['name'] = 'john' self.assertEqual(q.get('foo', 'default'), 'default') self.assertEqual(q.get('name', 'default'), 'john') @@ -117,13 +120,14 @@ class QueryDictTests(unittest.TestCase): q.appendlist('foo', 'another') self.assertEqual(q.getlist('foo'), ['bar', 'baz', 'another']) self.assertEqual(q['foo'], 'another') - self.assertTrue(q.has_key('foo')) + if not six.PY3: + self.assertTrue(q.has_key('foo')) self.assertTrue('foo' in q) - self.assertEqual(q.items(), [('foo', 'another'), ('name', 'john')]) - self.assertEqual(q.lists(), [('foo', ['bar', 'baz', 'another']), ('name', ['john'])]) - self.assertEqual(q.keys(), ['foo', 'name']) - self.assertEqual(q.values(), ['another', 'john']) + self.assertEqual(list(six.iteritems(q)), [('foo', 'another'), ('name', 'john')]) + self.assertEqual(list(six.iterlists(q)), [('foo', ['bar', 'baz', 'another']), ('name', ['john'])]) + self.assertEqual(list(six.iterkeys(q)), ['foo', 'name']) + self.assertEqual(list(six.itervalues(q)), ['another', 'john']) self.assertEqual(len(q), 2) q.update({'foo': 'hello'}) @@ -144,7 +148,7 @@ class QueryDictTests(unittest.TestCase): def test_multiple_keys(self): """Test QueryDict with two key/value pairs with same keys.""" - q = QueryDict('vote=yes&vote=no') + q = QueryDict(str('vote=yes&vote=no')) self.assertEqual(q['vote'], 'no') self.assertRaises(AttributeError, q.__setitem__, 'something', 'bar') @@ -158,14 +162,16 @@ class QueryDictTests(unittest.TestCase): self.assertRaises(AttributeError, q.setlist, 'foo', ['bar', 'baz']) self.assertRaises(AttributeError, q.appendlist, 'foo', ['bar']) - self.assertEqual(q.has_key('vote'), True) + if not six.PY3: + self.assertEqual(q.has_key('vote'), True) self.assertEqual('vote' in q, True) - self.assertEqual(q.has_key('foo'), False) + if not six.PY3: + self.assertEqual(q.has_key('foo'), False) self.assertEqual('foo' in q, False) - self.assertEqual(q.items(), [('vote', 'no')]) - self.assertEqual(q.lists(), [('vote', ['yes', 'no'])]) - self.assertEqual(q.keys(), ['vote']) - self.assertEqual(q.values(), ['no']) + self.assertEqual(list(six.iteritems(q)), [('vote', 'no')]) + self.assertEqual(list(six.iterlists(q)), [('vote', ['yes', 'no'])]) + self.assertEqual(list(six.iterkeys(q)), ['vote']) + self.assertEqual(list(six.itervalues(q)), ['no']) self.assertEqual(len(q), 1) self.assertRaises(AttributeError, q.update, {'foo': 'bar'}) @@ -175,45 +181,49 @@ class QueryDictTests(unittest.TestCase): self.assertRaises(AttributeError, q.setdefault, 'foo', 'bar') self.assertRaises(AttributeError, q.__delitem__, 'vote') - def test_invalid_input_encoding(self): - """ - QueryDicts must be able to handle invalid input encoding (in this - case, bad UTF-8 encoding). - """ - q = QueryDict(b'foo=bar&foo=\xff') - self.assertEqual(q['foo'], '\ufffd') - self.assertEqual(q.getlist('foo'), ['bar', '\ufffd']) + if not six.PY3: + def test_invalid_input_encoding(self): + """ + QueryDicts must be able to handle invalid input encoding (in this + case, bad UTF-8 encoding). + + This test doesn't apply under Python 3 because the URL is a string + and not a bytestring. + """ + q = QueryDict(str(b'foo=bar&foo=\xff')) + self.assertEqual(q['foo'], '\ufffd') + self.assertEqual(q.getlist('foo'), ['bar', '\ufffd']) def test_pickle(self): - q = QueryDict('') + q = QueryDict(str('')) q1 = pickle.loads(pickle.dumps(q, 2)) self.assertEqual(q == q1, True) - q = QueryDict('a=b&c=d') + q = QueryDict(str('a=b&c=d')) q1 = pickle.loads(pickle.dumps(q, 2)) self.assertEqual(q == q1, True) - q = QueryDict('a=b&c=d&a=1') + q = QueryDict(str('a=b&c=d&a=1')) q1 = pickle.loads(pickle.dumps(q, 2)) self.assertEqual(q == q1, True) def test_update_from_querydict(self): """Regression test for #8278: QueryDict.update(QueryDict)""" - x = QueryDict("a=1&a=2", mutable=True) - y = QueryDict("a=3&a=4") + x = QueryDict(str("a=1&a=2"), mutable=True) + y = QueryDict(str("a=3&a=4")) x.update(y) self.assertEqual(x.getlist('a'), ['1', '2', '3', '4']) def test_non_default_encoding(self): """#13572 - QueryDict with a non-default encoding""" - q = QueryDict(b'sbb=one', encoding='rot_13') - self.assertEqual(q.encoding , 'rot_13' ) - self.assertEqual(q.items() , [('foo', 'bar')] ) - self.assertEqual(q.urlencode() , 'sbb=one' ) + q = QueryDict(str('cur=%A4'), encoding='iso-8859-15') + self.assertEqual(q.encoding, 'iso-8859-15') + self.assertEqual(list(six.iteritems(q)), [('cur', '€')]) + self.assertEqual(q.urlencode(), 'cur=%A4') q = q.copy() - self.assertEqual(q.encoding , 'rot_13' ) - self.assertEqual(q.items() , [('foo', 'bar')] ) - self.assertEqual(q.urlencode() , 'sbb=one' ) - self.assertEqual(copy.copy(q).encoding , 'rot_13' ) - self.assertEqual(copy.deepcopy(q).encoding , 'rot_13') + self.assertEqual(q.encoding, 'iso-8859-15') + self.assertEqual(list(six.iteritems(q)), [('cur', '€')]) + self.assertEqual(q.urlencode(), 'cur=%A4') + self.assertEqual(copy.copy(q).encoding, 'iso-8859-15') + self.assertEqual(copy.deepcopy(q).encoding, 'iso-8859-15') class HttpResponseTests(unittest.TestCase): def test_unicode_headers(self): @@ -283,13 +293,8 @@ class HttpResponseTests(unittest.TestCase): #test retrieval explicitly using iter and odd inputs r = HttpResponse() r.content = ['1', '2', 3, '\u079e'] - result = [] my_iter = r.__iter__() - while True: - try: - result.append(next(my_iter)) - except StopIteration: - break + result = list(my_iter) #'\xde\x9e' == unichr(1950).encode('utf-8') self.assertEqual(result, ['1', '2', '3', b'\xde\x9e']) self.assertEqual(r.content, b'123\xde\x9e') diff --git a/tests/regressiontests/wsgi/tests.py b/tests/regressiontests/wsgi/tests.py index a482a5c1eb..5a40bd88e2 100644 --- a/tests/regressiontests/wsgi/tests.py +++ b/tests/regressiontests/wsgi/tests.py @@ -40,8 +40,8 @@ class WSGITest(TestCase): response_data["headers"], [('Content-Type', 'text/html; charset=utf-8')]) self.assertEqual( - six.text_type(response), - "Content-Type: text/html; charset=utf-8\n\nHello World!") + bytes(response), + b"Content-Type: text/html; charset=utf-8\r\n\r\nHello World!") class GetInternalWSGIApplicationTest(unittest.TestCase):