Merge branch 'master' into schema-alteration

AUTHORS

@@ -373,6 +373,7 @@ answer newbie questions, and generally made Django that much better:
     michael.mcewan@gmail.com
     Paul McLanahan <paul@mclanahan.net>
     Tobias McNulty <http://www.caktusgroup.com/blog>
+    Andrews Medina <andrewsmedina@gmail.com>
     Zain Memon
     Christian Metts
     michal@plovarna.cz
@@ -467,6 +468,7 @@ answer newbie questions, and generally made Django that much better:
     Vinay Sajip <vinay_sajip@yahoo.co.uk>
     Bartolome Sanchez Salado <i42sasab@uco.es>
     Kadesarin Sanjek
+    Tim Saylor <tim.saylor@gmail.com>
     Massimo Scamarcia <massimo.scamarcia@gmail.com>
     Paulo Scardine <paulo@scardine.com.br>
     David Schein

MANIFEST.in

@@ -1,4 +1,4 @@
-include README
+include README.rst
 include AUTHORS
 include INSTALL
 include LICENSE

django/bin/django-2to3.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+# This works exactly like 2to3, except that it uses Django's fixers rather
+# than 2to3's built-in fixers.
+
+import sys
+from lib2to3.main import main
+
+sys.exit(main("django.utils.2to3_fixes"))
+

django/conf/__init__.py

@@ -152,17 +152,25 @@ class UserSettingsHolder(BaseSettings):
         Requests for configuration variables not in this class are satisfied
         from the module specified in default_settings (if possible).
         """
+        self.__dict__['_deleted'] = set()
         self.default_settings = default_settings
 
     def __getattr__(self, name):
+        if name in self._deleted:
+            raise AttributeError
         return getattr(self.default_settings, name)
 
+    def __setattr__(self, name, value):
+        self._deleted.discard(name)
+        return super(UserSettingsHolder, self).__setattr__(name, value)
+
+    def __delattr__(self, name):
+        self._deleted.add(name)
+        return super(UserSettingsHolder, self).__delattr__(name)
+
     def __dir__(self):
         return list(self.__dict__) + dir(self.default_settings)
 
-    # For Python < 2.6:
-    __members__ = property(lambda self: self.__dir__())
-
 settings = LazySettings()
 
 

django/conf/project_template/project_name/settings.py

@@ -13,10 +13,11 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'NAME': '',                      # Or path to database file if using sqlite3.
-        'USER': '',                      # Not used with sqlite3.
+        # The following settings are not used with sqlite3:
-        'PASSWORD': '',                  # Not used with sqlite3.
+        'USER': '',
-        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
+        'PASSWORD': '',
-        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
+        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
+        'PORT': '',                      # Set to empty string for default.
     }
 }
 

django/contrib/admin/models.py

@@ -6,6 +6,7 @@ from django.contrib.auth.models import User
 from django.contrib.admin.util import quote
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_text
+from django.utils.encoding import python_2_unicode_compatible
 
 ADDITION = 1
 CHANGE = 2
@@ -16,6 +17,7 @@ class LogEntryManager(models.Manager):
         e = self.model(None, None, user_id, content_type_id, smart_text(object_id), object_repr[:200], action_flag, change_message)
         e.save()
 
+@python_2_unicode_compatible
 class LogEntry(models.Model):
     action_time = models.DateTimeField(_('action time'), auto_now=True)
     user = models.ForeignKey(User)
@@ -36,7 +38,7 @@ class LogEntry(models.Model):
     def __repr__(self):
         return smart_text(self.action_time)
 
-    def __unicode__(self):
+    def __str__(self):
         if self.action_flag == ADDITION:
             return _('Added "%(object)s".') % {'object': self.object_repr}
         elif self.action_flag == CHANGE:

django/contrib/admin/templates/admin/change_form.html

@@ -29,7 +29,7 @@
 {% if change %}{% if not is_popup %}
   <ul class="object-tools">
     {% block object-tools-items %}
-    <li><a href="history/" class="historylink">{% trans "History" %}</a></li>
+    <li><a href="{% url opts|admin_urlname:'history' original.pk %}" class="historylink">{% trans "History" %}</a></li>
     {% if has_absolute_url %}<li><a href="{% url 'admin:view_on_site' content_type_id original.pk %}" class="viewsitelink">{% trans "View on site" %}</a></li>{% endif%}
     {% endblock %}
   </ul>

django/contrib/admin/templatetags/log.py

@@ -17,7 +17,7 @@ class AdminLogNode(template.Node):
             user_id = self.user
             if not user_id.isdigit():
                 user_id = context[self.user].id
-            context[self.varname] = LogEntry.objects.filter(user__id__exact=user_id).select_related('content_type', 'user')[:self.limit]
+            context[self.varname] = LogEntry.objects.filter(user__id__exact=user_id).select_related('content_type', 'user')[:int(self.limit)]
         return ''
 
 @register.tag

django/contrib/admin/util.py

@@ -12,7 +12,7 @@ from django.utils import formats
 from django.utils.html import format_html
 from django.utils.text import capfirst
 from django.utils import timezone
-from django.utils.encoding import force_text, smart_text, smart_bytes
+from django.utils.encoding import force_str, force_text, smart_text
 from django.utils import six
 from django.utils.translation import ungettext
 from django.core.urlresolvers import reverse
@@ -277,7 +277,7 @@ def label_for_field(name, model, model_admin=None, return_attr=False):
             label = force_text(model._meta.verbose_name)
             attr = six.text_type
         elif name == "__str__":
-            label = smart_bytes(model._meta.verbose_name)
+            label = force_str(model._meta.verbose_name)
             attr = bytes
         else:
             if callable(name):

django/contrib/admin/views/main.py

@@ -6,7 +6,7 @@ from django.core.paginator import InvalidPage
 from django.db import models
 from django.db.models.fields import FieldDoesNotExist
 from django.utils.datastructures import SortedDict
-from django.utils.encoding import force_text, smart_bytes
+from django.utils.encoding import force_str, force_text
 from django.utils.translation import ugettext, ugettext_lazy
 from django.utils.http import urlencode
 
@@ -94,7 +94,7 @@ class ChangeList(object):
                 # 'key' will be used as a keyword argument later, so Python
                 # requires it to be a string.
                 del lookup_params[key]
-                lookup_params[smart_bytes(key)] = value
+                lookup_params[force_str(key)] = value
 
             if not self.model_admin.lookup_allowed(key, value):
                 raise SuspiciousOperation("Filtering by %s not allowed" % key)
@@ -148,7 +148,7 @@ class ChangeList(object):
         if remove is None: remove = []
         p = self.params.copy()
         for r in remove:
-            for k in p.keys():
+            for k in list(p):
                 if k.startswith(r):
                     del p[k]
         for k, v in new_params.items():

django/contrib/admindocs/utils.py

@@ -6,7 +6,7 @@ from email.errors import HeaderParseError
 
 from django.utils.safestring import mark_safe
 from django.core.urlresolvers import reverse
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
 try:
     import docutils.core
     import docutils.nodes
@@ -66,7 +66,7 @@ def parse_rst(text, default_reference_context, thing_being_parsed=None):
         "link_base" : reverse('django-admindocs-docroot').rstrip('/')
     }
     if thing_being_parsed:
-        thing_being_parsed = smart_bytes("<%s>" % thing_being_parsed)
+        thing_being_parsed = force_bytes("<%s>" % thing_being_parsed)
     parts = docutils.core.publish_parts(text, source_path=thing_being_parsed,
                 destination_path=None, writer_name='html',
                 settings_overrides=overrides)

django/contrib/auth/decorators.py

@@ -7,6 +7,7 @@ from django.conf import settings
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.core.exceptions import PermissionDenied
 from django.utils.decorators import available_attrs
+from django.utils.encoding import force_str
 
 
 def user_passes_test(test_func, login_url=None, redirect_field_name=REDIRECT_FIELD_NAME):
@@ -22,9 +23,11 @@ def user_passes_test(test_func, login_url=None, redirect_field_name=REDIRECT_FIE
             if test_func(request.user):
                 return view_func(request, *args, **kwargs)
             path = request.build_absolute_uri()
+            # urlparse chokes on lazy objects in Python 3
+            login_url_as_str = force_str(login_url or settings.LOGIN_URL)
             # If the login url is the same scheme and net location then just
             # use the path as the "next" url.
-            login_scheme, login_netloc = urlparse(login_url or settings.LOGIN_URL)[:2]
+            login_scheme, login_netloc = urlparse(login_url_as_str)[:2]
             current_scheme, current_netloc = urlparse(path)[:2]
             if ((not login_scheme or login_scheme == current_scheme) and
                 (not login_netloc or login_netloc == current_netloc)):

django/contrib/auth/hashers.py

@@ -8,7 +8,7 @@ from django.conf import settings
 from django.test.signals import setting_changed
 from django.utils import importlib
 from django.utils.datastructures import SortedDict
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
 from django.core.exceptions import ImproperlyConfigured
 from django.utils.crypto import (
     pbkdf2, constant_time_compare, get_random_string)
@@ -219,7 +219,7 @@ class PBKDF2PasswordHasher(BasePasswordHasher):
         if not iterations:
             iterations = self.iterations
         hash = pbkdf2(password, salt, iterations, digest=self.digest)
-        hash = base64.b64encode(hash).strip()
+        hash = base64.b64encode(hash).decode('ascii').strip()
         return "%s$%d$%s$%s" % (self.algorithm, iterations, salt, hash)
 
     def verify(self, password, encoded):
@@ -299,7 +299,7 @@ class SHA1PasswordHasher(BasePasswordHasher):
     def encode(self, password, salt):
         assert password
         assert salt and '$' not in salt
-        hash = hashlib.sha1(smart_bytes(salt + password)).hexdigest()
+        hash = hashlib.sha1(force_bytes(salt + password)).hexdigest()
         return "%s$%s$%s" % (self.algorithm, salt, hash)
 
     def verify(self, password, encoded):
@@ -327,7 +327,7 @@ class MD5PasswordHasher(BasePasswordHasher):
     def encode(self, password, salt):
         assert password
         assert salt and '$' not in salt
-        hash = hashlib.md5(smart_bytes(salt + password)).hexdigest()
+        hash = hashlib.md5(force_bytes(salt + password)).hexdigest()
         return "%s$%s$%s" % (self.algorithm, salt, hash)
 
     def verify(self, password, encoded):
@@ -361,7 +361,7 @@ class UnsaltedMD5PasswordHasher(BasePasswordHasher):
         return ''
 
     def encode(self, password, salt):
-        return hashlib.md5(smart_bytes(password)).hexdigest()
+        return hashlib.md5(force_bytes(password)).hexdigest()
 
     def verify(self, password, encoded):
         encoded_2 = self.encode(password, '')

django/contrib/auth/management/__init__.py

@@ -9,6 +9,7 @@ import unicodedata
 from django.contrib.auth import models as auth_app
 from django.db.models import get_models, signals
 from django.contrib.auth.models import User
+from django.utils import six
 from django.utils.six.moves import input
 
 
@@ -85,13 +86,22 @@ def get_system_username():
         username could not be determined.
     """
     try:
-        return getpass.getuser().decode(locale.getdefaultlocale()[1])
+        result = getpass.getuser()
-    except (ImportError, KeyError, UnicodeDecodeError):
+    except (ImportError, KeyError):
         # KeyError will be raised by os.getpwuid() (called by getuser())
         # if there is no corresponding entry in the /etc/passwd file
         # (a very restricted chroot environment, for example).
+        return ''
+    if not six.PY3:
+        default_locale = locale.getdefaultlocale()[1]
+        if not default_locale:
+            return ''
+        try:
+            result = result.decode(default_locale)
+        except UnicodeDecodeError:
             # UnicodeDecodeError - preventive treatment for non-latin Windows.
             return ''
+    return result
 
 
 def get_default_username(check_db=True):
@@ -108,7 +118,7 @@ def get_default_username(check_db=True):
     default_username = get_system_username()
     try:
         default_username = unicodedata.normalize('NFKD', default_username)\
-            .encode('ascii', 'ignore').replace(' ', '').lower()
+            .encode('ascii', 'ignore').decode('ascii').replace(' ', '').lower()
     except UnicodeDecodeError:
         return ''
     if not RE_VALID_USERNAME.match(default_username):

django/contrib/auth/models.py

@@ -16,6 +16,7 @@ from django.contrib.auth.hashers import (
     check_password, make_password, is_password_usable, UNUSABLE_PASSWORD)
 from django.contrib.auth.signals import user_logged_in
 from django.contrib.contenttypes.models import ContentType
+from django.utils.encoding import python_2_unicode_compatible
 
 
 def update_last_login(sender, user, **kwargs):
@@ -41,6 +42,7 @@ class PermissionManager(models.Manager):
         )
 
 
+@python_2_unicode_compatible
 class Permission(models.Model):
     """
     The permissions system provides a way to assign permissions to specific
@@ -76,7 +78,7 @@ class Permission(models.Model):
         ordering = ('content_type__app_label', 'content_type__model',
                     'codename')
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s | %s | %s" % (
             six.text_type(self.content_type.app_label),
             six.text_type(self.content_type),
@@ -94,6 +96,7 @@ class GroupManager(models.Manager):
     def get_by_natural_key(self, name):
         return self.get(name=name)
 
+@python_2_unicode_compatible
 class Group(models.Model):
     """
     Groups are a generic way of categorizing users to apply permissions, or
@@ -121,7 +124,7 @@ class Group(models.Model):
         verbose_name = _('group')
         verbose_name_plural = _('groups')
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
     def natural_key(self):
@@ -221,6 +224,7 @@ def _user_has_module_perms(user, app_label):
     return False
 
 
+@python_2_unicode_compatible
 class User(models.Model):
     """
     Users within the Django authentication system are represented by this
@@ -259,7 +263,7 @@ class User(models.Model):
         verbose_name = _('user')
         verbose_name_plural = _('users')
 
-    def __unicode__(self):
+    def __str__(self):
         return self.username
 
     def natural_key(self):
@@ -403,6 +407,7 @@ class User(models.Model):
         return self._profile_cache
 
 
+@python_2_unicode_compatible
 class AnonymousUser(object):
     id = None
     pk = None
@@ -416,11 +421,8 @@ class AnonymousUser(object):
     def __init__(self):
         pass
 
-    def __unicode__(self):
-        return 'AnonymousUser'
-
     def __str__(self):
-        return six.text_type(self).encode('utf-8')
+        return 'AnonymousUser'
 
     def __eq__(self, other):
         return isinstance(other, self.__class__)

django/contrib/auth/tests/basic.py

@@ -1,13 +1,11 @@
+import locale
+import traceback
+
+from django.contrib.auth.management.commands import createsuperuser
 from django.contrib.auth.models import User, AnonymousUser
 from django.core.management import call_command
 from django.test import TestCase
 from django.utils.six import StringIO
-from django.utils.unittest import skipUnless
-
-try:
-    import crypt as crypt_module
-except ImportError:
-    crypt_module = None
 
 
 class BasicTestCase(TestCase):
@@ -111,3 +109,37 @@ class BasicTestCase(TestCase):
         u = User.objects.get(username="joe+admin@somewhere.org")
         self.assertEqual(u.email, 'joe@somewhere.org')
         self.assertFalse(u.has_usable_password())
+
+    def test_createsuperuser_nolocale(self):
+        """
+        Check that createsuperuser does not break when no locale is set. See
+        ticket #16017.
+        """
+
+        old_getdefaultlocale = locale.getdefaultlocale
+        old_getpass = createsuperuser.getpass
+        try:
+            # Temporarily remove locale information
+            locale.getdefaultlocale = lambda: (None, None)
+
+            # Temporarily replace getpass to allow interactive code to be used
+            # non-interactively
+            class mock_getpass: pass
+            mock_getpass.getpass = staticmethod(lambda p=None: "nopasswd")
+            createsuperuser.getpass = mock_getpass
+
+            # Call the command in this new environment
+            new_io = StringIO()
+            call_command("createsuperuser", interactive=True, username="nolocale@somewhere.org", email="nolocale@somewhere.org", stdout=new_io)
+
+        except TypeError as e:
+            self.fail("createsuperuser fails if the OS provides no information about the current locale")
+
+        finally:
+            # Re-apply locale and getpass information
+            createsuperuser.getpass = old_getpass
+            locale.getdefaultlocale = old_getdefaultlocale
+
+        # If we were successful, a user should have been created
+        u = User.objects.get(username="nolocale@somewhere.org")
+        self.assertEqual(u.email, 'nolocale@somewhere.org')

django/contrib/auth/tests/management.py

@@ -4,16 +4,20 @@ from django.contrib.auth import models, management
 from django.contrib.auth.management.commands import changepassword
 from django.core.management.base import CommandError
 from django.test import TestCase
+from django.utils import six
 from django.utils.six import StringIO
 
 
 class GetDefaultUsernameTestCase(TestCase):
 
     def setUp(self):
-        self._getpass_getuser = management.get_system_username
+        self.old_get_system_username = management.get_system_username
 
     def tearDown(self):
-        management.get_system_username = self._getpass_getuser
+        management.get_system_username = self.old_get_system_username
+
+    def test_actual_implementation(self):
+        self.assertIsInstance(management.get_system_username(), six.text_type)
 
     def test_simple(self):
         management.get_system_username = lambda: 'joe'

django/contrib/auth/tests/tokens.py

@@ -1,9 +1,11 @@
+import sys
 from datetime import date, timedelta
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.auth.tokens import PasswordResetTokenGenerator
 from django.test import TestCase
+from django.utils import unittest
 
 
 class TokenGeneratorTest(TestCase):
@@ -51,6 +53,7 @@ class TokenGeneratorTest(TestCase):
         p2 = Mocked(date.today() + timedelta(settings.PASSWORD_RESET_TIMEOUT_DAYS + 1))
         self.assertFalse(p2.check_token(user, tk1))
 
+    @unittest.skipIf(sys.version_info[:2] >= (3, 0), "Unnecessary test with Python 3")
     def test_date_length(self):
         """
         Make sure we don't allow overly long dates, causing a potential DoS.

django/contrib/auth/views.py

@@ -7,6 +7,7 @@ from django.conf import settings
 from django.core.urlresolvers import reverse
 from django.http import HttpResponseRedirect, QueryDict
 from django.template.response import TemplateResponse
+from django.utils.encoding import force_str
 from django.utils.http import base36_to_int
 from django.utils.translation import ugettext as _
 from django.views.decorators.debug import sensitive_post_parameters
@@ -116,10 +117,10 @@ def redirect_to_login(next, login_url=None,
     """
     Redirects the user to the login page, passing the given 'next' page
     """
-    if not login_url:
+    # urlparse chokes on lazy objects in Python 3
-        login_url = settings.LOGIN_URL
+    login_url_as_str = force_str(login_url or settings.LOGIN_URL)
 
-    login_url_parts = list(urlparse(login_url))
+    login_url_parts = list(urlparse(login_url_as_str))
     if redirect_field_name:
         querystring = QueryDict(login_url_parts[4], mutable=True)
         querystring[redirect_field_name] = next
@@ -200,7 +201,7 @@ def password_reset_confirm(request, uidb36=None, token=None,
     try:
         uid_int = base36_to_int(uidb36)
         user = User.objects.get(id=uid_int)
-    except (ValueError, User.DoesNotExist):
+    except (ValueError, OverflowError, User.DoesNotExist):
         user = None
 
     if user is not None and token_generator.check_token(user, token):

django/contrib/comments/models.py

@@ -8,6 +8,7 @@ from django.core import urlresolvers
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 from django.conf import settings
+from django.utils.encoding import python_2_unicode_compatible
 
 COMMENT_MAX_LENGTH = getattr(settings,'COMMENT_MAX_LENGTH',3000)
 
@@ -39,6 +40,7 @@ class BaseCommentAbstractModel(models.Model):
             args=(self.content_type_id, self.object_pk)
         )
 
+@python_2_unicode_compatible
 class Comment(BaseCommentAbstractModel):
     """
     A user comment about some object.
@@ -76,7 +78,7 @@ class Comment(BaseCommentAbstractModel):
         verbose_name = _('comment')
         verbose_name_plural = _('comments')
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s..." % (self.name, self.comment[:50])
 
     def save(self, *args, **kwargs):
@@ -153,6 +155,7 @@ class Comment(BaseCommentAbstractModel):
         }
         return _('Posted by %(user)s at %(date)s\n\n%(comment)s\n\nhttp://%(domain)s%(url)s') % d
 
+@python_2_unicode_compatible
 class CommentFlag(models.Model):
     """
     Records a flag on a comment. This is intentionally flexible; right now, a
@@ -182,7 +185,7 @@ class CommentFlag(models.Model):
         verbose_name = _('comment flag')
         verbose_name_plural = _('comment flags')
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s flag of comment ID %s by %s" % \
             (self.flag, self.comment_id, self.user.username)
 

django/contrib/contenttypes/models.py

@@ -1,6 +1,7 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_text, force_text
+from django.utils.encoding import python_2_unicode_compatible
 
 class ContentTypeManager(models.Manager):
 
@@ -122,6 +123,7 @@ class ContentTypeManager(models.Manager):
         self.__class__._cache.setdefault(using, {})[key] = ct
         self.__class__._cache.setdefault(using, {})[ct.id] = ct
 
+@python_2_unicode_compatible
 class ContentType(models.Model):
     name = models.CharField(max_length=100)
     app_label = models.CharField(max_length=100)
@@ -135,7 +137,7 @@ class ContentType(models.Model):
         ordering = ('name',)
         unique_together = (('app_label', 'model'),)
 
-    def __unicode__(self):
+    def __str__(self):
         # self.name is deprecated in favor of using model's verbose_name, which
         # can be translated. Formal deprecation is delayed until we have DB
         # migration to be able to remove the field from the database along with

django/contrib/contenttypes/tests.py

@@ -8,6 +8,7 @@ from django.http import HttpRequest, Http404
 from django.test import TestCase
 from django.utils.http import urlquote
 from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
 
 
 class ConcreteModel(models.Model):
@@ -17,13 +18,14 @@ class ProxyModel(ConcreteModel):
     class Meta:
         proxy = True
 
+@python_2_unicode_compatible
 class FooWithoutUrl(models.Model):
     """
     Fake model not defining ``get_absolute_url`` for
     :meth:`ContentTypesTests.test_shortcut_view_without_get_absolute_url`"""
     name = models.CharField(max_length=30, unique=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 

django/contrib/databrowse/datastructures.py

@@ -7,8 +7,9 @@ from __future__ import unicode_literals
 from django.db import models
 from django.utils import formats
 from django.utils.text import capfirst
-from django.utils.encoding import smart_text, smart_bytes, iri_to_uri
+from django.utils.encoding import smart_text, force_str, iri_to_uri
 from django.db.models.query import QuerySet
+from django.utils.encoding import python_2_unicode_compatible
 
 EMPTY_VALUE = '(None)'
 DISPLAY_SIZE = 100
@@ -22,7 +23,7 @@ class EasyModel(object):
         self.verbose_name_plural = model._meta.verbose_name_plural
 
     def __repr__(self):
-        return '<EasyModel for %s>' % smart_bytes(self.model._meta.object_name)
+        return force_str('<EasyModel for %s>' % self.model._meta.object_name)
 
     def model_databrowse(self):
         "Returns the ModelDatabrowse class for this model."
@@ -61,7 +62,7 @@ class EasyField(object):
         self.model, self.field = easy_model, field
 
     def __repr__(self):
-        return smart_bytes('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+        return force_str('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def choices(self):
         for value, label in self.field.choices:
@@ -79,27 +80,25 @@ class EasyChoice(object):
         self.value, self.label = value, label
 
     def __repr__(self):
-        return smart_bytes('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+        return force_str('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def url(self):
         return '%s%s/%s/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.field.name, iri_to_uri(self.value))
 
+@python_2_unicode_compatible
 class EasyInstance(object):
     def __init__(self, easy_model, instance):
         self.model, self.instance = easy_model, instance
 
     def __repr__(self):
-        return smart_bytes('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val()))
+        return force_str('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val()))
 
-    def __unicode__(self):
+    def __str__(self):
         val = smart_text(self.instance)
         if len(val) > DISPLAY_SIZE:
             return val[:DISPLAY_SIZE] + '...'
         return val
 
-    def __str__(self):
-        return self.__unicode__().encode('utf-8')
-
     def pk(self):
         return self.instance._get_pk_val()
 
@@ -136,7 +135,7 @@ class EasyInstanceField(object):
         self.raw_value = getattr(instance.instance, field.name)
 
     def __repr__(self):
-        return smart_bytes('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+        return force_str('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
 
     def values(self):
         """

django/contrib/databrowse/tests.py

@@ -1,26 +1,30 @@
 from django.contrib import databrowse
 from django.db import models
 from django.test import TestCase
+from django.utils.encoding import python_2_unicode_compatible
 
 
+@python_2_unicode_compatible
 class SomeModel(models.Model):
     some_field = models.CharField(max_length=50)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.some_field
 
 
+@python_2_unicode_compatible
 class SomeOtherModel(models.Model):
     some_other_field = models.CharField(max_length=50)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.some_other_field
 
 
+@python_2_unicode_compatible
 class YetAnotherModel(models.Model):
     yet_another_field = models.CharField(max_length=50)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.yet_another_field
 
 

django/contrib/flatpages/models.py

@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 from django.db import models
 from django.contrib.sites.models import Site
 from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import python_2_unicode_compatible
 
 
+@python_2_unicode_compatible
 class FlatPage(models.Model):
     url = models.CharField(_('URL'), max_length=100, db_index=True)
     title = models.CharField(_('title'), max_length=200)
@@ -21,7 +23,7 @@ class FlatPage(models.Model):
         verbose_name_plural = _('flat pages')
         ordering = ('url',)
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s -- %s" % (self.url, self.title)
 
     def get_absolute_url(self):

django/contrib/formtools/tests/__init__.py

@@ -1,6 +1,9 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import datetime
 import os
+import pickle
 import re
 import warnings
 
@@ -16,6 +19,7 @@ from django.contrib.formtools.tests.wizard import *
 from django.contrib.formtools.tests.forms import *
 
 success_string = "Done was called!"
+success_string_encoded = success_string.encode()
 
 class TestFormPreview(preview.FormPreview):
     def get_context(self, request, form):
@@ -78,7 +82,7 @@ class PreviewTests(TestCase):
         """
         # Pass strings for form submittal and add stage variable to
         # show we previously saw first stage of the form.
-        self.test_data.update({'stage': 1})
+        self.test_data.update({'stage': 1, 'date1': datetime.date(2006, 10, 25)})
         response = self.client.post('/preview/', self.test_data)
         # Check to confirm stage is set to 2 in output form.
         stage = self.input % 2
@@ -96,13 +100,13 @@ class PreviewTests(TestCase):
         """
         # Pass strings for form submittal and add stage variable to
         # show we previously saw first stage of the form.
-        self.test_data.update({'stage':2})
+        self.test_data.update({'stage': 2, 'date1': datetime.date(2006, 10, 25)})
         response = self.client.post('/preview/', self.test_data)
-        self.assertNotEqual(response.content, success_string)
+        self.assertNotEqual(response.content, success_string_encoded)
         hash = self.preview.security_hash(None, TestForm(self.test_data))
         self.test_data.update({'hash': hash})
         response = self.client.post('/preview/', self.test_data)
-        self.assertEqual(response.content, success_string)
+        self.assertEqual(response.content, success_string_encoded)
 
     def test_bool_submit(self):
         """
@@ -122,7 +126,7 @@ class PreviewTests(TestCase):
         self.test_data.update({'hash': hash, 'bool1': 'False'})
         with warnings.catch_warnings(record=True):
             response = self.client.post('/preview/', self.test_data)
-            self.assertEqual(response.content, success_string)
+            self.assertEqual(response.content, success_string_encoded)
 
     def test_form_submit_good_hash(self):
         """
@@ -133,11 +137,11 @@ class PreviewTests(TestCase):
         # show we previously saw first stage of the form.
         self.test_data.update({'stage':2})
         response = self.client.post('/preview/', self.test_data)
-        self.assertNotEqual(response.content, success_string)
+        self.assertNotEqual(response.content, success_string_encoded)
         hash = utils.form_hmac(TestForm(self.test_data))
         self.test_data.update({'hash': hash})
         response = self.client.post('/preview/', self.test_data)
-        self.assertEqual(response.content, success_string)
+        self.assertEqual(response.content, success_string_encoded)
 
 
     def test_form_submit_bad_hash(self):
@@ -150,11 +154,11 @@ class PreviewTests(TestCase):
         self.test_data.update({'stage':2})
         response = self.client.post('/preview/', self.test_data)
         self.assertEqual(response.status_code, 200)
-        self.assertNotEqual(response.content, success_string)
+        self.assertNotEqual(response.content, success_string_encoded)
         hash = utils.form_hmac(TestForm(self.test_data)) + "bad"
         self.test_data.update({'hash': hash})
         response = self.client.post('/previewpreview/', self.test_data)
-        self.assertNotEqual(response.content, success_string)
+        self.assertNotEqual(response.content, success_string_encoded)
 
 
 class FormHmacTests(unittest.TestCase):
@@ -165,8 +169,8 @@ class FormHmacTests(unittest.TestCase):
         leading/trailing whitespace so as to be friendly to broken browsers that
         submit it (usually in textareas).
         """
-        f1 = HashTestForm({'name': 'joe', 'bio': 'Nothing notable.'})
+        f1 = HashTestForm({'name': 'joe', 'bio': 'Speaking español.'})
-        f2 = HashTestForm({'name': '  joe', 'bio': 'Nothing notable.  '})
+        f2 = HashTestForm({'name': '  joe', 'bio': 'Speaking español.  '})
         hash1 = utils.form_hmac(f1)
         hash2 = utils.form_hmac(f2)
         self.assertEqual(hash1, hash2)
@@ -270,7 +274,10 @@ class WizardTests(TestCase):
         """
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "1"}
         response = self.client.post('/wizard1/', data)
         self.assertEqual(2, response.context['step0'])
@@ -295,15 +302,24 @@ class WizardTests(TestCase):
         wizard = WizardWithProcessStep([WizardPageOneForm])
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "1"}
         wizard(DummyRequest(POST=data))
         self.assertTrue(reached[0])
 
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
-                "hash_1": "1e6f6315da42e62f33a30640ec7e007ad3fbf1a1",
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
+                "hash_1": {
+                    2: "1e6f6315da42e62f33a30640ec7e007ad3fbf1a1",
+                    3: "c33142ef9d01b1beae238adf22c3c6c57328f51a",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "2"}
         self.assertRaises(http.Http404, wizard, DummyRequest(POST=data))
 
@@ -325,7 +341,10 @@ class WizardTests(TestCase):
                                         WizardPageThreeForm])
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "1"}
         wizard(DummyRequest(POST=data))
         self.assertTrue(reached[0])
@@ -349,7 +368,10 @@ class WizardTests(TestCase):
 
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "1"}
         wizard(DummyRequest(POST=data))
         self.assertTrue(reached[0])
@@ -375,7 +397,10 @@ class WizardTests(TestCase):
                                         WizardPageThreeForm])
         data = {"0-field": "test",
                 "1-field": "test2",
-                "hash_0": "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                "hash_0": {
+                    2: "cd13b1db3e8f55174bc5745a1b1a53408d4fd1ca",
+                    3: "9355d5dff22d49dbad58e46189982cec649f9f5b",
+                }[pickle.HIGHEST_PROTOCOL],
                 "wizard_step": "1"}
         wizard(DummyRequest(POST=data))
         self.assertTrue(reached[0])

django/contrib/formtools/tests/forms.py

@@ -21,6 +21,7 @@ class TestForm(forms.Form):
     field1 = forms.CharField()
     field1_ = forms.CharField()
     bool1 = forms.BooleanField(required=False)
+    date1 = forms.DateField(required=False)
 
 class HashTestForm(forms.Form):
     name = forms.CharField()

django/contrib/formtools/tests/wizard/namedwizardtests/tests.py

@@ -122,6 +122,7 @@ class NamedWizardTests(object):
         self.assertEqual(response.context['wizard']['steps'].current, 'form2')
 
         post_data = self.wizard_step_data[1]
+        post_data['form2-file1'].close()
         post_data['form2-file1'] = open(__file__, 'rb')
         response = self.client.post(
             reverse(self.wizard_urlname,
@@ -149,7 +150,9 @@ class NamedWizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['form_list']
-        self.assertEqual(all_data[1]['file1'].read(), open(__file__, 'rb').read())
+        with open(__file__, 'rb') as f:
+            self.assertEqual(all_data[1]['file1'].read(), f.read())
+        all_data[1]['file1'].close()
         del all_data[1]['file1']
         self.assertEqual(all_data, [
             {'name': 'Pony', 'thirsty': True, 'user': self.testuser},
@@ -182,9 +185,10 @@ class NamedWizardTests(object):
         response = self.client.get(step2_url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.context['wizard']['steps'].current, 'form2')
+        with open(__file__, 'rb') as f:
             self.assertEqual(
                 response.context['wizard']['form'].files['form2-file1'].read(),
-            open(__file__, 'rb').read())
+                f.read())
 
         response = self.client.post(
             reverse(self.wizard_urlname,
@@ -201,7 +205,9 @@ class NamedWizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['all_cleaned_data']
-        self.assertEqual(all_data['file1'].read(), open(__file__, 'rb').read())
+        with open(__file__, 'rb') as f:
+            self.assertEqual(all_data['file1'].read(), f.read())
+        all_data['file1'].close()
         del all_data['file1']
         self.assertEqual(
             all_data,
@@ -225,6 +231,7 @@ class NamedWizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         post_data = self.wizard_step_data[1]
+        post_data['form2-file1'].close()
         post_data['form2-file1'] = open(__file__, 'rb')
         response = self.client.post(
             reverse(self.wizard_urlname,

django/contrib/formtools/tests/wizard/wizardtests/tests.py

@@ -95,7 +95,9 @@ class WizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['form_list']
-        self.assertEqual(all_data[1]['file1'].read(), open(__file__, 'rb').read())
+        with open(__file__, 'rb') as f:
+            self.assertEqual(all_data[1]['file1'].read(), f.read())
+        all_data[1]['file1'].close()
         del all_data[1]['file1']
         self.assertEqual(all_data, [
             {'name': 'Pony', 'thirsty': True, 'user': self.testuser},
@@ -112,7 +114,8 @@ class WizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         post_data = self.wizard_step_data[1]
-        post_data['form2-file1'] = open(__file__, 'rb')
+        with open(__file__, 'rb') as post_file:
+            post_data['form2-file1'] = post_file
             response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
 
@@ -123,7 +126,9 @@ class WizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['all_cleaned_data']
-        self.assertEqual(all_data['file1'].read(), open(__file__, 'rb').read())
+        with open(__file__, 'rb') as f:
+            self.assertEqual(all_data['file1'].read(), f.read())
+        all_data['file1'].close()
         del all_data['file1']
         self.assertEqual(all_data, {
             'name': 'Pony', 'thirsty': True, 'user': self.testuser,
@@ -140,6 +145,7 @@ class WizardTests(object):
         self.assertEqual(response.status_code, 200)
 
         post_data = self.wizard_step_data[1]
+        post_data['form2-file1'].close()
         post_data['form2-file1'] = open(__file__, 'rb')
         response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
@@ -167,6 +173,7 @@ class WizardTests(object):
         self.assertEqual(response.context['wizard']['steps'].current, 'form2')
 
         post_data = self.wizard_step_data[1]
+        post_data['form2-file1'].close()
         post_data['form2-file1'] = open(__file__, 'rb')
         response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)

django/contrib/formtools/utils.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 # Do not try cPickle here (see #18340)
 import pickle
 

django/contrib/formtools/wizard/storage/base.py

@@ -1,6 +1,5 @@
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import smart_bytes
 from django.utils.functional import lazy_property
 from django.utils import six
 
@@ -74,8 +73,7 @@ class BaseStorage(object):
 
         files = {}
         for field, field_dict in six.iteritems(wizard_files):
-            field_dict = dict((smart_bytes(k), v)
+            field_dict = field_dict.copy()
-                              for k, v in six.iteritems(field_dict))
             tmp_name = field_dict.pop('tmp_name')
             files[field] = UploadedFile(
                 file=self.file_storage.open(tmp_name), **field_dict)

django/contrib/gis/db/backends/base.py

@@ -5,6 +5,7 @@ Base/mixin classes for the spatial backend database operations and the
 import re
 from django.contrib.gis import gdal
 from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
 
 class BaseSpatialOperations(object):
     """
@@ -131,6 +132,7 @@ class BaseSpatialOperations(object):
     def spatial_ref_sys(self):
         raise NotImplementedError
 
+@python_2_unicode_compatible
 class SpatialRefSysMixin(object):
     """
     The SpatialRefSysMixin is a class used by the database-dependent
@@ -325,7 +327,7 @@ class SpatialRefSysMixin(object):
                 radius, flattening = sphere_params
             return 'SPHEROID["%s",%s,%s]' % (sphere_name, radius, flattening)
 
-    def __unicode__(self):
+    def __str__(self):
         """
         Returns the string representation.  If GDAL is installed,
         it will be 'pretty' OGC WKT.

django/contrib/gis/db/backends/oracle/models.py

@@ -9,7 +9,9 @@
 """
 from django.contrib.gis.db import models
 from django.contrib.gis.db.backends.base import SpatialRefSysMixin
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class GeometryColumns(models.Model):
     "Maps to the Oracle USER_SDO_GEOM_METADATA table."
     table_name = models.CharField(max_length=32)
@@ -36,7 +38,7 @@ class GeometryColumns(models.Model):
         """
         return 'column_name'
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s - %s (SRID: %s)' % (self.table_name, self.column_name, self.srid)
 
 class SpatialRefSys(models.Model, SpatialRefSysMixin):

django/contrib/gis/db/backends/postgis/models.py

@@ -3,7 +3,9 @@
 """
 from django.db import models
 from django.contrib.gis.db.backends.base import SpatialRefSysMixin
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class GeometryColumns(models.Model):
     """
     The 'geometry_columns' table from the PostGIS. See the PostGIS
@@ -37,7 +39,7 @@ class GeometryColumns(models.Model):
         """
         return 'f_geometry_column'
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s.%s - %dD %s field (SRID: %d)" % \
                (self.f_table_name, self.f_geometry_column,
                 self.coord_dimension, self.type, self.srid)

django/contrib/gis/db/backends/spatialite/creation.py

@@ -30,6 +30,7 @@ class SpatiaLiteCreation(DatabaseCreation):
 
         self.connection.close()
         self.connection.settings_dict["NAME"] = test_database_name
+        self.connection.ops.confirm_spatial_components_versions()
 
         # Need to load the SpatiaLite initialization SQL before running `syncdb`.
         self.load_spatialite_sql()

django/contrib/gis/db/backends/spatialite/models.py

@@ -3,7 +3,9 @@
 """
 from django.db import models
 from django.contrib.gis.db.backends.base import SpatialRefSysMixin
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class GeometryColumns(models.Model):
     """
     The 'geometry_columns' table from SpatiaLite.
@@ -35,7 +37,7 @@ class GeometryColumns(models.Model):
         """
         return 'f_geometry_column'
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s.%s - %dD %s field (SRID: %d)" % \
                (self.f_table_name, self.f_geometry_column,
                 self.coord_dimension, self.type, self.srid)

django/contrib/gis/db/backends/spatialite/operations.py

@@ -113,6 +113,12 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
     def __init__(self, connection):
         super(DatabaseOperations, self).__init__(connection)
 
+        # Creating the GIS terms dictionary.
+        gis_terms = ['isnull']
+        gis_terms += self.geometry_functions.keys()
+        self.gis_terms = dict([(term, None) for term in gis_terms])
+
+    def confirm_spatial_components_versions(self):
         # Determine the version of the SpatiaLite library.
         try:
             vtup = self.spatialite_version_tuple()
@@ -129,11 +135,6 @@ class SpatiaLiteOperations(DatabaseOperations, BaseSpatialOperations):
                                        'SQL loaded on this database?' %
                                        (self.connection.settings_dict['NAME'], msg))
 
-        # Creating the GIS terms dictionary.
-        gis_terms = ['isnull']
-        gis_terms += list(self.geometry_functions)
-        self.gis_terms = dict([(term, None) for term in gis_terms])
-
         if version >= (2, 4, 0):
             # Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
             # RC2 (shipped in popular Debian/Ubuntu packages) and RC4

django/contrib/gis/db/models/query.py

@@ -1,16 +1,15 @@
 from django.db import connections
 from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet
-from django.utils import six
 
 from django.contrib.gis.db.models import aggregates
 from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField
 from django.contrib.gis.db.models.sql import AreaField, DistanceField, GeomField, GeoQuery
 from django.contrib.gis.geometry.backend import Geometry
 from django.contrib.gis.measure import Area, Distance
-from django.utils import six
 
 from django.utils import six
 
+
 class GeoQuerySet(QuerySet):
     "The Geographic QuerySet."
 

django/contrib/gis/gdal/tests/__init__.py

@@ -19,7 +19,8 @@ test_suites = [test_driver.suite(),
 def suite():
     "Builds a test suite for the GDAL tests."
     s = TestSuite()
-    map(s.addTest, test_suites)
+    for test_suite in test_suites:
+        s.addTest(test_suite)
     return s
 
 def run(verbosity=1):

django/contrib/gis/maps/google/overlays.py

@@ -2,8 +2,10 @@ from django.contrib.gis.geos import fromstr, Point, LineString, LinearRing, Poly
 from django.utils.functional import total_ordering
 from django.utils.safestring import mark_safe
 from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
 
 
+@python_2_unicode_compatible
 class GEvent(object):
     """
     A Python wrapper for the Google GEvent object.
@@ -48,10 +50,11 @@ class GEvent(object):
         self.event = event
         self.action = action
 
-    def __unicode__(self):
+    def __str__(self):
         "Returns the parameter part of a GEvent."
         return mark_safe('"%s", %s' %(self.event, self.action))
 
+@python_2_unicode_compatible
 class GOverlayBase(object):
     def __init__(self):
         self.events = []
@@ -64,7 +67,7 @@ class GOverlayBase(object):
         "Attaches a GEvent to the overlay object."
         self.events.append(event)
 
-    def __unicode__(self):
+    def __str__(self):
         "The string representation is the JavaScript API call."
         return mark_safe('%s(%s)' % (self.__class__.__name__, self.js_params))
 

django/contrib/gis/sitemaps/views.py

@@ -8,7 +8,6 @@ from django.core.paginator import EmptyPage, PageNotAnInteger
 from django.contrib.gis.db.models.fields import GeometryField
 from django.db import connections, DEFAULT_DB_ALIAS
 from django.db.models import get_model
-from django.utils.encoding import smart_bytes
 from django.utils import six
 from django.utils.translation import ugettext as _
 
@@ -61,7 +60,7 @@ def sitemap(request, sitemaps, section=None):
             raise Http404(_("Page %s empty") % page)
         except PageNotAnInteger:
             raise Http404(_("No page '%s'") % page)
-    xml = smart_bytes(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls}))
+    xml = loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})
     return HttpResponse(xml, content_type='application/xml')
 
 def kml(request, label, model, field_name=None, compress=False, using=DEFAULT_DB_ALIAS):

django/contrib/gis/tests/distapp/models.py

@@ -1,50 +1,58 @@
 from django.contrib.gis.db import models
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class SouthTexasCity(models.Model):
     "City model on projected coordinate system for South Texas."
     name = models.CharField(max_length=30)
     point = models.PointField(srid=32140)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class SouthTexasCityFt(models.Model):
     "Same City model as above, but U.S. survey feet are the units."
     name = models.CharField(max_length=30)
     point = models.PointField(srid=2278)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class AustraliaCity(models.Model):
     "City model for Australia, using WGS84."
     name = models.CharField(max_length=30)
     point = models.PointField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class CensusZipcode(models.Model):
     "Model for a few South Texas ZIP codes (in original Census NAD83)."
     name = models.CharField(max_length=5)
     poly = models.PolygonField(srid=4269)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class SouthTexasZipcode(models.Model):
     "Model for a few South Texas ZIP codes."
     name = models.CharField(max_length=5)
     poly = models.PolygonField(srid=32140, null=True)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class Interstate(models.Model):
     "Geodetic model for U.S. Interstates."
     name = models.CharField(max_length=10)
     path = models.LineStringField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class SouthTexasInterstate(models.Model):
     "Projected model for South Texas Interstates."
     name = models.CharField(max_length=10)
     path = models.LineStringField(srid=32140)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name

django/contrib/gis/tests/geo3d/models.py

@@ -1,59 +1,67 @@
 from django.contrib.gis.db import models
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class City3D(models.Model):
     name = models.CharField(max_length=30)
     point = models.PointField(dim=3)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class Interstate2D(models.Model):
     name = models.CharField(max_length=30)
     line = models.LineStringField(srid=4269)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class Interstate3D(models.Model):
     name = models.CharField(max_length=30)
     line = models.LineStringField(dim=3, srid=4269)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class InterstateProj2D(models.Model):
     name = models.CharField(max_length=30)
     line = models.LineStringField(srid=32140)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class InterstateProj3D(models.Model):
     name = models.CharField(max_length=30)
     line = models.LineStringField(dim=3, srid=32140)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class Polygon2D(models.Model):
     name = models.CharField(max_length=30)
     poly = models.PolygonField(srid=32140)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
+@python_2_unicode_compatible
 class Polygon3D(models.Model):
     name = models.CharField(max_length=30)
     poly = models.PolygonField(dim=3, srid=32140)
     objects = models.GeoManager()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 class Point2D(models.Model):

django/contrib/gis/tests/geoadmin/models.py

@@ -1,10 +1,12 @@
 from django.contrib.gis.db import models
 from django.contrib.gis import admin
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class City(models.Model):
     name = models.CharField(max_length=30)
     point = models.PointField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
 admin.site.register(City, admin.OSMGeoAdmin)

django/contrib/gis/tests/geoapp/models.py

@@ -1,20 +1,23 @@
 from django.contrib.gis.db import models
 from django.contrib.gis.tests.utils import mysql, spatialite
+from django.utils.encoding import python_2_unicode_compatible
 
 # MySQL spatial indices can't handle NULL geometries.
 null_flag = not mysql
 
+@python_2_unicode_compatible
 class Country(models.Model):
     name = models.CharField(max_length=30)
     mpoly = models.MultiPolygonField() # SRID, by default, is 4326
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class City(models.Model):
     name = models.CharField(max_length=30)
     point = models.PointField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
 # This is an inherited model from City
 class PennsylvaniaCity(City):
@@ -22,28 +25,31 @@ class PennsylvaniaCity(City):
     founded = models.DateTimeField(null=True)
     objects = models.GeoManager() # TODO: This should be implicitly inherited.
 
+@python_2_unicode_compatible
 class State(models.Model):
     name = models.CharField(max_length=30)
     poly = models.PolygonField(null=null_flag) # Allowing NULL geometries here.
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class Track(models.Model):
     name = models.CharField(max_length=30)
     line = models.LineStringField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
 class Truth(models.Model):
     val = models.BooleanField()
     objects = models.GeoManager()
 
 if not spatialite:
+    @python_2_unicode_compatible
     class Feature(models.Model):
         name = models.CharField(max_length=20)
         geom = models.GeometryField()
         objects = models.GeoManager()
-        def __unicode__(self): return self.name
+        def __str__(self): return self.name
 
     class MinusOneSRID(models.Model):
         geom = models.PointField(srid=-1) # Minus one SRID.

django/contrib/gis/tests/geoapp/tests.py

@@ -186,6 +186,15 @@ class GeoModelTest(TestCase):
         self.assertEqual(1, qs.count())
         for pc in qs: self.assertEqual(32128, pc.point.srid)
 
+    def test_raw_sql_query(self):
+        "Testing raw SQL query."
+        cities1 = City.objects.all()
+        # Only PostGIS would support a 'select *' query because of its recognized
+        # HEXEWKB format for geometry fields
+        cities2 = City.objects.raw('select id, name, asText(point) from geoapp_city')
+        self.assertEqual(len(cities1), len(list(cities2)))
+        self.assertTrue(isinstance(cities2[0].point, Point))
+
 
 class GeoLookupTest(TestCase):
 

django/contrib/gis/tests/geogapp/models.py

@@ -1,20 +1,24 @@
 from django.contrib.gis.db import models
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class City(models.Model):
     name = models.CharField(max_length=30)
     point = models.PointField(geography=True)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
+@python_2_unicode_compatible
 class Zipcode(models.Model):
     code = models.CharField(max_length=10)
     poly = models.PolygonField(geography=True)
     objects = models.GeoManager()
-    def __unicode__(self): return self.code
+    def __str__(self): return self.code
 
+@python_2_unicode_compatible
 class County(models.Model):
     name = models.CharField(max_length=25)
     state = models.CharField(max_length=20)
     mpoly = models.MultiPolygonField(geography=True)
     objects = models.GeoManager()
-    def __unicode__(self): return ' County, '.join([self.name, self.state])
+    def __str__(self): return ' County, '.join([self.name, self.state])

django/contrib/gis/tests/relatedapp/models.py

@@ -1,17 +1,20 @@
 from django.contrib.gis.db import models
 from django.contrib.localflavor.us.models import USStateField
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class Location(models.Model):
     point = models.PointField()
     objects = models.GeoManager()
-    def __unicode__(self): return self.point.wkt
+    def __str__(self): return self.point.wkt
 
+@python_2_unicode_compatible
 class City(models.Model):
     name = models.CharField(max_length=50)
     state = USStateField()
     location = models.ForeignKey(Location)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
 class AugmentedLocation(Location):
     extra_text = models.TextField(blank=True)
@@ -22,6 +25,7 @@ class DirectoryEntry(models.Model):
     location = models.ForeignKey(AugmentedLocation)
     objects = models.GeoManager()
 
+@python_2_unicode_compatible
 class Parcel(models.Model):
     name = models.CharField(max_length=30)
     city = models.ForeignKey(City)
@@ -31,7 +35,7 @@ class Parcel(models.Model):
     border1 = models.PolygonField()
     border2 = models.PolygonField(srid=2276)
     objects = models.GeoManager()
-    def __unicode__(self): return self.name
+    def __str__(self): return self.name
 
 # These use the GeoManager but do not have any geographic fields.
 class Author(models.Model):

django/contrib/gis/utils/ogrinspect.py

@@ -223,4 +223,4 @@ def _ogrinspect(data_source, model_name, geom_name='geom', layer_key=0, srid=Non
 
     if name_field:
         yield ''
-        yield '    def __unicode__(self): return self.%s' % name_field
+        yield '    def __str__(self): return self.%s' % name_field

django/contrib/markup/templatetags/markup.py

@@ -13,7 +13,7 @@ markup syntaxes to HTML; currently there is support for:
 
 from django import template
 from django.conf import settings
-from django.utils.encoding import smart_bytes, force_text
+from django.utils.encoding import force_bytes, force_text
 from django.utils.safestring import mark_safe
 
 register = template.Library()
@@ -27,7 +27,7 @@ def textile(value):
             raise template.TemplateSyntaxError("Error in 'textile' filter: The Python textile library isn't installed.")
         return force_text(value)
     else:
-        return mark_safe(force_text(textile.textile(smart_bytes(value), encoding='utf-8', output='utf-8')))
+        return mark_safe(force_text(textile.textile(force_bytes(value), encoding='utf-8', output='utf-8')))
 
 @register.filter(is_safe=True)
 def markdown(value, arg=''):
@@ -80,5 +80,5 @@ def restructuredtext(value):
         return force_text(value)
     else:
         docutils_settings = getattr(settings, "RESTRUCTUREDTEXT_FILTER_SETTINGS", {})
-        parts = publish_parts(source=smart_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings)
+        parts = publish_parts(source=force_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings)
         return mark_safe(force_text(parts["fragment"]))

django/contrib/messages/storage/base.py

@@ -1,14 +1,15 @@
 from __future__ import unicode_literals
 
 from django.conf import settings
-from django.utils.encoding import force_text, StrAndUnicode
+from django.utils.encoding import force_text, python_2_unicode_compatible
 from django.contrib.messages import constants, utils
 
 
 LEVEL_TAGS = utils.get_level_tags()
 
 
-class Message(StrAndUnicode):
+@python_2_unicode_compatible
+class Message(object):
     """
     Represents an actual message that can be stored in any of the supported
     storage classes (typically session- or cookie-based) and rendered in a view
@@ -35,7 +36,7 @@ class Message(StrAndUnicode):
         return isinstance(other, Message) and self.level == other.level and \
                                               self.message == other.message
 
-    def __unicode__(self):
+    def __str__(self):
         return force_text(self.message)
 
     def _get_tags(self):

django/contrib/redirects/models.py

@@ -1,7 +1,9 @@
 from django.db import models
 from django.contrib.sites.models import Site
 from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class Redirect(models.Model):
     site = models.ForeignKey(Site)
     old_path = models.CharField(_('redirect from'), max_length=200, db_index=True,
@@ -16,5 +18,5 @@ class Redirect(models.Model):
         unique_together=(('site', 'old_path'),)
         ordering = ('old_path',)
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s ---> %s" % (self.old_path, self.new_path)

django/contrib/sessions/backends/base.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import base64
 import time
 from datetime import datetime, timedelta
@@ -12,6 +14,7 @@ from django.utils.crypto import constant_time_compare
 from django.utils.crypto import get_random_string
 from django.utils.crypto import salted_hmac
 from django.utils import timezone
+from django.utils.encoding import force_bytes
 
 class CreateError(Exception):
     """
@@ -78,15 +81,15 @@ class SessionBase(object):
         "Returns the given session dictionary pickled and encoded as a string."
         pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL)
         hash = self._hash(pickled)
-        return base64.encodestring(hash + ":" + pickled)
+        return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii')
 
     def decode(self, session_data):
-        encoded_data = base64.decodestring(session_data)
+        encoded_data = base64.b64decode(force_bytes(session_data))
         try:
             # could produce ValueError if there is no ':'
-            hash, pickled = encoded_data.split(':', 1)
+            hash, pickled = encoded_data.split(b':', 1)
             expected_hash = self._hash(pickled)
-            if not constant_time_compare(hash, expected_hash):
+            if not constant_time_compare(hash.decode(), expected_hash):
                 raise SuspiciousOperation("Session data corrupted")
             else:
                 return pickle.loads(pickled)

django/contrib/sessions/backends/db.py

@@ -1,7 +1,6 @@
 from django.contrib.sessions.backends.base import SessionBase, CreateError
 from django.core.exceptions import SuspiciousOperation
 from django.db import IntegrityError, transaction, router
-from django.utils.encoding import force_text
 from django.utils import timezone
 
 
@@ -18,7 +17,7 @@ class SessionStore(SessionBase):
                 session_key = self.session_key,
                 expire_date__gt=timezone.now()
             )
-            return self.decode(force_text(s.session_data))
+            return self.decode(s.session_data)
         except (Session.DoesNotExist, SuspiciousOperation):
             self.create()
             return {}

django/contrib/sessions/backends/file.py

@@ -115,7 +115,7 @@ class SessionStore(SessionBase):
             renamed = False
             try:
                 try:
-                    os.write(output_file_fd, self.encode(session_data))
+                    os.write(output_file_fd, self.encode(session_data).encode())
                 finally:
                     os.close(output_file_fd)
                 os.rename(output_file_name, session_file_name)

django/contrib/sitemaps/tests/generic.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from django.contrib.auth.models import User
 from django.test.utils import override_settings
 
@@ -12,8 +14,9 @@ class GenericViewsSitemapTests(SitemapTestsBase):
         expected = ''
         for username in User.objects.values_list("username", flat=True):
             expected += "<url><loc>%s/users/%s/</loc></url>" % (self.base_url, username)
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 %s
 </urlset>
-""" % expected)
+""" % expected
+        self.assertEqual(response.content, expected_content.encode('utf-8'))

django/contrib/sitemaps/tests/http.py

@@ -21,11 +21,12 @@ class HTTPSitemapTests(SitemapTestsBase):
     def test_simple_sitemap_index(self):
         "A simple sitemap index can be rendered"
         response = self.client.get('/simple/index.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
 </sitemapindex>
-""" % self.base_url)
+""" % self.base_url
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     @override_settings(
         TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), 'templates'),)
@@ -33,30 +34,34 @@ class HTTPSitemapTests(SitemapTestsBase):
     def test_simple_sitemap_custom_index(self):
         "A simple sitemap index can be rendered with a custom template"
         response = self.client.get('/simple/custom-index.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <!-- This is a customised template -->
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
 </sitemapindex>
-""" % self.base_url)
+""" % self.base_url
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
+
 
     def test_simple_sitemap_section(self):
         "A simple sitemap section can be rendered"
         response = self.client.get('/simple/sitemap-simple.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % (self.base_url, date.today()))
+""" % (self.base_url, date.today())
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     def test_simple_sitemap(self):
         "A simple sitemap can be rendered"
         response = self.client.get('/simple/sitemap.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % (self.base_url, date.today()))
+""" % (self.base_url, date.today())
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     @override_settings(
         TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), 'templates'),)
@@ -64,12 +69,13 @@ class HTTPSitemapTests(SitemapTestsBase):
     def test_simple_custom_sitemap(self):
         "A simple sitemap can be rendered with a custom template"
         response = self.client.get('/simple/custom-sitemap.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <!-- This is a customised template -->
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % (self.base_url, date.today()))
+""" % (self.base_url, date.today())
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     @skipUnless(settings.USE_I18N, "Internationalization is not enabled")
     @override_settings(USE_L10N=True)
@@ -90,11 +96,12 @@ class HTTPSitemapTests(SitemapTestsBase):
         # installed doesn't raise an exception
         Site._meta.installed = False
         response = self.client.get('/simple/sitemap.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>http://testserver/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % date.today())
+""" % date.today()
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     @skipUnless("django.contrib.sites" in settings.INSTALLED_APPS,
                 "django.contrib.sites app not installed.")
@@ -131,8 +138,9 @@ class HTTPSitemapTests(SitemapTestsBase):
         Check that a cached sitemap index can be rendered (#2713).
         """
         response = self.client.get('/cached/index.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <sitemap><loc>%s/cached/sitemap-simple.xml</loc></sitemap>
 </sitemapindex>
-""" % self.base_url)
+""" % self.base_url
+        self.assertEqual(response.content, expected_content.encode('utf-8'))

django/contrib/sitemaps/tests/https.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from datetime import date
 
 from django.test.utils import override_settings
@@ -11,20 +13,22 @@ class HTTPSSitemapTests(SitemapTestsBase):
     def test_secure_sitemap_index(self):
         "A secure sitemap index can be rendered"
         response = self.client.get('/secure/index.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <sitemap><loc>%s/secure/sitemap-simple.xml</loc></sitemap>
 </sitemapindex>
-""" % self.base_url)
+""" % self.base_url
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     def test_secure_sitemap_section(self):
         "A secure sitemap section can be rendered"
         response = self.client.get('/secure/sitemap-simple.xml')
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % (self.base_url, date.today()))
+""" % (self.base_url, date.today())
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
 
 @override_settings(SECURE_PROXY_SSL_HEADER=False)
@@ -34,17 +38,19 @@ class HTTPSDetectionSitemapTests(SitemapTestsBase):
     def test_sitemap_index_with_https_request(self):
         "A sitemap index requested in HTTPS is rendered with HTTPS links"
         response = self.client.get('/simple/index.xml', **self.extra)
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <sitemap><loc>%s/simple/sitemap-simple.xml</loc></sitemap>
 </sitemapindex>
-""" % self.base_url.replace('http://', 'https://'))
+""" % self.base_url.replace('http://', 'https://')
+        self.assertEqual(response.content, expected_content.encode('utf-8'))
 
     def test_sitemap_section_with_https_request(self):
         "A sitemap section requested in HTTPS is rendered with HTTPS links"
         response = self.client.get('/simple/sitemap-simple.xml', **self.extra)
-        self.assertEqual(response.content, """<?xml version="1.0" encoding="UTF-8"?>
+        expected_content = """<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 <url><loc>%s/location/</loc><lastmod>%s</lastmod><changefreq>never</changefreq><priority>0.5</priority></url>
 </urlset>
-""" % (self.base_url.replace('http://', 'https://'), date.today()))
+""" % (self.base_url.replace('http://', 'https://'), date.today())
+        self.assertEqual(response.content, expected_content.encode('utf-8'))

django/contrib/sites/models.py

@@ -1,5 +1,6 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
+from django.utils.encoding import python_2_unicode_compatible
 
 
 SITE_CACHE = {}
@@ -32,6 +33,7 @@ class SiteManager(models.Manager):
         SITE_CACHE = {}
 
 
+@python_2_unicode_compatible
 class Site(models.Model):
 
     domain = models.CharField(_('domain name'), max_length=100)
@@ -44,7 +46,7 @@ class Site(models.Model):
         verbose_name_plural = _('sites')
         ordering = ('domain',)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.domain
 
     def save(self, *args, **kwargs):
@@ -62,6 +64,7 @@ class Site(models.Model):
             pass
 
 
+@python_2_unicode_compatible
 class RequestSite(object):
     """
     A class that shares the primary interface of Site (i.e., it has
@@ -73,7 +76,7 @@ class RequestSite(object):
     def __init__(self, request):
         self.domain = self.name = request.get_host()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.domain
 
     def save(self, force_insert=False, force_update=False):

django/contrib/staticfiles/management/commands/collectstatic.py

@@ -192,7 +192,7 @@ Type 'yes' to continue, or 'no' to cancel: """
 
     def clear_dir(self, path):
         """
-        Deletes the given relative path using the destinatin storage backend.
+        Deletes the given relative path using the destination storage backend.
         """
         dirs, files = self.storage.listdir(path)
         for f in files:

django/contrib/staticfiles/storage.py

@@ -16,7 +16,7 @@ from django.core.exceptions import ImproperlyConfigured
 from django.core.files.base import ContentFile
 from django.core.files.storage import FileSystemStorage, get_storage_class
 from django.utils.datastructures import SortedDict
-from django.utils.encoding import force_text, smart_bytes
+from django.utils.encoding import force_bytes, force_text
 from django.utils.functional import LazyObject
 from django.utils.importlib import import_module
 
@@ -51,8 +51,8 @@ class CachedFilesMixin(object):
     default_template = """url("%s")"""
     patterns = (
         ("*.css", (
-            br"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""",
+            r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""",
-            (br"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""),
+            (r"""(@import\s*["']\s*(.*?)["'])""", """@import url("%s")"""),
         )),
     )
 
@@ -87,6 +87,7 @@ class CachedFilesMixin(object):
     def hashed_name(self, name, content=None):
         parsed_name = urlsplit(unquote(name))
         clean_name = parsed_name.path.strip()
+        opened = False
         if content is None:
             if not self.exists(clean_name):
                 raise ValueError("The file '%s' could not be found with %r." %
@@ -96,9 +97,14 @@ class CachedFilesMixin(object):
             except IOError:
                 # Handle directory paths and fragments
                 return name
+            opened = True
+        try:
+            file_hash = self.file_hash(clean_name, content)
+        finally:
+            if opened:
+                content.close()
         path, filename = os.path.split(clean_name)
         root, ext = os.path.splitext(filename)
-        file_hash = self.file_hash(clean_name, content)
         if file_hash is not None:
             file_hash = ".%s" % file_hash
         hashed_name = os.path.join(path, "%s%s%s" %
@@ -112,7 +118,7 @@ class CachedFilesMixin(object):
         return urlunsplit(unparsed_name)
 
     def cache_key(self, name):
-        return 'staticfiles:%s' % hashlib.md5(smart_bytes(name)).hexdigest()
+        return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()
 
     def url(self, name, force=False):
         """
@@ -248,7 +254,7 @@ class CachedFilesMixin(object):
                     if hashed_file_exists:
                         self.delete(hashed_name)
                     # then save the processed result
-                    content_file = ContentFile(smart_bytes(content))
+                    content_file = ContentFile(force_bytes(content))
                     saved_name = self._save(hashed_name, content_file)
                     hashed_name = force_text(saved_name.replace('\\', '/'))
                     processed = True
@@ -261,7 +267,7 @@ class CachedFilesMixin(object):
                         hashed_name = force_text(saved_name.replace('\\', '/'))
 
                 # and then set the cache accordingly
-                hashed_paths[self.cache_key(name)] = hashed_name
+                hashed_paths[self.cache_key(name.replace('\\', '/'))] = hashed_name
                 yield name, hashed_name, processed
 
         # Finally set the cache

django/contrib/syndication/views.py

@@ -106,7 +106,7 @@ class Feed(object):
             subtitle = self.__get_dynamic_attr('subtitle', obj),
             link = link,
             description = self.__get_dynamic_attr('description', obj),
-            language = settings.LANGUAGE_CODE.decode(),
+            language = settings.LANGUAGE_CODE,
             feed_url = add_domain(
                 current_site.domain,
                 self.__get_dynamic_attr('feed_url', obj) or request.path,

django/core/cache/backends/base.py

@@ -1,9 +1,9 @@
 "Base Cache class."
+from __future__ import unicode_literals
 
 import warnings
 
 from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning
-from django.utils.encoding import smart_bytes
 from django.utils.importlib import import_module
 
 class InvalidCacheBackendError(ImproperlyConfigured):
@@ -23,7 +23,7 @@ def default_key_func(key, key_prefix, version):
     the `key_prefix'. KEY_FUNCTION can be used to specify an alternate
     function with custom key making behavior.
     """
-    return ':'.join([key_prefix, str(version), smart_bytes(key)])
+    return ':'.join([key_prefix, str(version), key])
 
 def get_key_func(key_func):
     """
@@ -62,7 +62,7 @@ class BaseCache(object):
         except (ValueError, TypeError):
             self._cull_frequency = 3
 
-        self.key_prefix = smart_bytes(params.get('KEY_PREFIX', ''))
+        self.key_prefix = params.get('KEY_PREFIX', '')
         self.version = params.get('VERSION', 1)
         self.key_func = get_key_func(params.get('KEY_FUNCTION', None))
 

django/core/cache/backends/db.py

@@ -12,6 +12,7 @@ from django.conf import settings
 from django.core.cache.backends.base import BaseCache
 from django.db import connections, router, transaction, DatabaseError
 from django.utils import timezone
+from django.utils.encoding import force_bytes
 
 
 class Options(object):
@@ -72,7 +73,7 @@ class DatabaseCache(BaseDatabaseCache):
             transaction.commit_unless_managed(using=db)
             return default
         value = connections[db].ops.process_clob(row[1])
-        return pickle.loads(base64.decodestring(value))
+        return pickle.loads(base64.b64decode(force_bytes(value)))
 
     def set(self, key, value, timeout=None, version=None):
         key = self.make_key(key, version=version)
@@ -103,7 +104,7 @@ class DatabaseCache(BaseDatabaseCache):
         if num > self._max_entries:
             self._cull(db, cursor, now)
         pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
-        encoded = base64.encodestring(pickled).strip()
+        encoded = base64.b64encode(pickled).strip()
         cursor.execute("SELECT cache_key, expires FROM %s "
                        "WHERE cache_key = %%s" % table, [key])
         try:
@@ -166,7 +167,7 @@ class DatabaseCache(BaseDatabaseCache):
             cursor.execute("SELECT COUNT(*) FROM %s" % table)
             num = cursor.fetchone()[0]
             if num > self._max_entries:
-                cull_num = num / self._cull_frequency
+                cull_num = num // self._cull_frequency
                 cursor.execute(
                     connections[db].ops.cache_key_culling_sql() % table,
                     [cull_num])

django/core/cache/backends/filebased.py

@@ -10,6 +10,7 @@ except ImportError:
     import pickle
 
 from django.core.cache.backends.base import BaseCache
+from django.utils.encoding import force_bytes
 
 class FileBasedCache(BaseCache):
     def __init__(self, dir, params):
@@ -136,7 +137,7 @@ class FileBasedCache(BaseCache):
         Thus, a cache key of "foo" gets turnned into a file named
         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
         """
-        path = hashlib.md5(key).hexdigest()
+        path = hashlib.md5(force_bytes(key)).hexdigest()
         path = os.path.join(path[:2], path[2:4], path[4:])
         return os.path.join(self._dir, path)
 

django/core/cache/backends/memcached.py

@@ -6,6 +6,7 @@ from threading import local
 from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
 
 from django.utils import six
+from django.utils.encoding import force_str
 
 class BaseMemcachedCache(BaseCache):
     def __init__(self, server, params, library, value_not_found_exception):
@@ -50,6 +51,10 @@ class BaseMemcachedCache(BaseCache):
             timeout += int(time.time())
         return int(timeout)
 
+    def make_key(self, key, version=None):
+        # Python 2 memcache requires the key to be a byte string.
+        return force_str(super(BaseMemcachedCache, self).make_key(key, version))
+
     def add(self, key, value, timeout=0, version=None):
         key = self.make_key(key, version=version)
         return self._cache.add(key, value, self._get_memcache_timeout(timeout))

django/core/context_processors.py

@@ -6,12 +6,15 @@ and returns a dictionary to add to the context.
 These are referenced from the setting TEMPLATE_CONTEXT_PROCESSORS and used by
 RequestContext.
 """
+from __future__ import unicode_literals
 
 from django.conf import settings
 from django.middleware.csrf import get_token
-from django.utils.encoding import smart_bytes
+from django.utils import six
+from django.utils.encoding import smart_text
 from django.utils.functional import lazy
 
+
 def csrf(request):
     """
     Context processor that provides a CSRF token, or the string 'NOTPROVIDED' if
@@ -23,10 +26,10 @@ def csrf(request):
             # In order to be able to provide debugging info in the
             # case of misconfiguration, we use a sentinel value
             # instead of returning an empty dict.
-            return b'NOTPROVIDED'
+            return 'NOTPROVIDED'
         else:
-            return smart_bytes(token)
+            return smart_text(token)
-    _get_val = lazy(_get_val, str)
+    _get_val = lazy(_get_val, six.text_type)
 
     return {'csrf_token': _get_val() }
 

django/core/files/base.py

@@ -1,11 +1,14 @@
 from __future__ import unicode_literals
 
 import os
-from io import BytesIO
+from io import BytesIO, StringIO, UnsupportedOperation
 
-from django.utils.encoding import smart_bytes, smart_text
+from django.utils.encoding import smart_text
 from django.core.files.utils import FileProxyMixin
+from django.utils import six
+from django.utils.encoding import python_2_unicode_compatible
 
+@python_2_unicode_compatible
 class File(FileProxyMixin):
     DEFAULT_CHUNK_SIZE = 64 * 2**10
 
@@ -18,9 +21,6 @@ class File(FileProxyMixin):
             self.mode = file.mode
 
     def __str__(self):
-        return smart_bytes(self.name or '')
-
-    def __unicode__(self):
         return smart_text(self.name or '')
 
     def __repr__(self):
@@ -65,8 +65,10 @@ class File(FileProxyMixin):
         if not chunk_size:
             chunk_size = self.DEFAULT_CHUNK_SIZE
 
-        if hasattr(self, 'seek'):
+        try:
             self.seek(0)
+        except (AttributeError, UnsupportedOperation):
+            pass
 
         while True:
             data = self.read(chunk_size)
@@ -124,13 +126,15 @@ class File(FileProxyMixin):
     def close(self):
         self.file.close()
 
+@python_2_unicode_compatible
 class ContentFile(File):
     """
     A File-like object that takes just raw content, rather than an actual file.
     """
     def __init__(self, content, name=None):
         content = content or b''
-        super(ContentFile, self).__init__(BytesIO(content), name=name)
+        stream_class = StringIO if isinstance(content, six.text_type) else BytesIO
+        super(ContentFile, self).__init__(stream_class(content), name=name)
         self.size = len(content)
 
     def __str__(self):

django/core/files/move.py

@@ -66,7 +66,7 @@ def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_ove
         try:
             locks.lock(fd, locks.LOCK_EX)
             current_chunk = None
-            while current_chunk != '':
+            while current_chunk != b'':
                 current_chunk = old_file.read(chunk_size)
                 os.write(fd, current_chunk)
         finally:

django/core/files/storage.py

@@ -195,10 +195,17 @@ class FileSystemStorage(Storage):
                     fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                     try:
                         locks.lock(fd, locks.LOCK_EX)
+                        _file = None
                         for chunk in content.chunks():
-                            os.write(fd, chunk)
+                            if _file is None:
+                                mode = 'wb' if isinstance(chunk, bytes) else 'wt'
+                                _file = os.fdopen(fd, mode)
+                            _file.write(chunk)
                     finally:
                         locks.unlock(fd)
+                        if _file is not None:
+                            _file.close()
+                        else:
                             os.close(fd)
             except OSError as e:
                 if e.errno == errno.EEXIST:

django/core/files/uploadedfile.py

@@ -8,7 +8,7 @@ from io import BytesIO
 from django.conf import settings
 from django.core.files.base import File
 from django.core.files import temp as tempfile
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_str
 
 __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile',
            'SimpleUploadedFile')
@@ -30,7 +30,7 @@ class UploadedFile(File):
         self.charset = charset
 
     def __repr__(self):
-        return smart_bytes("<%s: %s (%s)>" % (
+        return force_str("<%s: %s (%s)>" % (
             self.__class__.__name__, self.name, self.content_type))
 
     def _get_name(self):

django/core/files/uploadhandler.py

@@ -10,6 +10,7 @@ from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
 from django.utils import importlib
+from django.utils.encoding import python_2_unicode_compatible
 
 __all__ = ['UploadFileException','StopUpload', 'SkipFile', 'FileUploadHandler',
            'TemporaryFileUploadHandler', 'MemoryFileUploadHandler',
@@ -21,6 +22,7 @@ class UploadFileException(Exception):
     """
     pass
 
+@python_2_unicode_compatible
 class StopUpload(UploadFileException):
     """
     This exception is raised when an upload must abort.
@@ -33,7 +35,7 @@ class StopUpload(UploadFileException):
         """
         self.connection_reset = connection_reset
 
-    def __unicode__(self):
+    def __str__(self):
         if self.connection_reset:
             return 'StopUpload: Halt current upload.'
         else:

django/core/handlers/base.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import sys
+import types
 
 from django import http
 from django.core import signals
@@ -125,10 +126,10 @@ class BaseHandler(object):
 
                 # Complain if the view returned None (a common error).
                 if response is None:
-                    try:
+                    if isinstance(callback, types.FunctionType):    # FBV
-                        view_name = callback.func_name # If it's a function
+                        view_name = callback.__name__
-                    except AttributeError:
+                    else:                                           # CBV
-                        view_name = callback.__class__.__name__ + '.__call__' # If it's a class
+                        view_name = callback.__class__.__name__ + '.__call__'
                     raise ValueError("The view %s.%s didn't return an HttpResponse object." % (callback.__module__, view_name))
 
                 # If the response supports deferred rendering, apply template
@@ -152,10 +153,8 @@ class BaseHandler(object):
                         callback, param_dict = resolver.resolve404()
                         response = callback(request, **param_dict)
                     except:
-                        try:
-                            response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
-                        finally:
                         signals.got_request_exception.send(sender=self.__class__, request=request)
+                        response = self.handle_uncaught_exception(request, resolver, sys.exc_info())
             except exceptions.PermissionDenied:
                 logger.warning(
                     'Forbidden (Permission denied): %s', request.path,
@@ -167,12 +166,10 @@ class BaseHandler(object):
                     callback, param_dict = resolver.resolve403()
                     response = callback(request, **param_dict)
                 except:
-                    try:
-                        response = self.handle_uncaught_exception(request,
-                            resolver, sys.exc_info())
-                    finally:
                     signals.got_request_exception.send(
                             sender=self.__class__, request=request)
+                    response = self.handle_uncaught_exception(request,
+                            resolver, sys.exc_info())
             except SystemExit:
                 # Allow sys.exit() to actually exit. See tickets #1023 and #4701
                 raise
@@ -225,7 +222,7 @@ class BaseHandler(object):
 
         # If Http500 handler is not installed, re-raise last exception
         if resolver.urlconf_module is None:
-            six.reraise(exc_info[1], None, exc_info[2])
+            six.reraise(*exc_info)
         # Return an HttpResponse that displays a friendly error message.
         callback, param_dict = resolver.resolve500()
         return callback(request, **param_dict)

django/core/handlers/wsgi.py

@@ -9,7 +9,7 @@ from django.core import signals
 from django.core.handlers import base
 from django.core.urlresolvers import set_script_prefix
 from django.utils import datastructures
-from django.utils.encoding import force_text, smart_bytes, iri_to_uri
+from django.utils.encoding import force_str, force_text, iri_to_uri
 from django.utils.log import getLogger
 
 logger = getLogger('django.request')
@@ -246,5 +246,5 @@ class WSGIHandler(base.BaseHandler):
         response_headers = [(str(k), str(v)) for k, v in response.items()]
         for c in response.cookies.values():
             response_headers.append((str('Set-Cookie'), str(c.output(header=''))))
-        start_response(smart_bytes(status), response_headers)
+        start_response(force_str(status), response_headers)
         return response

django/core/mail/message.py

@@ -99,7 +99,12 @@ def sanitize_address(addr, encoding):
     if isinstance(addr, six.string_types):
         addr = parseaddr(force_text(addr))
     nm, addr = addr
+    # This try-except clause is needed on Python 3 < 3.2.4
+    # http://bugs.python.org/issue14291
+    try:
         nm = Header(nm, encoding).encode()
+    except UnicodeEncodeError:
+        nm = Header(nm, 'utf-8').encode()
     try:
         addr.encode('ascii')
     except UnicodeEncodeError:  # IDN

django/core/management/__init__.py

@@ -55,10 +55,15 @@ def find_management_module(app_name):
     except ImportError as e:
         if os.path.basename(os.getcwd()) != part:
             raise e
+    else:
+        if f:
+            f.close()
 
     while parts:
         part = parts.pop()
         f, path, descr = imp.find_module(part, path and [path] or None)
+        if f:
+            f.close()
     return path
 
 def load_command_class(app_name, name):

django/core/management/base.py

@@ -12,7 +12,7 @@ import traceback
 import django
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management.color import color_style
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
 from django.utils.six import StringIO
 
 
@@ -65,7 +65,7 @@ class OutputWrapper(object):
             msg += ending
         style_func = [f for f in (style_func, self.style_func, lambda x:x)
                       if f is not None][0]
-        self._out.write(smart_str(style_func(msg)))
+        self._out.write(force_str(style_func(msg)))
 
 
 class BaseCommand(object):

django/core/management/commands/inspectdb.py

@@ -1,4 +1,7 @@
+from __future__ import unicode_literals
+
 import keyword
+import re
 from optparse import make_option
 
 from django.core.management.base import NoArgsCommand, CommandError
@@ -31,6 +34,7 @@ class Command(NoArgsCommand):
         table_name_filter = options.get('table_name_filter')
 
         table2model = lambda table_name: table_name.title().replace('_', '').replace(' ', '').replace('-', '')
+        strip_prefix = lambda s: s.startswith("u'") and s[1:] or s
 
         cursor = connection.cursor()
         yield "# This is an auto-generated Django model module."
@@ -41,6 +45,7 @@ class Command(NoArgsCommand):
         yield "#"
         yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
         yield "# into your database."
+        yield "from __future__ import unicode_literals"
         yield ''
         yield 'from %s import models' % self.db_module
         yield ''
@@ -59,16 +64,19 @@ class Command(NoArgsCommand):
                 indexes = connection.introspection.get_indexes(cursor, table_name)
             except NotImplementedError:
                 indexes = {}
+            used_column_names = [] # Holds column names used in the table so far
             for i, row in enumerate(connection.introspection.get_table_description(cursor, table_name)):
-                column_name = row[0]
-                att_name = column_name.lower()
                 comment_notes = [] # Holds Field notes, to be displayed in a Python comment.
                 extra_params = {}  # Holds Field parameters such as 'db_column'.
+                column_name = row[0]
+                is_relation = i in relations
 
-                # If the column name can't be used verbatim as a Python
+                att_name, params, notes = self.normalize_col_name(
-                # attribute, set the "db_column" for this Field.
+                    column_name, used_column_names, is_relation)
-                if ' ' in att_name or '-' in att_name or keyword.iskeyword(att_name) or column_name != att_name:
+                extra_params.update(params)
-                    extra_params['db_column'] = column_name
+                comment_notes.extend(notes)
+
+                used_column_names.append(att_name)
 
                 # Add primary_key and unique, if necessary.
                 if column_name in indexes:
@@ -77,30 +85,12 @@ class Command(NoArgsCommand):
                     elif indexes[column_name]['unique']:
                         extra_params['unique'] = True
 
-                # Modify the field name to make it Python-compatible.
+                if is_relation:
-                if ' ' in att_name:
-                    att_name = att_name.replace(' ', '_')
-                    comment_notes.append('Field renamed to remove spaces.')
-
-                if '-' in att_name:
-                    att_name = att_name.replace('-', '_')
-                    comment_notes.append('Field renamed to remove dashes.')
-
-                if column_name != att_name:
-                    comment_notes.append('Field name made lowercase.')
-
-                if i in relations:
                     rel_to = relations[i][1] == table_name and "'self'" or table2model(relations[i][1])
-
                     if rel_to in known_models:
                         field_type = 'ForeignKey(%s' % rel_to
                     else:
                         field_type = "ForeignKey('%s'" % rel_to
-
-                    if att_name.endswith('_id'):
-                        att_name = att_name[:-3]
-                    else:
-                        extra_params['db_column'] = column_name
                 else:
                     # Calling `get_field_type` to get the field type string and any
                     # additional paramters and notes.
@@ -110,16 +100,6 @@ class Command(NoArgsCommand):
 
                     field_type += '('
 
-                if keyword.iskeyword(att_name):
-                    att_name += '_field'
-                    comment_notes.append('Field renamed because it was a Python reserved word.')
-
-                if att_name[0].isdigit():
-                    att_name = 'number_%s' % att_name
-                    extra_params['db_column'] = six.text_type(column_name)
-                    comment_notes.append("Field renamed because it wasn't a "
-                        "valid Python identifier.")
-
                 # Don't output 'id = meta.AutoField(primary_key=True)', because
                 # that's assumed if it doesn't exist.
                 if att_name == 'id' and field_type == 'AutoField(' and extra_params == {'primary_key': True}:
@@ -136,7 +116,9 @@ class Command(NoArgsCommand):
                 if extra_params:
                     if not field_desc.endswith('('):
                         field_desc += ', '
-                    field_desc += ', '.join(['%s=%r' % (k, v) for k, v in extra_params.items()])
+                    field_desc += ', '.join([
+                        '%s=%s' % (k, strip_prefix(repr(v)))
+                        for k, v in extra_params.items()])
                 field_desc += ')'
                 if comment_notes:
                     field_desc += ' # ' + ' '.join(comment_notes)
@@ -144,6 +126,62 @@ class Command(NoArgsCommand):
             for meta_line in self.get_meta(table_name):
                 yield meta_line
 
+    def normalize_col_name(self, col_name, used_column_names, is_relation):
+        """
+        Modify the column name to make it Python-compatible as a field name
+        """
+        field_params = {}
+        field_notes = []
+
+        new_name = col_name.lower()
+        if new_name != col_name:
+            field_notes.append('Field name made lowercase.')
+
+        if is_relation:
+            if new_name.endswith('_id'):
+                new_name = new_name[:-3]
+            else:
+                field_params['db_column'] = col_name
+
+        new_name, num_repl = re.subn(r'\W', '_', new_name)
+        if num_repl > 0:
+            field_notes.append('Field renamed to remove unsuitable characters.')
+
+        if new_name.find('__') >= 0:
+            while new_name.find('__') >= 0:
+                new_name = new_name.replace('__', '_')
+            if col_name.lower().find('__') >= 0:
+                # Only add the comment if the double underscore was in the original name
+                field_notes.append("Field renamed because it contained more than one '_' in a row.")
+
+        if new_name.startswith('_'):
+            new_name = 'field%s' % new_name
+            field_notes.append("Field renamed because it started with '_'.")
+
+        if new_name.endswith('_'):
+            new_name = '%sfield' % new_name
+            field_notes.append("Field renamed because it ended with '_'.")
+
+        if keyword.iskeyword(new_name):
+            new_name += '_field'
+            field_notes.append('Field renamed because it was a Python reserved word.')
+
+        if new_name[0].isdigit():
+            new_name = 'number_%s' % new_name
+            field_notes.append("Field renamed because it wasn't a valid Python identifier.")
+
+        if new_name in used_column_names:
+            num = 0
+            while '%s_%d' % (new_name, num) in used_column_names:
+                num += 1
+            new_name = '%s_%d' % (new_name, num)
+            field_notes.append('Field renamed because of name conflict.')
+
+        if col_name != new_name and field_notes:
+            field_params['db_column'] = col_name
+
+        return new_name, field_params, field_notes
+
     def get_field_type(self, connection, table_name, row):
         """
         Given the database connection, the table name, and the cursor row
@@ -181,6 +219,6 @@ class Command(NoArgsCommand):
         to construct the inner Meta class for the model corresponding
         to the given database table name.
         """
-        return ['    class Meta:',
+        return ["    class Meta:",
-                '        db_table = %r' % table_name,
+                "        db_table = '%s'" % table_name,
-                '']
+                ""]

django/core/management/commands/loaddata.py

@@ -196,6 +196,10 @@ class Command(BaseCommand):
                                     loaded_object_count += loaded_objects_in_fixture
                                     fixture_object_count += objects_in_fixture
                                     label_found = True
+                                except Exception as e:
+                                    if not isinstance(e, CommandError):
+                                        e.args = ("Problem installing fixture '%s': %s" % (full_path, e),)
+                                    raise
                                 finally:
                                     fixture.close()
 
@@ -209,7 +213,11 @@ class Command(BaseCommand):
             # Since we disabled constraint checks, we must manually check for
             # any invalid keys that might have been added
             table_names = [model._meta.db_table for model in models]
+            try:
                 connection.check_constraints(table_names=table_names)
+            except Exception as e:
+                e.args = ("Problem installing fixtures: %s" % e,)
+                raise
 
         except (SystemExit, KeyboardInterrupt):
             raise
@@ -217,8 +225,6 @@ class Command(BaseCommand):
             if commit:
                 transaction.rollback(using=using)
                 transaction.leave_transaction_management(using=using)
-            if not isinstance(e, CommandError):
-                e.args = ("Problem installing fixture '%s': %s" % (full_path, e),)
             raise
 
         # If we found even one object in a fixture, we need to reset the

django/core/management/commands/makemessages.py

@@ -47,32 +47,27 @@ def _popen(cmd):
     output, errors = p.communicate()
     return output, errors, p.returncode
 
-def walk(root, topdown=True, onerror=None, followlinks=False,
+def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False):
-         ignore_patterns=None, verbosity=0, stdout=sys.stdout):
     """
-    A version of os.walk that can follow symlinks for Python < 2.6
+    Helper function to get all files in the given root.
     """
-    if ignore_patterns is None:
-        ignore_patterns = []
     dir_suffix = '%s*' % os.sep
-    norm_patterns = map(lambda p: p.endswith(dir_suffix)
+    norm_patterns = [p[:-len(dir_suffix)] if p.endswith(dir_suffix) else p for p in ignore_patterns]
-                        and p[:-len(dir_suffix)] or p, ignore_patterns)
+    all_files = []
-    for dirpath, dirnames, filenames in os.walk(root, topdown, onerror):
+    for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=symlinks):
-        remove_dirs = []
+        for dirname in dirnames[:]:
-        for dirname in dirnames:
             if is_ignored(os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns):
-                remove_dirs.append(dirname)
-        for dirname in remove_dirs:
                 dirnames.remove(dirname)
                 if verbosity > 1:
                     stdout.write('ignoring directory %s\n' % dirname)
-        yield (dirpath, dirnames, filenames)
+        for filename in filenames:
-        if followlinks:
+            if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), ignore_patterns):
-            for d in dirnames:
+                if verbosity > 1:
-                p = os.path.join(dirpath, d)
+                    stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
-                if os.path.islink(p):
+            else:
-                    for link_dirpath, link_dirnames, link_filenames in walk(p):
+                all_files.extend([(dirpath, filename)])
-                        yield (link_dirpath, link_dirnames, link_filenames)
+    all_files.sort()
+    return all_files
 
 def is_ignored(path, ignore_patterns):
     """
@@ -83,23 +78,6 @@ def is_ignored(path, ignore_patterns):
             return True
     return False
 
-def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False):
-    """
-    Helper function to get all files in the given root.
-    """
-    all_files = []
-    for (dirpath, dirnames, filenames) in walk(root, followlinks=symlinks,
-            ignore_patterns=ignore_patterns, verbosity=verbosity, stdout=stdout):
-        for filename in filenames:
-            norm_filepath = os.path.normpath(os.path.join(dirpath, filename))
-            if is_ignored(norm_filepath, ignore_patterns):
-                if verbosity > 1:
-                    stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
-            else:
-                all_files.extend([(dirpath, filename)])
-    all_files.sort()
-    return all_files
-
 def copy_plural_forms(msgs, locale, domain, verbosity, stdout=sys.stdout):
     """
     Copies plural forms header contents from a Django catalog of locale to
@@ -144,7 +122,7 @@ def write_pot_file(potfile, msgs, file, work_file, is_templatized):
         msgs = '\n'.join(dropwhile(len, msgs.split('\n')))
     else:
         msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8')
-    with open(potfile, 'ab') as fp:
+    with open(potfile, 'a') as fp:
         fp.write(msgs)
 
 def process_file(file, dirpath, potfile, domain, verbosity,
@@ -252,7 +230,7 @@ def write_po_file(pofile, potfile, domain, locale, verbosity, stdout,
         msgs = copy_plural_forms(msgs, locale, domain, verbosity, stdout)
     msgs = msgs.replace(
         "#. #-#-#-#-#  %s.pot (PACKAGE VERSION)  #-#-#-#-#\n" % domain, "")
-    with open(pofile, 'wb') as fp:
+    with open(pofile, 'w') as fp:
         fp.write(msgs)
     os.unlink(potfile)
     if no_obsolete:

django/core/management/commands/shell.py

@@ -80,14 +80,14 @@ class Command(NoArgsCommand):
                 readline.parse_and_bind("tab:complete")
 
             # We want to honor both $PYTHONSTARTUP and .pythonrc.py, so follow system
-            # conventions and get $PYTHONSTARTUP first then import user.
+            # conventions and get $PYTHONSTARTUP first then .pythonrc.py.
             if not use_plain:
-                pythonrc = os.environ.get("PYTHONSTARTUP")
+                for pythonrc in (os.environ.get("PYTHONSTARTUP"),
+                                 os.path.expanduser('~/.pythonrc.py')):
                     if pythonrc and os.path.isfile(pythonrc):
                         try:
-                        execfile(pythonrc)
+                            with open(pythonrc) as handle:
+                                exec(compile(handle.read(), pythonrc, 'exec'))
                         except NameError:
                             pass
-                # This will import .pythonrc.py as a side-effect
-                import user
             code.interact(local=imported_objects)

django/core/management/commands/syncdb.py

@@ -75,7 +75,7 @@ class Command(NoArgsCommand):
                 (opts.auto_created and converter(opts.auto_created._meta.db_table) in tables))
 
         manifest = SortedDict(
-            (app_name, filter(model_installed, model_list))
+            (app_name, list(filter(model_installed, model_list)))
             for app_name, model_list in all_models
         )
 

django/core/management/sql.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import codecs
 import os
 import re
 
@@ -168,10 +169,10 @@ def custom_sql_for_model(model, style, connection):
                  os.path.join(app_dir, "%s.sql" % opts.object_name.lower())]
     for sql_file in sql_files:
         if os.path.exists(sql_file):
-            with open(sql_file, 'U') as fp:
+            with codecs.open(sql_file, 'U', encoding=settings.FILE_CHARSET) as fp:
                 # Some backends can't execute more than one SQL statement at a time,
                 # so split into separate statements.
-                output.extend(_split_statements(fp.read().decode(settings.FILE_CHARSET)))
+                output.extend(_split_statements(fp.read()))
     return output
 
 

django/core/management/templates.py

@@ -8,6 +8,8 @@ import shutil
 import stat
 import sys
 import tempfile
+import codecs
+
 try:
     from urllib.request import urlretrieve
 except ImportError:     # Python 2
@@ -154,12 +156,12 @@ class TemplateCommand(BaseCommand):
 
                 # Only render the Python files, as we don't want to
                 # accidentally render Django templates files
-                with open(old_path, 'r') as template_file:
+                with codecs.open(old_path, 'r', 'utf-8') as template_file:
                     content = template_file.read()
                 if filename.endswith(extensions) or filename in extra_files:
                     template = Template(content)
                     content = template.render(context)
-                with open(new_path, 'w') as new_file:
+                with codecs.open(new_path, 'w', 'utf-8') as new_file:
                     new_file.write(content)
 
                 if self.verbosity >= 2:

django/core/management/validation.py

@@ -1,7 +1,7 @@
 import sys
 
 from django.core.management.color import color_style
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
 from django.utils.itercompat import is_iterable
 from django.utils import six
 
@@ -13,7 +13,7 @@ class ModelErrorCollection:
 
     def add(self, context, error):
         self.errors.append((context, error))
-        self.outfile.write(self.style.ERROR(smart_str("%s: %s\n" % (context, error))))
+        self.outfile.write(self.style.ERROR(force_str("%s: %s\n" % (context, error))))
 
 def get_validation_errors(outfile, app=None):
     """

django/core/serializers/base.py

@@ -2,8 +2,6 @@
 Module for abstract serializer/unserializer base classes.
 """
 
-from io import BytesIO
-
 from django.db import models
 from django.utils.encoding import smart_text
 from django.utils import six
@@ -35,7 +33,7 @@ class Serializer(object):
         """
         self.options = options
 
-        self.stream = options.pop("stream", BytesIO())
+        self.stream = options.pop("stream", six.StringIO())
         self.selected_fields = options.pop("fields", None)
         self.use_natural_keys = options.pop("use_natural_keys", False)
 
@@ -125,7 +123,7 @@ class Deserializer(object):
         """
         self.options = options
         if isinstance(stream_or_string, six.string_types):
-            self.stream = BytesIO(stream_or_string)
+            self.stream = six.StringIO(stream_or_string)
         else:
             self.stream = stream_or_string
         # hack to make sure that the models have all been loaded before

django/core/serializers/json.py

@@ -12,7 +12,6 @@ import json
 from django.core.serializers.base import DeserializationError
 from django.core.serializers.python import Serializer as PythonSerializer
 from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_bytes
 from django.utils import six
 from django.utils.timezone import is_aware
 
@@ -61,13 +60,12 @@ def Deserializer(stream_or_string, **options):
     """
     Deserialize a stream or string of JSON data.
     """
+    if not isinstance(stream_or_string, (bytes, six.string_types)):
+        stream_or_string = stream_or_string.read()
     if isinstance(stream_or_string, bytes):
         stream_or_string = stream_or_string.decode('utf-8')
     try:
-        if isinstance(stream_or_string, six.string_types):
         objects = json.loads(stream_or_string)
-        else:
-            objects = json.load(stream_or_string)
         for obj in PythonDeserializer(objects, **options):
             yield obj
     except GeneratorExit:

django/core/serializers/pyyaml.py

@@ -12,7 +12,6 @@ from django.db import models
 from django.core.serializers.base import DeserializationError
 from django.core.serializers.python import Serializer as PythonSerializer
 from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_bytes
 from django.utils import six
 
 

django/core/servers/basehttp.py

@@ -7,6 +7,8 @@ This is a simple server for use in testing or debugging Django apps. It hasn't
 been reviewed for security issues. DON'T USE IT FOR PRODUCTION USE!
 """
 
+from __future__ import unicode_literals
+
 import os
 import socket
 import sys
@@ -71,12 +73,12 @@ class WSGIServerException(Exception):
 
 
 class ServerHandler(simple_server.ServerHandler, object):
-    error_status = "500 INTERNAL SERVER ERROR"
+    error_status = str("500 INTERNAL SERVER ERROR")
 
     def write(self, data):
-        """'write()' callable as specified by PEP 333"""
+        """'write()' callable as specified by PEP 3333"""
 
-        assert isinstance(data, str), "write() argument must be string"
+        assert isinstance(data, bytes), "write() argument must be bytestring"
 
         if not self.status:
             raise AssertionError("write() before start_response()")
@@ -200,7 +202,7 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler, object):
 def run(addr, port, wsgi_handler, ipv6=False, threading=False):
     server_address = (addr, port)
     if threading:
-        httpd_cls = type('WSGIServer', (socketserver.ThreadingMixIn, WSGIServer), {})
+        httpd_cls = type(str('WSGIServer'), (socketserver.ThreadingMixIn, WSGIServer), {})
     else:
         httpd_cls = WSGIServer
     httpd = httpd_cls(server_address, WSGIRequestHandler, ipv6=ipv6)

django/core/signing.py

@@ -32,6 +32,9 @@ start of the base64 JSON.
 There are 65 url-safe characters: the 64 used by url-safe base64 and the ':'.
 These functions make use of all of them.
 """
+
+from __future__ import unicode_literals
+
 import base64
 import json
 import time
@@ -41,7 +44,7 @@ from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.utils import baseconv
 from django.utils.crypto import constant_time_compare, salted_hmac
-from django.utils.encoding import force_text, smart_bytes
+from django.utils.encoding import force_bytes, force_str, force_text
 from django.utils.importlib import import_module
 
 
@@ -60,11 +63,11 @@ class SignatureExpired(BadSignature):
 
 
 def b64_encode(s):
-    return base64.urlsafe_b64encode(s).strip('=')
+    return base64.urlsafe_b64encode(s).strip(b'=')
 
 
 def b64_decode(s):
-    pad = '=' * (-len(s) % 4)
+    pad = b'=' * (-len(s) % 4)
     return base64.urlsafe_b64decode(s + pad)
 
 
@@ -114,7 +117,7 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer,
     value or re-using a salt value across different parts of your
     application without good cause is a security risk.
     """
-    data = serializer().dumps(obj)
+    data = force_bytes(serializer().dumps(obj))
 
     # Flag for if it's been compressed or not
     is_compressed = False
@@ -127,7 +130,7 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer,
             is_compressed = True
     base64d = b64_encode(data)
     if is_compressed:
-        base64d = '.' + base64d
+        base64d = b'.' + base64d
     return TimestampSigner(key, salt=salt).sign(base64d)
 
 
@@ -135,35 +138,40 @@ def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, ma
     """
     Reverse of dumps(), raises BadSignature if signature fails
     """
-    base64d = smart_bytes(
+    # TimestampSigner.unsign always returns unicode but base64 and zlib
-        TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+    # compression operate on bytes.
+    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
     decompress = False
-    if base64d[0] == '.':
+    if base64d[0] == b'.':
         # It's compressed; uncompress it first
         base64d = base64d[1:]
         decompress = True
     data = b64_decode(base64d)
     if decompress:
         data = zlib.decompress(data)
-    return serializer().loads(data)
+    return serializer().loads(force_str(data))
 
 
 class Signer(object):
+
     def __init__(self, key=None, sep=':', salt=None):
-        self.sep = sep
+        # Use of native strings in all versions of Python
-        self.key = key or settings.SECRET_KEY
+        self.sep = str(sep)
-        self.salt = salt or ('%s.%s' %
+        self.key = str(key or settings.SECRET_KEY)
-            (self.__class__.__module__, self.__class__.__name__))
+        self.salt = str(salt or
+            '%s.%s' % (self.__class__.__module__, self.__class__.__name__))
 
     def signature(self, value):
-        return base64_hmac(self.salt + 'signer', value, self.key)
+        signature = base64_hmac(self.salt + 'signer', value, self.key)
+        # Convert the signature from bytes to str only on Python 3
+        return force_str(signature)
 
     def sign(self, value):
-        value = smart_bytes(value)
+        value = force_str(value)
-        return '%s%s%s' % (value, self.sep, self.signature(value))
+        return str('%s%s%s') % (value, self.sep, self.signature(value))
 
     def unsign(self, signed_value):
-        signed_value = smart_bytes(signed_value)
+        signed_value = force_str(signed_value)
         if not self.sep in signed_value:
             raise BadSignature('No "%s" found in value' % self.sep)
         value, sig = signed_value.rsplit(self.sep, 1)
@@ -178,8 +186,9 @@ class TimestampSigner(Signer):
         return baseconv.base62.encode(int(time.time()))
 
     def sign(self, value):
-        value = smart_bytes('%s%s%s' % (value, self.sep, self.timestamp()))
+        value = force_str(value)
-        return '%s%s%s' % (value, self.sep, self.signature(value))
+        value = str('%s%s%s') % (value, self.sep, self.timestamp())
+        return super(TimestampSigner, self).sign(value)
 
     def unsign(self, value, max_age=None):
         result =  super(TimestampSigner, self).unsign(value)

django/core/urlresolvers.py

@@ -14,7 +14,7 @@ from threading import local
 from django.http import Http404
 from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
 from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import iri_to_uri, force_text, smart_bytes
+from django.utils.encoding import force_str, force_text, iri_to_uri
 from django.utils.functional import memoize, lazy
 from django.utils.importlib import import_module
 from django.utils.module_loading import module_has_submodule
@@ -89,18 +89,11 @@ def get_callable(lookup_view, can_fail=False):
     """
     if not callable(lookup_view):
         mod_name, func_name = get_mod_func(lookup_view)
+        if func_name == '':
+            return lookup_view
+
         try:
-            if func_name != '':
+            mod = import_module(mod_name)
-                lookup_view = getattr(import_module(mod_name), func_name)
-                if not callable(lookup_view):
-                    raise ViewDoesNotExist(
-                        "Could not import %s.%s. View is not callable." %
-                        (mod_name, func_name))
-        except AttributeError:
-            if not can_fail:
-                raise ViewDoesNotExist(
-                    "Could not import %s. View does not exist in module %s." %
-                    (lookup_view, mod_name))
         except ImportError:
             parentmod, submod = get_mod_func(mod_name)
             if (not can_fail and submod != '' and
@@ -110,6 +103,18 @@ def get_callable(lookup_view, can_fail=False):
                     (lookup_view, mod_name))
             if not can_fail:
                 raise
+        else:
+            try:
+                lookup_view = getattr(mod, func_name)
+                if not callable(lookup_view):
+                    raise ViewDoesNotExist(
+                        "Could not import %s.%s. View is not callable." %
+                        (mod_name, func_name))
+            except AttributeError:
+                if not can_fail:
+                    raise ViewDoesNotExist(
+                        "Could not import %s. View does not exist in module %s." %
+                        (lookup_view, mod_name))
     return lookup_view
 get_callable = memoize(get_callable, _callable_cache, 1)
 
@@ -190,7 +195,7 @@ class RegexURLPattern(LocaleRegexProvider):
         self.name = name
 
     def __repr__(self):
-        return smart_bytes('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
+        return force_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
 
     def add_prefix(self, prefix):
         """
@@ -240,7 +245,14 @@ class RegexURLResolver(LocaleRegexProvider):
         self._app_dict = {}
 
     def __repr__(self):
-        return smart_bytes('<%s %s (%s:%s) %s>' % (self.__class__.__name__, self.urlconf_name, self.app_name, self.namespace, self.regex.pattern))
+        if isinstance(self.urlconf_name, list) and len(self.urlconf_name):
+            # Don't bother to output the whole list, it can be huge
+            urlconf_repr = '<%s list>' % self.urlconf_name[0].__class__.__name__
+        else:
+            urlconf_repr = repr(self.urlconf_name)
+        return force_str('<%s %s (%s:%s) %s>' % (
+            self.__class__.__name__, urlconf_repr, self.app_name,
+            self.namespace, self.regex.pattern))
 
     def _populate(self):
         lookups = MultiValueDict()

django/core/validators.py

@@ -8,7 +8,7 @@ except ImportError:     # Python 2
 
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_text
+from django.utils.encoding import force_text
 from django.utils.ipv6 import is_valid_ipv6_address
 from django.utils import six
 
@@ -36,7 +36,7 @@ class RegexValidator(object):
         """
         Validates that the input matches the regular expression.
         """
-        if not self.regex.search(smart_text(value)):
+        if not self.regex.search(force_text(value)):
             raise ValidationError(self.message, code=self.code)
 
 class URLValidator(RegexValidator):
@@ -44,7 +44,8 @@ class URLValidator(RegexValidator):
         r'^(?:http|ftp)s?://' # http:// or https://
         r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
         r'localhost|' #localhost...
-        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
+        r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
         r'(?::\d+)?' # optional port
         r'(?:/?|[/?]\S+)$', re.IGNORECASE)
 
@@ -54,10 +55,10 @@ class URLValidator(RegexValidator):
         except ValidationError as e:
             # Trivial case failed. Try for possible IDN domain
             if value:
-                value = smart_text(value)
+                value = force_text(value)
                 scheme, netloc, path, query, fragment = urlsplit(value)
                 try:
-                    netloc = netloc.encode('idna') # IDN -> ACE
+                    netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
                 except UnicodeError: # invalid domain part
                     raise e
                 url = urlunsplit((scheme, netloc, path, query, fragment))
@@ -84,7 +85,7 @@ class EmailValidator(RegexValidator):
             if value and '@' in value:
                 parts = value.split('@')
                 try:
-                    parts[-1] = parts[-1].encode('idna')
+                    parts[-1] = parts[-1].encode('idna').decode('ascii')
                 except UnicodeError:
                     raise e
                 super(EmailValidator, self).__call__('@'.join(parts))
@@ -99,7 +100,7 @@ email_re = re.compile(
     r'|\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$', re.IGNORECASE)  # literal form, ipv4 address (SMTP 4.1.3)
 validate_email = EmailValidator(email_re, _('Enter a valid e-mail address.'), 'invalid')
 
-slug_re = re.compile(r'^[-\w]+$')
+slug_re = re.compile(r'^[-a-zA-Z0-9_]+$')
 validate_slug = RegexValidator(slug_re, _("Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens."), 'invalid')
 
 ipv4_re = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}$')

django/db/backends/__init__.py

@@ -1,7 +1,7 @@
 from django.db.utils import DatabaseError
 
 try:
-    import thread
+    from django.utils.six.moves import _thread as thread
 except ImportError:
     from django.utils.six.moves import _dummy_thread as thread
 from contextlib import contextmanager
@@ -47,6 +47,8 @@ class BaseDatabaseWrapper(object):
     def __ne__(self, other):
         return not self == other
 
+    __hash__ = object.__hash__
+
     def _commit(self):
         if self.connection is not None:
             return self.connection.commit()
@@ -621,7 +623,7 @@ class BaseDatabaseOperations(object):
         exists for database backends to provide a better implementation
         according to their own quoting schemes.
         """
-        from django.utils.encoding import smart_text, force_text
+        from django.utils.encoding import force_text
 
         # Convert params to contain Unicode values.
         to_unicode = lambda s: force_text(s, strings_only=True, errors='replace')
@@ -630,7 +632,7 @@ class BaseDatabaseOperations(object):
         else:
             u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()])
 
-        return smart_text(sql) % u_params
+        return force_text(sql) % u_params
 
     def last_insert_id(self, cursor, table_name, pk_name):
         """
@@ -814,8 +816,8 @@ class BaseDatabaseOperations(object):
 
     def prep_for_like_query(self, x):
         """Prepares a value for use in a LIKE query."""
-        from django.utils.encoding import smart_text
+        from django.utils.encoding import force_text
-        return smart_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
+        return force_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
 
     # Same as prep_for_like_query(), but called for "iexact" matches, which
     # need not necessarily be implemented using "LIKE" in the backend.
@@ -892,19 +894,21 @@ class BaseDatabaseOperations(object):
         return self.year_lookup_bounds(value)
 
     def convert_values(self, value, field):
-        """Coerce the value returned by the database backend into a consistent type that
+        """
-        is compatible with the field type.
+        Coerce the value returned by the database backend into a consistent type
+        that is compatible with the field type.
         """
         internal_type = field.get_internal_type()
         if internal_type == 'DecimalField':
             return value
-        elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField':
+        elif internal_type == 'FloatField':
+            return float(value)
+        elif (internal_type and (internal_type.endswith('IntegerField')
+                                 or internal_type == 'AutoField')):
             return int(value)
         elif internal_type in ('DateField', 'DateTimeField', 'TimeField'):
             return value
-        # No field, or the field isn't known to be a decimal or integer
+        return value
-        # Default to a float
-        return float(value)
 
     def check_aggregate_support(self, aggregate_func):
         """Check that the backend supports the provided aggregate
@@ -1003,7 +1007,7 @@ class BaseDatabaseIntrospection(object):
             for model in models.get_models(app):
                 if router.allow_syncdb(self.connection.alias, model):
                     all_models.append(model)
-        tables = map(self.table_name_converter, tables)
+        tables = list(map(self.table_name_converter, tables))
         return set([
             m for m in all_models
             if self.table_name_converter(m._meta.db_table) in tables

django/db/backends/mysql/base.py

@@ -39,7 +39,7 @@ from django.db.backends.mysql.introspection import DatabaseIntrospection
 from django.db.backends.mysql.validation import DatabaseValidation
 from django.db.backends.mysql.schema import DatabaseSchemaEditor
 from django.utils.functional import cached_property
-from django.utils.safestring import SafeString, SafeUnicode
+from django.utils.safestring import SafeBytes, SafeText
 from django.utils import six
 from django.utils import timezone
 
@@ -76,7 +76,7 @@ def adapt_datetime_with_timezone_support(value, conv):
 # MySQLdb-1.2.1 returns TIME columns as timedelta -- they are more like
 # timedelta in terms of actual behavior as they are signed and include days --
 # and Django expects time, so we still need to override that. We also need to
-# add special handling for SafeUnicode and SafeString as MySQLdb's type
+# add special handling for SafeText and SafeBytes as MySQLdb's type
 # checking is too tight to catch those (see Django ticket #6052).
 # Finally, MySQLdb always returns naive datetime objects. However, when
 # timezone support is active, Django expects timezone-aware datetime objects.
@@ -403,8 +403,8 @@ class DatabaseWrapper(BaseDatabaseWrapper):
             kwargs['client_flag'] = CLIENT.FOUND_ROWS
             kwargs.update(settings_dict['OPTIONS'])
             self.connection = Database.connect(**kwargs)
-            self.connection.encoders[SafeUnicode] = self.connection.encoders[six.text_type]
+            self.connection.encoders[SafeText] = self.connection.encoders[six.text_type]
-            self.connection.encoders[SafeString] = self.connection.encoders[bytes]
+            self.connection.encoders[SafeBytes] = self.connection.encoders[bytes]
             connection_created.send(sender=self.__class__, connection=self)
         cursor = self.connection.cursor()
         if new_connection:

django/db/backends/mysql/compiler.py

@@ -1,10 +1,16 @@
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest
+
 from django.db.models.sql import compiler
 
+
 class SQLCompiler(compiler.SQLCompiler):
     def resolve_columns(self, row, fields=()):
         values = []
-        index_extra_select = len(self.query.extra_select.keys())
+        index_extra_select = len(self.query.extra_select)
-        for value, field in map(None, row[index_extra_select:], fields):
+        for value, field in zip_longest(row[index_extra_select:], fields):
             if (field and field.get_internal_type() in ("BooleanField", "NullBooleanField") and
                 value in (0, 1)):
                 value = bool(value)

django/db/backends/mysql/introspection.py

@@ -1,8 +1,9 @@
+import re
+from .base import FIELD_TYPE
+
 from django.db.backends import BaseDatabaseIntrospection
 from django.utils import six
-from MySQLdb import ProgrammingError, OperationalError
+
-from MySQLdb.constants import FIELD_TYPE
-import re
 
 foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")
 
@@ -35,9 +36,20 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
         return [row[0] for row in cursor.fetchall()]
 
     def get_table_description(self, cursor, table_name):
-        "Returns a description of the table, with the DB-API cursor.description interface."
+        """
+        Returns a description of the table, with the DB-API cursor.description interface."
+        """
+        # varchar length returned by cursor.description is an internal length,
+        # not visible length (#5725), use information_schema database to fix this
+        cursor.execute("""
+            SELECT column_name, character_maximum_length FROM information_schema.columns
+            WHERE table_name = %s AND table_schema = DATABASE()
+                AND character_maximum_length IS NOT NULL""", [table_name])
+        length_map = dict(cursor.fetchall())
+
         cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
-        return cursor.description
+        return [line[:3] + (length_map.get(line[0], line[3]),) + line[4:]
+            for line in cursor.description]
 
     def _name_to_index(self, cursor, table_name):
         """

django/db/backends/oracle/base.py

@@ -10,8 +10,6 @@ import decimal
 import sys
 import warnings
 
-from django.utils import six
-
 def _setup_environment(environ):
     import platform
     # Cygwin requires some special voodoo to set the environment variables
@@ -53,7 +51,7 @@ from django.db.backends.signals import connection_created
 from django.db.backends.oracle.client import DatabaseClient
 from django.db.backends.oracle.creation import DatabaseCreation
 from django.db.backends.oracle.introspection import DatabaseIntrospection
-from django.utils.encoding import smart_bytes, force_text
+from django.utils.encoding import force_bytes, force_text
 from django.utils import six
 from django.utils import timezone
 
@@ -66,7 +64,7 @@ IntegrityError = Database.IntegrityError
 if int(Database.version.split('.', 1)[0]) >= 5 and not hasattr(Database, 'UNICODE'):
     convert_unicode = force_text
 else:
-    convert_unicode = smart_bytes
+    convert_unicode = force_bytes
 
 
 class DatabaseFeatures(BaseDatabaseFeatures):
@@ -221,6 +219,9 @@ WHEN (new.%(col_name)s IS NULL)
     def last_executed_query(self, cursor, sql, params):
         # http://cx-oracle.sourceforge.net/html/cursor.html#Cursor.statement
         # The DB API definition does not define this attribute.
+        if six.PY3:
+            return cursor.statement
+        else:
             return cursor.statement.decode("utf-8")
 
     def last_insert_id(self, cursor, table_name, pk_name):
@@ -594,10 +595,16 @@ class OracleParam(object):
                 param = timezone.make_aware(param, default_timezone)
             param = param.astimezone(timezone.utc).replace(tzinfo=None)
 
+        # Oracle doesn't recognize True and False correctly in Python 3.
+        # The conversion done below works both in 2 and 3.
+        if param is True:
+            param = "1"
+        elif param is False:
+            param = "0"
         if hasattr(param, 'bind_parameter'):
-            self.smart_bytes = param.bind_parameter(cursor)
+            self.force_bytes = param.bind_parameter(cursor)
         else:
-            self.smart_bytes = convert_unicode(param, cursor.charset,
+            self.force_bytes = convert_unicode(param, cursor.charset,
                                              strings_only)
         if hasattr(param, 'input_size'):
             # If parameter has `input_size` attribute, use that.
@@ -676,7 +683,7 @@ class FormatStylePlaceholderCursor(object):
         self.setinputsizes(*sizes)
 
     def _param_generator(self, params):
-        return [p.smart_bytes for p in params]
+        return [p.force_bytes for p in params]
 
     def execute(self, query, params=None):
         if params is None:

django/db/backends/oracle/compiler.py

@@ -1,4 +1,9 @@
 from django.db.models.sql import compiler
+# The izip_longest was renamed to zip_longest in py3
+try:
+    from itertools import zip_longest
+except ImportError:
+    from itertools import izip_longest as zip_longest
 
 
 class SQLCompiler(compiler.SQLCompiler):
@@ -10,10 +15,10 @@ class SQLCompiler(compiler.SQLCompiler):
             rn_offset = 1
         else:
             rn_offset = 0
-        index_start = rn_offset + len(self.query.extra_select.keys())
+        index_start = rn_offset + len(self.query.extra_select)
         values = [self.query.convert_values(v, None, connection=self.connection)
                   for v in row[rn_offset:index_start]]
-        for value, field in map(None, row[index_start:], fields):
+        for value, field in zip_longest(row[index_start:], fields):
             values.append(self.query.convert_values(value, field, connection=self.connection))
         return tuple(values)
 

django/db/backends/postgresql_psycopg2/base.py

@@ -15,7 +15,7 @@ from django.db.backends.postgresql_psycopg2.version import get_version
 from django.db.backends.postgresql_psycopg2.introspection import DatabaseIntrospection
 from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.utils.log import getLogger
-from django.utils.safestring import SafeUnicode, SafeString
+from django.utils.safestring import SafeText, SafeBytes
 from django.utils import six
 from django.utils.timezone import utc
 
@@ -30,8 +30,8 @@ DatabaseError = Database.DatabaseError
 IntegrityError = Database.IntegrityError
 
 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
-psycopg2.extensions.register_adapter(SafeString, psycopg2.extensions.QuotedString)
+psycopg2.extensions.register_adapter(SafeBytes, psycopg2.extensions.QuotedString)
-psycopg2.extensions.register_adapter(SafeUnicode, psycopg2.extensions.QuotedString)
+psycopg2.extensions.register_adapter(SafeText, psycopg2.extensions.QuotedString)
 
 logger = getLogger('django.db.backends')
 

django/db/backends/postgresql_psycopg2/introspection.py

@@ -45,7 +45,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
             WHERE table_name = %s""", [table_name])
         null_map = dict(cursor.fetchall())
         cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
-        return [tuple([item for item in line[:6]] + [null_map[line[0]]=='YES'])
+        return [line[:6] + (null_map[line[0]]=='YES',)
                 for line in cursor.description]
 
     def get_relations(self, cursor, table_name):