Merge remote-tracking branch 'core/master' into schema-alteration

Conflicts:
	django/db/backends/oracle/base.py
	django/db/backends/postgresql_psycopg2/base.py
	django/db/models/signals.py
	tests/queries/tests.py

django/conf/global_settings.py

@@ -475,6 +475,7 @@ SESSION_SAVE_EVERY_REQUEST = False                      # Whether to save the se
 SESSION_EXPIRE_AT_BROWSER_CLOSE = False                 # Whether a user's session cookie expires when the Web browser is closed.
 SESSION_ENGINE = 'django.contrib.sessions.backends.db'  # The module to store session data
 SESSION_FILE_PATH = None                                # Directory to store session files if using the file session module. If None, the backend will use a sensible default.
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer'  # class to serialize session data
 
 #########
 # CACHE #

django/contrib/admin/options.py

@@ -107,6 +107,7 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)):
         validator.validate(cls, model)
 
     def __init__(self):
+        self._orig_formfield_overrides = self.formfield_overrides
         overrides = FORMFIELD_FOR_DBFIELD_DEFAULTS.copy()
         overrides.update(self.formfield_overrides)
         self.formfield_overrides = overrides
@@ -123,6 +124,9 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)):
         # If the field specifies choices, we don't need to look for special
         # admin widgets - we just need to use a select widget of some kind.
         if db_field.choices:
+            # see #19303 for an explanation of self._orig_formfield_overrides
+            if db_field.__class__ in self._orig_formfield_overrides:
+                kwargs = dict(self._orig_formfield_overrides[db_field.__class__], **kwargs)
             return self.formfield_for_choice_field(db_field, request, **kwargs)
 
         # ForeignKey or ManyToManyFields

django/contrib/admin/templatetags/admin_list.py

@@ -9,6 +9,7 @@ from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE,
     ORDER_VAR, PAGE_VAR, SEARCH_VAR)
 from django.contrib.admin.templatetags.admin_static import static
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.urlresolvers import NoReverseMatch
 from django.db import models
 from django.utils import formats
 from django.utils.html import escapejs, format_html
@@ -216,25 +217,36 @@ def items_for_result(cl, result, form):
         row_class = mark_safe(' class="%s"' % ' '.join(row_classes))
         # If list_display_links not defined, add the link tag to the first field
         if (first and not cl.list_display_links) or field_name in cl.list_display_links:
-            table_tag = {True:'th', False:'td'}[first]
+            table_tag = 'th' if first else 'td'
             first = False
-            url = cl.url_for_result(result)
+
-            url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url)
+            # Display link to the result's change_view if the url exists, else
-            # Convert the pk to something that can be used in Javascript.
+            # display just the result's representation.
-            # Problem cases are long ints (23L) and non-ASCII strings.
+            try:
-            if cl.to_field:
+                url = cl.url_for_result(result)
-                attr = str(cl.to_field)
+            except NoReverseMatch:
+                link_or_text = result_repr
             else:
-                attr = pk
+                url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url)
-            value = result.serializable_value(attr)
+                # Convert the pk to something that can be used in Javascript.
-            result_id = escapejs(value)
+                # Problem cases are long ints (23L) and non-ASCII strings.
-            yield format_html('<{0}{1}><a href="{2}"{3}>{4}</a></{5}>',
+                if cl.to_field:
+                    attr = str(cl.to_field)
+                else:
+                    attr = pk
+                value = result.serializable_value(attr)
+                result_id = escapejs(value)
+                link_or_text = format_html(
+                    '<a href="{0}"{1}>{2}</a>',
+                    url,
+                    format_html(' onclick="opener.dismissRelatedLookupPopup(window, &#39;{0}&#39;); return false;"', result_id)
+                        if cl.is_popup else '',
+                    result_repr)
+
+            yield format_html('<{0}{1}>{2}</{3}>',
                               table_tag,
                               row_class,
-                              url,
+                              link_or_text,
-                              format_html(' onclick="opener.dismissRelatedLookupPopup(window, &#39;{0}&#39;); return false;"', result_id)
-                                if cl.is_popup else '',
-                              result_repr,
                               table_tag)
         else:
             # By default the fields come from ModelAdmin.list_editable, but if we pull

django/contrib/messages/storage/session.py

@@ -1,4 +1,8 @@
+import json
+
 from django.contrib.messages.storage.base import BaseStorage
+from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder
+from django.utils import six
 
 
 class SessionStorage(BaseStorage):
@@ -20,14 +24,23 @@ class SessionStorage(BaseStorage):
         always stores everything it is given, so return True for the
         all_retrieved flag.
         """
-        return self.request.session.get(self.session_key), True
+        return self.deserialize_messages(self.request.session.get(self.session_key)), True
 
     def _store(self, messages, response, *args, **kwargs):
         """
         Stores a list of messages to the request's session.
         """
         if messages:
-            self.request.session[self.session_key] = messages
+            self.request.session[self.session_key] = self.serialize_messages(messages)
         else:
             self.request.session.pop(self.session_key, None)
         return []
+
+    def serialize_messages(self, messages):
+        encoder = MessageEncoder(separators=(',', ':'))
+        return encoder.encode(messages)
+
+    def deserialize_messages(self, data):
+        if data and isinstance(data, six.string_types):
+            return json.loads(data, cls=MessageDecoder)
+        return data

django/contrib/messages/tests/base.py

@@ -61,6 +61,7 @@ class BaseTests(object):
             MESSAGE_TAGS    = '',
             MESSAGE_STORAGE = '%s.%s' % (self.storage_class.__module__,
                                          self.storage_class.__name__),
+            SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer',
         )
         self.settings_override.enable()
 

django/contrib/messages/tests/test_session.py

@@ -11,13 +11,13 @@ def set_session_data(storage, messages):
     Sets the messages into the backend request's session and remove the
     backend's loaded data cache.
     """
-    storage.request.session[storage.session_key] = messages
+    storage.request.session[storage.session_key] = storage.serialize_messages(messages)
     if hasattr(storage, '_loaded_data'):
         del storage._loaded_data
 
 
 def stored_session_messages_count(storage):
-    data = storage.request.session.get(storage.session_key, [])
+    data = storage.deserialize_messages(storage.request.session.get(storage.session_key, []))
     return len(data)
 
 

django/contrib/sessions/backends/base.py

@@ -3,11 +3,6 @@ from __future__ import unicode_literals
 import base64
 from datetime import datetime, timedelta
 import logging
-
-try:
-    from django.utils.six.moves import cPickle as pickle
-except ImportError:
-    import pickle
 import string
 
 from django.conf import settings
@@ -17,6 +12,7 @@ from django.utils.crypto import get_random_string
 from django.utils.crypto import salted_hmac
 from django.utils import timezone
 from django.utils.encoding import force_bytes, force_text
+from django.utils.module_loading import import_by_path
 
 from django.contrib.sessions.exceptions import SuspiciousSession
 
@@ -42,6 +38,7 @@ class SessionBase(object):
         self._session_key = session_key
         self.accessed = False
         self.modified = False
+        self.serializer = import_by_path(settings.SESSION_SERIALIZER)
 
     def __contains__(self, key):
         return key in self._session
@@ -86,21 +83,21 @@ class SessionBase(object):
         return salted_hmac(key_salt, value).hexdigest()
 
     def encode(self, session_dict):
-        "Returns the given session dictionary pickled and encoded as a string."
+        "Returns the given session dictionary serialized and encoded as a string."
-        pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL)
+        serialized = self.serializer().dumps(session_dict)
-        hash = self._hash(pickled)
+        hash = self._hash(serialized)
-        return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii')
+        return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii')
 
     def decode(self, session_data):
         encoded_data = base64.b64decode(force_bytes(session_data))
         try:
             # could produce ValueError if there is no ':'
-            hash, pickled = encoded_data.split(b':', 1)
+            hash, serialized = encoded_data.split(b':', 1)
-            expected_hash = self._hash(pickled)
+            expected_hash = self._hash(serialized)
             if not constant_time_compare(hash.decode(), expected_hash):
                 raise SuspiciousSession("Session data corrupted")
             else:
-                return pickle.loads(pickled)
+                return self.serializer().loads(serialized)
         except Exception as e:
             # ValueError, SuspiciousOperation, unpickling exceptions. If any of
             # these happen, just return an empty dictionary (an empty session).

django/contrib/sessions/backends/signed_cookies.py

@@ -1,26 +1,9 @@
-try:
-    from django.utils.six.moves import cPickle as pickle
-except ImportError:
-    import pickle
-
 from django.conf import settings
 from django.core import signing
 
 from django.contrib.sessions.backends.base import SessionBase
 
 
-class PickleSerializer(object):
-    """
-    Simple wrapper around pickle to be used in signing.dumps and
-    signing.loads.
-    """
-    def dumps(self, obj):
-        return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
-
-    def loads(self, data):
-        return pickle.loads(data)
-
-
 class SessionStore(SessionBase):
 
     def load(self):
@@ -31,7 +14,7 @@ class SessionStore(SessionBase):
         """
         try:
             return signing.loads(self.session_key,
-                serializer=PickleSerializer,
+                serializer=self.serializer,
                 # This doesn't handle non-default expiry dates, see #19201
                 max_age=settings.SESSION_COOKIE_AGE,
                 salt='django.contrib.sessions.backends.signed_cookies')
@@ -91,7 +74,7 @@ class SessionStore(SessionBase):
         session_cache = getattr(self, '_session_cache', {})
         return signing.dumps(session_cache, compress=True,
             salt='django.contrib.sessions.backends.signed_cookies',
-            serializer=PickleSerializer)
+            serializer=self.serializer)
 
     @classmethod
     def clear_expired(cls):

django/contrib/sessions/models.py

@@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _
 class SessionManager(models.Manager):
     def encode(self, session_dict):
         """
-        Returns the given session dictionary pickled and encoded as a string.
+        Returns the given session dictionary serialized and encoded as a string.
         """
         return SessionStore().encode(session_dict)
 

django/contrib/sessions/serializers.py

@@ -0,0 +1,20 @@
+from django.core.signing import JSONSerializer as BaseJSONSerializer
+try:
+    from django.utils.six.moves import cPickle as pickle
+except ImportError:
+    import pickle
+
+
+class PickleSerializer(object):
+    """
+    Simple wrapper around pickle to be used in signing.dumps and
+    signing.loads.
+    """
+    def dumps(self, obj):
+        return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
+
+    def loads(self, data):
+        return pickle.loads(data)
+
+
+JSONSerializer = BaseJSONSerializer

django/contrib/sessions/tests.py

@@ -285,21 +285,25 @@ class SessionTestsMixin(object):
 
 
     def test_actual_expiry(self):
-        # Regression test for #19200
+        # this doesn't work with JSONSerializer (serializing timedelta)
-        old_session_key = None
+        with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'):
-        new_session_key = None
+            self.session = self.backend()  # reinitialize after overriding settings
-        try:
+
-            self.session['foo'] = 'bar'
+            # Regression test for #19200
-            self.session.set_expiry(-timedelta(seconds=10))
+            old_session_key = None
-            self.session.save()
+            new_session_key = None
-            old_session_key = self.session.session_key
+            try:
-            # With an expiry date in the past, the session expires instantly.
+                self.session['foo'] = 'bar'
-            new_session = self.backend(self.session.session_key)
+                self.session.set_expiry(-timedelta(seconds=10))
-            new_session_key = new_session.session_key
+                self.session.save()
-            self.assertNotIn('foo', new_session)
+                old_session_key = self.session.session_key
-        finally:
+                # With an expiry date in the past, the session expires instantly.
-            self.session.delete(old_session_key)
+                new_session = self.backend(self.session.session_key)
-            self.session.delete(new_session_key)
+                new_session_key = new_session.session_key
+                self.assertNotIn('foo', new_session)
+            finally:
+                self.session.delete(old_session_key)
+                self.session.delete(new_session_key)
 
 
 class DatabaseSessionTests(SessionTestsMixin, TestCase):

django/db/backends/oracle/base.py

@@ -96,6 +96,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     nulls_order_largest = True
     requires_literal_defaults = True
     connection_persists_old_columns = True
+    nulls_order_largest = True
 
 
 class DatabaseOperations(BaseDatabaseOperations):

django/db/models/base.py

@@ -459,14 +459,21 @@ class Model(six.with_metaclass(ModelBase)):
         return '%s object' % self.__class__.__name__
 
     def __eq__(self, other):
-        return (isinstance(other, Model) and
+        if not isinstance(other, Model):
-                self._meta.concrete_model == other._meta.concrete_model and
+            return False
-                self._get_pk_val() == other._get_pk_val())
+        if self._meta.concrete_model != other._meta.concrete_model:
+            return False
+        my_pk = self._get_pk_val()
+        if my_pk is None:
+            return self is other
+        return my_pk == other._get_pk_val()
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     def __hash__(self):
+        if self._get_pk_val() is None:
+            raise TypeError("Model instances without primary key value are unhashable")
         return hash(self._get_pk_val())
 
     def __reduce__(self):

django/db/models/sql/compiler.py

@@ -664,9 +664,8 @@ class SQLCompiler(object):
                 # Use True here because we are looking at the _reverse_ side of
                 # the relation, which is always nullable.
                 new_nullable = True
-                table = model._meta.db_table
+                self.fill_related_selections(model._meta, alias, cur_depth + 1,
-                self.fill_related_selections(model._meta, table, cur_depth + 1,
+                                             next, restricted, new_nullable)
-                    next, restricted, new_nullable)
 
     def deferred_to_columns(self):
         """

django/forms/forms.py

@@ -526,9 +526,9 @@ class BoundField(object):
         """
         contents = contents or self.label
         # Only add the suffix if the label does not end in punctuation.
+        label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix
         # Translators: If found as last label character, these punctuation
         # characters will prevent the default label_suffix to be appended to the label
-        label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix
         if label_suffix and contents and contents[-1] not in _(':?.!'):
             contents = format_html('{0}{1}', contents, label_suffix)
         widget = self.field.widget

django/forms/models.py

@@ -631,7 +631,11 @@ class BaseModelFormSet(BaseFormSet):
             seen_data = set()
             for form in valid_forms:
                 # get data for each field of each of unique_check
-                row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data])
+                row_data = (form.cleaned_data[field]
+                            for field in unique_check if field in form.cleaned_data)
+                # Reduce Model instances to their primary key values
+                row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d
+                                 for d in row_data)
                 if row_data and not None in row_data:
                     # if we've already seen it then we have a uniqueness failure
                     if row_data in seen_data:

docs/ref/class-based-views/base.txt

@@ -79,10 +79,6 @@ View
         you can override the ``head()`` method. See
         :ref:`supporting-other-http-methods` for an example.
 
-        The default implementation also sets ``request``, ``args`` and
-        ``kwargs`` as instance variables, so any method on the view can know
-        the full details of the request that was made to invoke the view.
-
     .. method:: http_method_not_allowed(request, *args, **kwargs)
 
         If the view was called with a HTTP method it doesn't support, this

docs/ref/models/instances.txt

@@ -104,14 +104,9 @@ aren't present on your form from being validated since any errors raised could
 not be corrected by the user.
 
 Note that ``full_clean()`` will *not* be called automatically when you call
-your model's :meth:`~Model.save()` method, nor as a result of
+your model's :meth:`~Model.save()` method. You'll need to call it manually
-:class:`~django.forms.ModelForm` validation. In the case of
+when you want to run one-step model validation for your own manually created
-:class:`~django.forms.ModelForm` validation, :meth:`Model.clean_fields()`,
+models. For example::
-:meth:`Model.clean()`, and :meth:`Model.validate_unique()` are all called
-individually.
-
-You'll need to call ``full_clean`` manually when you want to run one-step model
-validation for your own manually created models. For example::
 
     from django.core.exceptions import ValidationError
     try:
@@ -526,6 +521,25 @@ For example::
   In previous versions only instances of the exact same class and same
   primary key value were considered equal.
 
+``__hash__``
+------------
+
+.. method:: Model.__hash__()
+
+The ``__hash__`` method is based on the instance's primary key value. It
+is effectively hash(obj.pk). If the instance doesn't have a primary key
+value then a ``TypeError`` will be raised (otherwise the ``__hash__``
+method would return different values before and after the instance is
+saved, but changing the ``__hash__`` value of an instance `is forbidden
+in Python`_).
+
+.. versionchanged:: 1.7
+
+  In previous versions instance's without primary key value were
+  hashable.
+
+.. _is forbidden in Python: http://docs.python.org/reference/datamodel.html#object.__hash__
+
 ``get_absolute_url``
 --------------------
 

docs/ref/settings.txt

@@ -1290,11 +1290,22 @@ LANGUAGE_CODE
 
 Default: ``'en-us'``
 
-A string representing the language code for this installation. This should be
+A string representing the language code for this installation. This should be in
-in standard :term:`language format<language code>`. For example, U.S. English
+standard :term:`language ID format <language code>`. For example, U.S. English
 is ``"en-us"``. See also the `list of language identifiers`_ and
 :doc:`/topics/i18n/index`.
 
+:setting:`USE_I18N` must be active for this setting to have any effect.
+
+It serves two purposes:
+
+* If the locale middleware isn't in use, it decides which translation is served
+  to all users.
+* If the locale middleware is active, it provides the fallback translation when
+  no translation exist for a given literal to the user's preferred language.
+
+See :ref:`how-django-discovers-language-preference` for more details.
+
 .. _list of language identifiers: http://www.i18nguy.com/unicode/language-identifiers.html
 
 .. setting:: LANGUAGE_COOKIE_NAME
@@ -2392,7 +2403,7 @@ SESSION_ENGINE
 
 Default: ``django.contrib.sessions.backends.db``
 
-Controls where Django stores session data. Valid values are:
+Controls where Django stores session data. Included engines are:
 
 * ``'django.contrib.sessions.backends.db'``
 * ``'django.contrib.sessions.backends.file'``
@@ -2435,6 +2446,28 @@ Whether to save the session data on every request. If this is ``False``
 (default), then the session data will only be saved if it has been modified --
 that is, if any of its dictionary values have been assigned or deleted.
 
+.. setting:: SESSION_SERIALIZER
+
+SESSION_SERIALIZER
+------------------
+
+Default: ``'django.contrib.sessions.serializers.JSONSerializer'``
+
+.. versionchanged:: 1.6
+
+    The default switched from
+    :class:`~django.contrib.sessions.serializers.PickleSerializer` to
+    :class:`~django.contrib.sessions.serializers.JSONSerializer` in Django 1.6.
+
+Full import path of a serializer class to use for serializing session data.
+Included serializers are:
+
+* ``'django.contrib.sessions.serializers.PickleSerializer'``
+* ``'django.contrib.sessions.serializers.JSONSerializer'``
+
+See :ref:`session_serialization` for details, including a warning regarding
+possible remote code execution when using
+:class:`~django.contrib.sessions.serializers.PickleSerializer`.
 
 Sites
 =====

docs/releases/1.6.txt

@@ -727,6 +727,29 @@ the ``name`` argument so it doesn't conflict with the new url::
 You can remove this url pattern after your app has been deployed with Django
 1.6 for :setting:`PASSWORD_RESET_TIMEOUT_DAYS`.
 
+Default session serialization switched to JSON
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Historically, :mod:`django.contrib.sessions` used :mod:`pickle` to serialize
+session data before storing it in the backend. If you're using the :ref:`signed
+cookie session backend<cookie-session-backend>` and :setting:`SECRET_KEY` is
+known by an attacker, the attacker could insert a string into his session
+which, when unpickled, executes arbitrary code on the server. The technique for
+doing so is simple and easily available on the internet. Although the cookie
+session storage signs the cookie-stored data to prevent tampering, a
+:setting:`SECRET_KEY` leak immediately escalates to a remote code execution
+vulnerability.
+
+This attack can be mitigated by serializing session data using JSON rather
+than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting,
+:setting:`SESSION_SERIALIZER`, to customize the session serialization format.
+For backwards compatibility, this setting defaulted to using :mod:`pickle`
+in Django 1.5.3, but we've changed the default to JSON in 1.6. If you upgrade
+and switch from pickle to JSON, sessions created before the upgrade will be
+lost. While JSON serialization does not support all Python objects like
+:mod:`pickle` does, we highly recommend using JSON-serialized sessions. See the
+:ref:`session_serialization` documentation for more details.
+
 Miscellaneous
 ~~~~~~~~~~~~~
 

docs/releases/1.7.txt

@@ -266,6 +266,14 @@ Miscellaneous
   equal when primary keys match. Previously only instances of exact same
   class were considered equal on primary key match.
 
+* The :meth:`django.db.models.Model.__eq__` method has changed such that
+  two ``Model`` instances without primary key values won't be considered
+  equal (unless they are the same instance).
+  
+* The :meth:`django.db.models.Model.__hash__` will now raise ``TypeError``
+  when called on an instance without a primary key value. This is done to
+  avoid mutable ``__hash__`` values in containers.
+
 Features deprecated in 1.7
 ==========================
 

docs/topics/http/sessions.txt

@@ -128,8 +128,9 @@ and the :setting:`SECRET_KEY` setting.
 
 .. warning::
 
-    **If the SECRET_KEY is not kept secret, this can lead to arbitrary remote
+    **If the SECRET_KEY is not kept secret and you are using the**
-    code execution.**
+    :class:`~django.contrib.sessions.serializers.PickleSerializer`, **this can
+    lead to arbitrary remote code execution.**
 
     An attacker in possession of the :setting:`SECRET_KEY` can not only
     generate falsified session data, which your site will trust, but also
@@ -256,7 +257,9 @@ You can edit it multiple times.
         in 5 minutes.
 
       * If ``value`` is a ``datetime`` or ``timedelta`` object, the
-        session will expire at that specific date/time.
+        session will expire at that specific date/time. Note that ``datetime``
+        and ``timedelta`` values are only serializable if you are using the
+        :class:`~django.contrib.sessions.serializers.PickleSerializer`.
 
       * If ``value`` is ``0``, the user's session cookie will expire
         when the user's Web browser is closed.
@@ -301,6 +304,72 @@ You can edit it multiple times.
       Removes expired sessions from the session store. This class method is
       called by :djadmin:`clearsessions`.
 
+.. _session_serialization:
+
+Session serialization
+---------------------
+
+.. versionchanged:: 1.6
+
+Before version 1.6, Django defaulted to using :mod:`pickle` to serialize
+session data before storing it in the backend. If you're using the :ref:`signed
+cookie session backend<cookie-session-backend>` and :setting:`SECRET_KEY` is
+known by an attacker, the attacker could insert a string into his session
+which, when unpickled, executes arbitrary code on the server. The technique for
+doing so is simple and easily available on the internet. Although the cookie
+session storage signs the cookie-stored data to prevent tampering, a
+:setting:`SECRET_KEY` leak immediately escalates to a remote code execution
+vulnerability.
+
+This attack can be mitigated by serializing session data using JSON rather
+than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting,
+:setting:`SESSION_SERIALIZER`, to customize the session serialization format.
+For backwards compatibility, this setting defaults to
+using :class:`django.contrib.sessions.serializers.PickleSerializer` in
+Django 1.5.x, but, for security hardening, defaults to
+:class:`django.contrib.sessions.serializers.JSONSerializer` in Django 1.6.
+Even with the caveats described in :ref:`custom-serializers`, we highly
+recommend sticking with JSON serialization *especially if you are using the
+cookie backend*.
+
+Bundled Serializers
+^^^^^^^^^^^^^^^^^^^
+
+.. class:: serializers.JSONSerializer
+
+    A wrapper around the JSON serializer from :mod:`django.core.signing`. Can
+    only serialize basic data types. See the :ref:`custom-serializers` section
+    for more details.
+
+.. class:: serializers.PickleSerializer
+
+    Supports arbitrary Python objects, but, as described above, can lead to a
+    remote code execution vulnerability if :setting:`SECRET_KEY` becomes known
+    by an attacker.
+
+.. _custom-serializers:
+
+Write Your Own Serializer
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Note that unlike :class:`~django.contrib.sessions.serializers.PickleSerializer`,
+the :class:`~django.contrib.sessions.serializers.JSONSerializer` cannot handle
+arbitrary Python data types. As is often the case, there is a trade-off between
+convenience and security. If you wish to store more advanced data types
+including ``datetime`` and ``Decimal`` in JSON backed sessions, you will need
+to write a custom serializer (or convert such values to a JSON serializable
+object before storing them in ``request.session``). While serializing these
+values is fairly straightforward
+(``django.core.serializers.json.DateTimeAwareJSONEncoder`` may be helpful),
+writing a decoder that can reliably get back the same thing that you put in is
+more fragile. For example, you run the risk of returning a ``datetime`` that
+was actually a string that just happened to be in the same format chosen for
+``datetime``\s).
+
+Your serializer class must implement two methods,
+``dumps(self, obj)`` and ``loads(self, data)``, to serialize and deserialize
+the dictionary of session data, respectively.
+
 Session object guidelines
 -------------------------
 
@@ -390,14 +459,15 @@ An API is available to manipulate session data outside of a view::
     >>> from django.contrib.sessions.backends.db import SessionStore
     >>> import datetime
     >>> s = SessionStore()
-    >>> s['last_login'] = datetime.datetime(2005, 8, 20, 13, 35, 10)
+    >>> # stored as seconds since epoch since datetimes are not serializable in JSON.
+    >>> s['last_login'] = 1376587691
     >>> s.save()
     >>> s.session_key
     '2b1189a188b44ad18c35e113ac6ceead'
 
     >>> s = SessionStore(session_key='2b1189a188b44ad18c35e113ac6ceead')
     >>> s['last_login']
-    datetime.datetime(2005, 8, 20, 13, 35, 0)
+    1376587691
 
 In order to prevent session fixation attacks, sessions keys that don't exist
 are regenerated::
@@ -543,8 +613,11 @@ behavior:
 Technical details
 =================
 
-* The session dictionary should accept any pickleable Python object. See
+* The session dictionary accepts any :mod:`json` serializable value when using
-  the :mod:`pickle` module for more information.
+  :class:`~django.contrib.sessions.serializers.JSONSerializer` or any
+  pickleable Python object when using
+  :class:`~django.contrib.sessions.serializers.PickleSerializer`. See the
+  :mod:`pickle` module for more information.
 
 * Session data is stored in a database table named ``django_session`` .
 

docs/topics/i18n/translation.txt

@@ -1550,14 +1550,17 @@ should be used -- installation-wide, for a particular user, or both.
 
 To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`.
 Django uses this language as the default translation -- the final attempt if no
-other translator finds a translation.
+better matching translation is found through one of the methods employed by the
+locale middleware (see below).
 
-If all you want to do is run Django with your native language, and a language
+If all you want is to run Django with your native language all you need to do
-file is available for it, all you need to do is set :setting:`LANGUAGE_CODE`.
+is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message
+files <message file>` and their compiled versions (``.mo``) exist.
 
 If you want to let each individual user specify which language he or she
-prefers, use ``LocaleMiddleware``. ``LocaleMiddleware`` enables language
+prefers, then you also need to use use the ``LocaleMiddleware``.
-selection based on data from the request. It customizes content for each user.
+``LocaleMiddleware`` enables language selection based on data from the request.
+It customizes content for each user.
 
 To use ``LocaleMiddleware``, add ``'django.middleware.locale.LocaleMiddleware'``
 to your :setting:`MIDDLEWARE_CLASSES` setting. Because middleware order

docs/topics/testing/overview.txt

@@ -328,7 +328,8 @@ Some of the things you can do with the test client are:
   everything from low-level HTTP (result headers and status codes) to
   page content.
 
-* Test that the correct view is executed for a given URL.
+* See the chain of redirects (if any) and check the URL and status code at
+  each step.
 
 * Test that a given request is rendered by a given Django template, with
   a template context that contains certain values.
@@ -337,8 +338,8 @@ Note that the test client is not intended to be a replacement for Selenium_ or
 other "in-browser" frameworks. Django's test client has a different focus. In
 short:
 
-* Use Django's test client to establish that the correct view is being
+* Use Django's test client to establish that the correct template is being
-  called and that the view is collecting the correct context data.
+  rendered and that the template is passed the correct context data.
 
 * Use in-browser frameworks like Selenium_ to test *rendered* HTML and the
   *behavior* of Web pages, namely JavaScript functionality. Django also

tests/admin_views/tests.py

@@ -630,6 +630,19 @@ class AdminViewBasicTest(AdminViewBasicTestCase):
         with self.assertRaises(AttributeError):
             self.client.get('/test_admin/%s/admin_views/simple/' % self.urlbit)
 
+    def test_changelist_with_no_change_url(self):
+        """
+        ModelAdmin.changelist_view shouldn't result in a NoReverseMatch if url
+        for change_view is removed from get_urls
+
+        Regression test for #20934
+        """
+        UnchangeableObject.objects.create()
+        response = self.client.get('/test_admin/admin/admin_views/unchangeableobject/')
+        self.assertEqual(response.status_code, 200)
+        # Check the format of the shown object -- shouldn't contain a change link
+        self.assertContains(response, '<th class="field-__str__">UnchangeableObject object</th>', html=True)
+
 
 @override_settings(PASSWORD_HASHERS=('django.contrib.auth.hashers.SHA1PasswordHasher',))
 class AdminViewFormUrlTest(TestCase):

tests/admin_widgets/tests.py

@@ -132,6 +132,23 @@ class AdminFormfieldForDBFieldTests(TestCase):
         self.assertEqual(f2.widget.attrs['maxlength'], '20')
         self.assertEqual(f2.widget.attrs['size'], '10')
 
+    def testFormfieldOverridesWidgetInstancesForFieldsWithChoices(self):
+        """
+        Test that widget is actually overridden for fields with choices.
+        (#194303)
+        """
+        class MemberAdmin(admin.ModelAdmin):
+            formfield_overrides = {
+                CharField: {'widget': forms.TextInput}
+            }
+        ma = MemberAdmin(models.Member, admin.site)
+        name_field = models.Member._meta.get_field('name')
+        gender_field = models.Member._meta.get_field('gender')
+        name = ma.formfield_for_dbfield(name_field, request=None)
+        gender = ma.formfield_for_dbfield(gender_field, request=None)
+        self.assertIsInstance(name.widget, forms.TextInput)
+        self.assertIsInstance(gender.widget, forms.TextInput)
+
     def testFieldWithChoices(self):
         self.assertFormfield(models.Member, 'gender', forms.Select)
 

tests/basic/tests.py

@@ -708,9 +708,20 @@ class ModelTest(TestCase):
             SelfRef.objects.get(selfref=sr)
 
     def test_eq(self):
+        self.assertEqual(Article(id=1), Article(id=1))
         self.assertNotEqual(Article(id=1), object())
         self.assertNotEqual(object(), Article(id=1))
+        a = Article()
+        self.assertEqual(a, a)
+        self.assertNotEqual(Article(), a)
 
+    def test_hash(self):
+        # Value based on PK
+        self.assertEqual(hash(Article(id=1)), hash(1))
+        with self.assertRaises(TypeError):
+            # No PK value -> unhashable (because save() would then change
+            # hash)
+            hash(Article())
 
 class ConcurrentSaveTests(TransactionTestCase):
 

tests/defer_regress/tests.py

@@ -7,6 +7,7 @@ from django.contrib.sessions.backends.db import SessionStore
 from django.db.models import Count
 from django.db.models.loading import cache
 from django.test import TestCase
+from django.test.utils import override_settings
 
 from .models import (
     ResolveThis, Item, RelatedItem, Child, Leaf, Proxy, SimpleItem, Feature,
@@ -83,24 +84,6 @@ class DeferRegressionTest(TestCase):
         self.assertEqual(results[0].child.name, "c1")
         self.assertEqual(results[0].second_child.name, "c2")
 
-        # Test for #12163 - Pickling error saving session with unsaved model
-        # instances.
-        SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead'
-
-        item = Item()
-        item._deferred = False
-        s = SessionStore(SESSION_KEY)
-        s.clear()
-        s["item"] = item
-        s.save()
-
-        s = SessionStore(SESSION_KEY)
-        s.modified = True
-        s.save()
-
-        i2 = s["item"]
-        self.assertFalse(i2._deferred)
-
         # Regression for #16409 - make sure defer() and only() work with annotate()
         self.assertIsInstance(
             list(SimpleItem.objects.annotate(Count('feature')).defer('name')),
@@ -147,6 +130,27 @@ class DeferRegressionTest(TestCase):
                 cache.get_app("defer_regress"), include_deferred=True))
         )
 
+    @override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer')
+    def test_ticket_12163(self):
+        # Test for #12163 - Pickling error saving session with unsaved model
+        # instances.
+        SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead'
+
+        item = Item()
+        item._deferred = False
+        s = SessionStore(SESSION_KEY)
+        s.clear()
+        s["item"] = item
+        s.save()
+
+        s = SessionStore(SESSION_KEY)
+        s.modified = True
+        s.save()
+
+        i2 = s["item"]
+        self.assertFalse(i2._deferred)
+
+    def test_ticket_16409(self):
         # Regression for #16409 - make sure defer() and only() work with annotate()
         self.assertIsInstance(
             list(SimpleItem.objects.annotate(Count('feature')).defer('name')),

tests/queries/models.py

@@ -501,3 +501,29 @@ class OrderItem(models.Model):
 
     def __str__(self):
         return '%s' % self.pk
+
+class BaseUser(models.Model):
+    pass
+
+@python_2_unicode_compatible
+class Task(models.Model):
+    title = models.CharField(max_length=10)
+    owner = models.ForeignKey(BaseUser, related_name='owner')
+    creator = models.ForeignKey(BaseUser, related_name='creator')
+
+    def __str__(self):
+        return self.title
+
+@python_2_unicode_compatible
+class Staff(models.Model):
+    name = models.CharField(max_length=10)
+
+    def __str__(self):
+        return self.name
+
+@python_2_unicode_compatible
+class StaffUser(BaseUser):
+    staff = models.OneToOneField(Staff, related_name='user')
+
+    def __str__(self):
+        return self.staff

tests/queries/tests.py

@@ -25,7 +25,7 @@ from .models import (
     OneToOneCategory, NullableName, ProxyCategory, SingleObject, RelatedObject,
     ModelA, ModelB, ModelC, ModelD, Responsibility, Job, JobResponsibilities,
     BaseA, FK1, Identifier, Program, Channel, Page, Paragraph, Chapter, Book,
-    MyObject, Order, OrderItem, SharedConnection)
+    MyObject, Order, OrderItem, SharedConnection, Task, Staff, StaffUser)
 
 class BaseQuerysetTest(TestCase):
     def assertValueQuerysetEqual(self, qs, values):
@@ -2992,3 +2992,23 @@ class Ticket14056Tests(TestCase):
             SharedConnection.objects.order_by('-pointera__connection', 'pk'),
             expected_ordering, lambda x: x
         )
+
+class Ticket20955Tests(TestCase):
+    def test_ticket_20955(self):
+        jack = Staff.objects.create(name='jackstaff')
+        jackstaff = StaffUser.objects.create(staff=jack)
+        jill = Staff.objects.create(name='jillstaff')
+        jillstaff = StaffUser.objects.create(staff=jill)
+        task = Task.objects.create(creator=jackstaff, owner=jillstaff, title="task")
+        task_get = Task.objects.get(pk=task.pk)
+        # Load data so that assertNumQueries doesn't complain about the get
+        # version's queries.
+        task_get.creator.staffuser.staff
+        task_get.owner.staffuser.staff
+        task_select_related = Task.objects.select_related(
+            'creator__staffuser__staff', 'owner__staffuser__staff').get(pk=task.pk)
+        with self.assertNumQueries(0):
+            self.assertEqual(task_select_related.creator.staffuser.staff,
+                             task_get.creator.staffuser.staff)
+            self.assertEqual(task_select_related.owner.staffuser.staff,
+                             task_get.owner.staffuser.staff)