diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 4e8430aceb7..6dd25e18f9b 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -475,6 +475,7 @@ SESSION_SAVE_EVERY_REQUEST = False # Whether to save the se SESSION_EXPIRE_AT_BROWSER_CLOSE = False # Whether a user's session cookie expires when the Web browser is closed. SESSION_ENGINE = 'django.contrib.sessions.backends.db' # The module to store session data SESSION_FILE_PATH = None # Directory to store session files if using the file session module. If None, the backend will use a sensible default. +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer' # class to serialize session data ######### # CACHE # diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index b475868598d..805868306d3 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -107,6 +107,7 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): validator.validate(cls, model) def __init__(self): + self._orig_formfield_overrides = self.formfield_overrides overrides = FORMFIELD_FOR_DBFIELD_DEFAULTS.copy() overrides.update(self.formfield_overrides) self.formfield_overrides = overrides @@ -123,6 +124,9 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): # If the field specifies choices, we don't need to look for special # admin widgets - we just need to use a select widget of some kind. if db_field.choices: + # see #19303 for an explanation of self._orig_formfield_overrides + if db_field.__class__ in self._orig_formfield_overrides: + kwargs = dict(self._orig_formfield_overrides[db_field.__class__], **kwargs) return self.formfield_for_choice_field(db_field, request, **kwargs) # ForeignKey or ManyToManyFields diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py index 6c3c3e85113..ba109d14540 100644 --- a/django/contrib/admin/templatetags/admin_list.py +++ b/django/contrib/admin/templatetags/admin_list.py @@ -9,6 +9,7 @@ from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE, ORDER_VAR, PAGE_VAR, SEARCH_VAR) from django.contrib.admin.templatetags.admin_static import static from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import NoReverseMatch from django.db import models from django.utils import formats from django.utils.html import escapejs, format_html @@ -216,25 +217,36 @@ def items_for_result(cl, result, form): row_class = mark_safe(' class="%s"' % ' '.join(row_classes)) # If list_display_links not defined, add the link tag to the first field if (first and not cl.list_display_links) or field_name in cl.list_display_links: - table_tag = {True:'th', False:'td'}[first] + table_tag = 'th' if first else 'td' first = False - url = cl.url_for_result(result) - url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url) - # Convert the pk to something that can be used in Javascript. - # Problem cases are long ints (23L) and non-ASCII strings. - if cl.to_field: - attr = str(cl.to_field) + + # Display link to the result's change_view if the url exists, else + # display just the result's representation. + try: + url = cl.url_for_result(result) + except NoReverseMatch: + link_or_text = result_repr else: - attr = pk - value = result.serializable_value(attr) - result_id = escapejs(value) - yield format_html('<{0}{1}>{4}', + url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url) + # Convert the pk to something that can be used in Javascript. + # Problem cases are long ints (23L) and non-ASCII strings. + if cl.to_field: + attr = str(cl.to_field) + else: + attr = pk + value = result.serializable_value(attr) + result_id = escapejs(value) + link_or_text = format_html( + '{2}', + url, + format_html(' onclick="opener.dismissRelatedLookupPopup(window, '{0}'); return false;"', result_id) + if cl.is_popup else '', + result_repr) + + yield format_html('<{0}{1}>{2}', table_tag, row_class, - url, - format_html(' onclick="opener.dismissRelatedLookupPopup(window, '{0}'); return false;"', result_id) - if cl.is_popup else '', - result_repr, + link_or_text, table_tag) else: # By default the fields come from ModelAdmin.list_editable, but if we pull diff --git a/django/contrib/messages/storage/session.py b/django/contrib/messages/storage/session.py index 225dfda2895..c3e293c22e9 100644 --- a/django/contrib/messages/storage/session.py +++ b/django/contrib/messages/storage/session.py @@ -1,4 +1,8 @@ +import json + from django.contrib.messages.storage.base import BaseStorage +from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder +from django.utils import six class SessionStorage(BaseStorage): @@ -20,14 +24,23 @@ class SessionStorage(BaseStorage): always stores everything it is given, so return True for the all_retrieved flag. """ - return self.request.session.get(self.session_key), True + return self.deserialize_messages(self.request.session.get(self.session_key)), True def _store(self, messages, response, *args, **kwargs): """ Stores a list of messages to the request's session. """ if messages: - self.request.session[self.session_key] = messages + self.request.session[self.session_key] = self.serialize_messages(messages) else: self.request.session.pop(self.session_key, None) return [] + + def serialize_messages(self, messages): + encoder = MessageEncoder(separators=(',', ':')) + return encoder.encode(messages) + + def deserialize_messages(self, data): + if data and isinstance(data, six.string_types): + return json.loads(data, cls=MessageDecoder) + return data diff --git a/django/contrib/messages/tests/base.py b/django/contrib/messages/tests/base.py index 7011a5779a6..5241436daf9 100644 --- a/django/contrib/messages/tests/base.py +++ b/django/contrib/messages/tests/base.py @@ -61,6 +61,7 @@ class BaseTests(object): MESSAGE_TAGS = '', MESSAGE_STORAGE = '%s.%s' % (self.storage_class.__module__, self.storage_class.__name__), + SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer', ) self.settings_override.enable() diff --git a/django/contrib/messages/tests/test_session.py b/django/contrib/messages/tests/test_session.py index 2ce564b773a..940e1c02d03 100644 --- a/django/contrib/messages/tests/test_session.py +++ b/django/contrib/messages/tests/test_session.py @@ -11,13 +11,13 @@ def set_session_data(storage, messages): Sets the messages into the backend request's session and remove the backend's loaded data cache. """ - storage.request.session[storage.session_key] = messages + storage.request.session[storage.session_key] = storage.serialize_messages(messages) if hasattr(storage, '_loaded_data'): del storage._loaded_data def stored_session_messages_count(storage): - data = storage.request.session.get(storage.session_key, []) + data = storage.deserialize_messages(storage.request.session.get(storage.session_key, [])) return len(data) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 759d7ac7ad5..7f5e958a608 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -3,11 +3,6 @@ from __future__ import unicode_literals import base64 from datetime import datetime, timedelta import logging - -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle import string from django.conf import settings @@ -17,6 +12,7 @@ from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone from django.utils.encoding import force_bytes, force_text +from django.utils.module_loading import import_by_path from django.contrib.sessions.exceptions import SuspiciousSession @@ -42,6 +38,7 @@ class SessionBase(object): self._session_key = session_key self.accessed = False self.modified = False + self.serializer = import_by_path(settings.SESSION_SERIALIZER) def __contains__(self, key): return key in self._session @@ -86,21 +83,21 @@ class SessionBase(object): return salted_hmac(key_salt, value).hexdigest() def encode(self, session_dict): - "Returns the given session dictionary pickled and encoded as a string." - pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL) - hash = self._hash(pickled) - return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii') + "Returns the given session dictionary serialized and encoded as a string." + serialized = self.serializer().dumps(session_dict) + hash = self._hash(serialized) + return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii') def decode(self, session_data): encoded_data = base64.b64decode(force_bytes(session_data)) try: # could produce ValueError if there is no ':' - hash, pickled = encoded_data.split(b':', 1) - expected_hash = self._hash(pickled) + hash, serialized = encoded_data.split(b':', 1) + expected_hash = self._hash(serialized) if not constant_time_compare(hash.decode(), expected_hash): raise SuspiciousSession("Session data corrupted") else: - return pickle.loads(pickled) + return self.serializer().loads(serialized) except Exception as e: # ValueError, SuspiciousOperation, unpickling exceptions. If any of # these happen, just return an empty dictionary (an empty session). diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py index c2b7a3123f5..77a6750ce47 100644 --- a/django/contrib/sessions/backends/signed_cookies.py +++ b/django/contrib/sessions/backends/signed_cookies.py @@ -1,26 +1,9 @@ -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle - from django.conf import settings from django.core import signing from django.contrib.sessions.backends.base import SessionBase -class PickleSerializer(object): - """ - Simple wrapper around pickle to be used in signing.dumps and - signing.loads. - """ - def dumps(self, obj): - return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) - - def loads(self, data): - return pickle.loads(data) - - class SessionStore(SessionBase): def load(self): @@ -31,7 +14,7 @@ class SessionStore(SessionBase): """ try: return signing.loads(self.session_key, - serializer=PickleSerializer, + serializer=self.serializer, # This doesn't handle non-default expiry dates, see #19201 max_age=settings.SESSION_COOKIE_AGE, salt='django.contrib.sessions.backends.signed_cookies') @@ -91,7 +74,7 @@ class SessionStore(SessionBase): session_cache = getattr(self, '_session_cache', {}) return signing.dumps(session_cache, compress=True, salt='django.contrib.sessions.backends.signed_cookies', - serializer=PickleSerializer) + serializer=self.serializer) @classmethod def clear_expired(cls): diff --git a/django/contrib/sessions/models.py b/django/contrib/sessions/models.py index 0179c358b3a..3a6e31152f0 100644 --- a/django/contrib/sessions/models.py +++ b/django/contrib/sessions/models.py @@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _ class SessionManager(models.Manager): def encode(self, session_dict): """ - Returns the given session dictionary pickled and encoded as a string. + Returns the given session dictionary serialized and encoded as a string. """ return SessionStore().encode(session_dict) diff --git a/django/contrib/sessions/serializers.py b/django/contrib/sessions/serializers.py new file mode 100644 index 00000000000..92a31c054bb --- /dev/null +++ b/django/contrib/sessions/serializers.py @@ -0,0 +1,20 @@ +from django.core.signing import JSONSerializer as BaseJSONSerializer +try: + from django.utils.six.moves import cPickle as pickle +except ImportError: + import pickle + + +class PickleSerializer(object): + """ + Simple wrapper around pickle to be used in signing.dumps and + signing.loads. + """ + def dumps(self, obj): + return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) + + def loads(self, data): + return pickle.loads(data) + + +JSONSerializer = BaseJSONSerializer diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index f2a35c544ef..4caefe938c2 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -285,21 +285,25 @@ class SessionTestsMixin(object): def test_actual_expiry(self): - # Regression test for #19200 - old_session_key = None - new_session_key = None - try: - self.session['foo'] = 'bar' - self.session.set_expiry(-timedelta(seconds=10)) - self.session.save() - old_session_key = self.session.session_key - # With an expiry date in the past, the session expires instantly. - new_session = self.backend(self.session.session_key) - new_session_key = new_session.session_key - self.assertNotIn('foo', new_session) - finally: - self.session.delete(old_session_key) - self.session.delete(new_session_key) + # this doesn't work with JSONSerializer (serializing timedelta) + with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'): + self.session = self.backend() # reinitialize after overriding settings + + # Regression test for #19200 + old_session_key = None + new_session_key = None + try: + self.session['foo'] = 'bar' + self.session.set_expiry(-timedelta(seconds=10)) + self.session.save() + old_session_key = self.session.session_key + # With an expiry date in the past, the session expires instantly. + new_session = self.backend(self.session.session_key) + new_session_key = new_session.session_key + self.assertNotIn('foo', new_session) + finally: + self.session.delete(old_session_key) + self.session.delete(new_session_key) class DatabaseSessionTests(SessionTestsMixin, TestCase): diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index a363c09001a..fe6ac0e70c9 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -96,6 +96,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): nulls_order_largest = True requires_literal_defaults = True connection_persists_old_columns = True + nulls_order_largest = True class DatabaseOperations(BaseDatabaseOperations): diff --git a/django/db/models/base.py b/django/db/models/base.py index 6a21544baf2..d63017c4386 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -459,14 +459,21 @@ class Model(six.with_metaclass(ModelBase)): return '%s object' % self.__class__.__name__ def __eq__(self, other): - return (isinstance(other, Model) and - self._meta.concrete_model == other._meta.concrete_model and - self._get_pk_val() == other._get_pk_val()) + if not isinstance(other, Model): + return False + if self._meta.concrete_model != other._meta.concrete_model: + return False + my_pk = self._get_pk_val() + if my_pk is None: + return self is other + return my_pk == other._get_pk_val() def __ne__(self, other): return not self.__eq__(other) def __hash__(self): + if self._get_pk_val() is None: + raise TypeError("Model instances without primary key value are unhashable") return hash(self._get_pk_val()) def __reduce__(self): diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index caaeaefa6e7..6c8b850c84f 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -664,9 +664,8 @@ class SQLCompiler(object): # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. new_nullable = True - table = model._meta.db_table - self.fill_related_selections(model._meta, table, cur_depth + 1, - next, restricted, new_nullable) + self.fill_related_selections(model._meta, alias, cur_depth + 1, + next, restricted, new_nullable) def deferred_to_columns(self): """ diff --git a/django/forms/forms.py b/django/forms/forms.py index ec51507981e..d8d08e18fef 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -526,9 +526,9 @@ class BoundField(object): """ contents = contents or self.label # Only add the suffix if the label does not end in punctuation. + label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix # Translators: If found as last label character, these punctuation # characters will prevent the default label_suffix to be appended to the label - label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix if label_suffix and contents and contents[-1] not in _(':?.!'): contents = format_html('{0}{1}', contents, label_suffix) widget = self.field.widget diff --git a/django/forms/models.py b/django/forms/models.py index a5b82e521d7..4c6ee9c6ed6 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -631,7 +631,11 @@ class BaseModelFormSet(BaseFormSet): seen_data = set() for form in valid_forms: # get data for each field of each of unique_check - row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data]) + row_data = (form.cleaned_data[field] + for field in unique_check if field in form.cleaned_data) + # Reduce Model instances to their primary key values + row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d + for d in row_data) if row_data and not None in row_data: # if we've already seen it then we have a uniqueness failure if row_data in seen_data: diff --git a/docs/ref/class-based-views/base.txt b/docs/ref/class-based-views/base.txt index 0db1e15ea95..f0543e60956 100644 --- a/docs/ref/class-based-views/base.txt +++ b/docs/ref/class-based-views/base.txt @@ -79,10 +79,6 @@ View you can override the ``head()`` method. See :ref:`supporting-other-http-methods` for an example. - The default implementation also sets ``request``, ``args`` and - ``kwargs`` as instance variables, so any method on the view can know - the full details of the request that was made to invoke the view. - .. method:: http_method_not_allowed(request, *args, **kwargs) If the view was called with a HTTP method it doesn't support, this diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 015393a4088..da657a9a01b 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -104,14 +104,9 @@ aren't present on your form from being validated since any errors raised could not be corrected by the user. Note that ``full_clean()`` will *not* be called automatically when you call -your model's :meth:`~Model.save()` method, nor as a result of -:class:`~django.forms.ModelForm` validation. In the case of -:class:`~django.forms.ModelForm` validation, :meth:`Model.clean_fields()`, -:meth:`Model.clean()`, and :meth:`Model.validate_unique()` are all called -individually. - -You'll need to call ``full_clean`` manually when you want to run one-step model -validation for your own manually created models. For example:: +your model's :meth:`~Model.save()` method. You'll need to call it manually +when you want to run one-step model validation for your own manually created +models. For example:: from django.core.exceptions import ValidationError try: @@ -526,6 +521,25 @@ For example:: In previous versions only instances of the exact same class and same primary key value were considered equal. +``__hash__`` +------------ + +.. method:: Model.__hash__() + +The ``__hash__`` method is based on the instance's primary key value. It +is effectively hash(obj.pk). If the instance doesn't have a primary key +value then a ``TypeError`` will be raised (otherwise the ``__hash__`` +method would return different values before and after the instance is +saved, but changing the ``__hash__`` value of an instance `is forbidden +in Python`_). + +.. versionchanged:: 1.7 + + In previous versions instance's without primary key value were + hashable. + +.. _is forbidden in Python: http://docs.python.org/reference/datamodel.html#object.__hash__ + ``get_absolute_url`` -------------------- diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 424f7d5795f..2f531803bcd 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1290,11 +1290,22 @@ LANGUAGE_CODE Default: ``'en-us'`` -A string representing the language code for this installation. This should be -in standard :term:`language format`. For example, U.S. English +A string representing the language code for this installation. This should be in +standard :term:`language ID format `. For example, U.S. English is ``"en-us"``. See also the `list of language identifiers`_ and :doc:`/topics/i18n/index`. +:setting:`USE_I18N` must be active for this setting to have any effect. + +It serves two purposes: + +* If the locale middleware isn't in use, it decides which translation is served + to all users. +* If the locale middleware is active, it provides the fallback translation when + no translation exist for a given literal to the user's preferred language. + +See :ref:`how-django-discovers-language-preference` for more details. + .. _list of language identifiers: http://www.i18nguy.com/unicode/language-identifiers.html .. setting:: LANGUAGE_COOKIE_NAME @@ -2392,7 +2403,7 @@ SESSION_ENGINE Default: ``django.contrib.sessions.backends.db`` -Controls where Django stores session data. Valid values are: +Controls where Django stores session data. Included engines are: * ``'django.contrib.sessions.backends.db'`` * ``'django.contrib.sessions.backends.file'`` @@ -2435,6 +2446,28 @@ Whether to save the session data on every request. If this is ``False`` (default), then the session data will only be saved if it has been modified -- that is, if any of its dictionary values have been assigned or deleted. +.. setting:: SESSION_SERIALIZER + +SESSION_SERIALIZER +------------------ + +Default: ``'django.contrib.sessions.serializers.JSONSerializer'`` + +.. versionchanged:: 1.6 + + The default switched from + :class:`~django.contrib.sessions.serializers.PickleSerializer` to + :class:`~django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. + +Full import path of a serializer class to use for serializing session data. +Included serializers are: + +* ``'django.contrib.sessions.serializers.PickleSerializer'`` +* ``'django.contrib.sessions.serializers.JSONSerializer'`` + +See :ref:`session_serialization` for details, including a warning regarding +possible remote code execution when using +:class:`~django.contrib.sessions.serializers.PickleSerializer`. Sites ===== diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index c0f5c511940..556edddda19 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -727,6 +727,29 @@ the ``name`` argument so it doesn't conflict with the new url:: You can remove this url pattern after your app has been deployed with Django 1.6 for :setting:`PASSWORD_RESET_TIMEOUT_DAYS`. +Default session serialization switched to JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Historically, :mod:`django.contrib.sessions` used :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaulted to using :mod:`pickle` +in Django 1.5.3, but we've changed the default to JSON in 1.6. If you upgrade +and switch from pickle to JSON, sessions created before the upgrade will be +lost. While JSON serialization does not support all Python objects like +:mod:`pickle` does, we highly recommend using JSON-serialized sessions. See the +:ref:`session_serialization` documentation for more details. + Miscellaneous ~~~~~~~~~~~~~ diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index a7b26c07fc2..4c295f58e0e 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -266,6 +266,14 @@ Miscellaneous equal when primary keys match. Previously only instances of exact same class were considered equal on primary key match. +* The :meth:`django.db.models.Model.__eq__` method has changed such that + two ``Model`` instances without primary key values won't be considered + equal (unless they are the same instance). + +* The :meth:`django.db.models.Model.__hash__` will now raise ``TypeError`` + when called on an instance without a primary key value. This is done to + avoid mutable ``__hash__`` values in containers. + Features deprecated in 1.7 ========================== diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 6ac17ccbd8d..24b9ef44622 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -128,8 +128,9 @@ and the :setting:`SECRET_KEY` setting. .. warning:: - **If the SECRET_KEY is not kept secret, this can lead to arbitrary remote - code execution.** + **If the SECRET_KEY is not kept secret and you are using the** + :class:`~django.contrib.sessions.serializers.PickleSerializer`, **this can + lead to arbitrary remote code execution.** An attacker in possession of the :setting:`SECRET_KEY` can not only generate falsified session data, which your site will trust, but also @@ -256,7 +257,9 @@ You can edit it multiple times. in 5 minutes. * If ``value`` is a ``datetime`` or ``timedelta`` object, the - session will expire at that specific date/time. + session will expire at that specific date/time. Note that ``datetime`` + and ``timedelta`` values are only serializable if you are using the + :class:`~django.contrib.sessions.serializers.PickleSerializer`. * If ``value`` is ``0``, the user's session cookie will expire when the user's Web browser is closed. @@ -301,6 +304,72 @@ You can edit it multiple times. Removes expired sessions from the session store. This class method is called by :djadmin:`clearsessions`. +.. _session_serialization: + +Session serialization +--------------------- + +.. versionchanged:: 1.6 + +Before version 1.6, Django defaulted to using :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaults to +using :class:`django.contrib.sessions.serializers.PickleSerializer` in +Django 1.5.x, but, for security hardening, defaults to +:class:`django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. +Even with the caveats described in :ref:`custom-serializers`, we highly +recommend sticking with JSON serialization *especially if you are using the +cookie backend*. + +Bundled Serializers +^^^^^^^^^^^^^^^^^^^ + +.. class:: serializers.JSONSerializer + + A wrapper around the JSON serializer from :mod:`django.core.signing`. Can + only serialize basic data types. See the :ref:`custom-serializers` section + for more details. + +.. class:: serializers.PickleSerializer + + Supports arbitrary Python objects, but, as described above, can lead to a + remote code execution vulnerability if :setting:`SECRET_KEY` becomes known + by an attacker. + +.. _custom-serializers: + +Write Your Own Serializer +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that unlike :class:`~django.contrib.sessions.serializers.PickleSerializer`, +the :class:`~django.contrib.sessions.serializers.JSONSerializer` cannot handle +arbitrary Python data types. As is often the case, there is a trade-off between +convenience and security. If you wish to store more advanced data types +including ``datetime`` and ``Decimal`` in JSON backed sessions, you will need +to write a custom serializer (or convert such values to a JSON serializable +object before storing them in ``request.session``). While serializing these +values is fairly straightforward +(``django.core.serializers.json.DateTimeAwareJSONEncoder`` may be helpful), +writing a decoder that can reliably get back the same thing that you put in is +more fragile. For example, you run the risk of returning a ``datetime`` that +was actually a string that just happened to be in the same format chosen for +``datetime``\s). + +Your serializer class must implement two methods, +``dumps(self, obj)`` and ``loads(self, data)``, to serialize and deserialize +the dictionary of session data, respectively. + Session object guidelines ------------------------- @@ -390,14 +459,15 @@ An API is available to manipulate session data outside of a view:: >>> from django.contrib.sessions.backends.db import SessionStore >>> import datetime >>> s = SessionStore() - >>> s['last_login'] = datetime.datetime(2005, 8, 20, 13, 35, 10) + >>> # stored as seconds since epoch since datetimes are not serializable in JSON. + >>> s['last_login'] = 1376587691 >>> s.save() >>> s.session_key '2b1189a188b44ad18c35e113ac6ceead' >>> s = SessionStore(session_key='2b1189a188b44ad18c35e113ac6ceead') >>> s['last_login'] - datetime.datetime(2005, 8, 20, 13, 35, 0) + 1376587691 In order to prevent session fixation attacks, sessions keys that don't exist are regenerated:: @@ -543,8 +613,11 @@ behavior: Technical details ================= -* The session dictionary should accept any pickleable Python object. See - the :mod:`pickle` module for more information. +* The session dictionary accepts any :mod:`json` serializable value when using + :class:`~django.contrib.sessions.serializers.JSONSerializer` or any + pickleable Python object when using + :class:`~django.contrib.sessions.serializers.PickleSerializer`. See the + :mod:`pickle` module for more information. * Session data is stored in a database table named ``django_session`` . diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 6436e7dcf90..120db8e5b0d 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1550,14 +1550,17 @@ should be used -- installation-wide, for a particular user, or both. To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`. Django uses this language as the default translation -- the final attempt if no -other translator finds a translation. +better matching translation is found through one of the methods employed by the +locale middleware (see below). -If all you want to do is run Django with your native language, and a language -file is available for it, all you need to do is set :setting:`LANGUAGE_CODE`. +If all you want is to run Django with your native language all you need to do +is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message +files ` and their compiled versions (``.mo``) exist. If you want to let each individual user specify which language he or she -prefers, use ``LocaleMiddleware``. ``LocaleMiddleware`` enables language -selection based on data from the request. It customizes content for each user. +prefers, then you also need to use use the ``LocaleMiddleware``. +``LocaleMiddleware`` enables language selection based on data from the request. +It customizes content for each user. To use ``LocaleMiddleware``, add ``'django.middleware.locale.LocaleMiddleware'`` to your :setting:`MIDDLEWARE_CLASSES` setting. Because middleware order diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index dc12244d726..89b38f7573a 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -328,7 +328,8 @@ Some of the things you can do with the test client are: everything from low-level HTTP (result headers and status codes) to page content. -* Test that the correct view is executed for a given URL. +* See the chain of redirects (if any) and check the URL and status code at + each step. * Test that a given request is rendered by a given Django template, with a template context that contains certain values. @@ -337,8 +338,8 @@ Note that the test client is not intended to be a replacement for Selenium_ or other "in-browser" frameworks. Django's test client has a different focus. In short: -* Use Django's test client to establish that the correct view is being - called and that the view is collecting the correct context data. +* Use Django's test client to establish that the correct template is being + rendered and that the template is passed the correct context data. * Use in-browser frameworks like Selenium_ to test *rendered* HTML and the *behavior* of Web pages, namely JavaScript functionality. Django also diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py index 8f83324a37f..66a5735b6cb 100644 --- a/tests/admin_views/tests.py +++ b/tests/admin_views/tests.py @@ -630,6 +630,19 @@ class AdminViewBasicTest(AdminViewBasicTestCase): with self.assertRaises(AttributeError): self.client.get('/test_admin/%s/admin_views/simple/' % self.urlbit) + def test_changelist_with_no_change_url(self): + """ + ModelAdmin.changelist_view shouldn't result in a NoReverseMatch if url + for change_view is removed from get_urls + + Regression test for #20934 + """ + UnchangeableObject.objects.create() + response = self.client.get('/test_admin/admin/admin_views/unchangeableobject/') + self.assertEqual(response.status_code, 200) + # Check the format of the shown object -- shouldn't contain a change link + self.assertContains(response, 'UnchangeableObject object', html=True) + @override_settings(PASSWORD_HASHERS=('django.contrib.auth.hashers.SHA1PasswordHasher',)) class AdminViewFormUrlTest(TestCase): diff --git a/tests/admin_widgets/tests.py b/tests/admin_widgets/tests.py index d2ecf46358f..7d2f70f69b6 100644 --- a/tests/admin_widgets/tests.py +++ b/tests/admin_widgets/tests.py @@ -132,6 +132,23 @@ class AdminFormfieldForDBFieldTests(TestCase): self.assertEqual(f2.widget.attrs['maxlength'], '20') self.assertEqual(f2.widget.attrs['size'], '10') + def testFormfieldOverridesWidgetInstancesForFieldsWithChoices(self): + """ + Test that widget is actually overridden for fields with choices. + (#194303) + """ + class MemberAdmin(admin.ModelAdmin): + formfield_overrides = { + CharField: {'widget': forms.TextInput} + } + ma = MemberAdmin(models.Member, admin.site) + name_field = models.Member._meta.get_field('name') + gender_field = models.Member._meta.get_field('gender') + name = ma.formfield_for_dbfield(name_field, request=None) + gender = ma.formfield_for_dbfield(gender_field, request=None) + self.assertIsInstance(name.widget, forms.TextInput) + self.assertIsInstance(gender.widget, forms.TextInput) + def testFieldWithChoices(self): self.assertFormfield(models.Member, 'gender', forms.Select) diff --git a/tests/basic/tests.py b/tests/basic/tests.py index 2b051621ef4..611944902ac 100644 --- a/tests/basic/tests.py +++ b/tests/basic/tests.py @@ -708,9 +708,20 @@ class ModelTest(TestCase): SelfRef.objects.get(selfref=sr) def test_eq(self): + self.assertEqual(Article(id=1), Article(id=1)) self.assertNotEqual(Article(id=1), object()) self.assertNotEqual(object(), Article(id=1)) + a = Article() + self.assertEqual(a, a) + self.assertNotEqual(Article(), a) + def test_hash(self): + # Value based on PK + self.assertEqual(hash(Article(id=1)), hash(1)) + with self.assertRaises(TypeError): + # No PK value -> unhashable (because save() would then change + # hash) + hash(Article()) class ConcurrentSaveTests(TransactionTestCase): diff --git a/tests/defer_regress/tests.py b/tests/defer_regress/tests.py index 619f65163cc..ffb47a81338 100644 --- a/tests/defer_regress/tests.py +++ b/tests/defer_regress/tests.py @@ -7,6 +7,7 @@ from django.contrib.sessions.backends.db import SessionStore from django.db.models import Count from django.db.models.loading import cache from django.test import TestCase +from django.test.utils import override_settings from .models import ( ResolveThis, Item, RelatedItem, Child, Leaf, Proxy, SimpleItem, Feature, @@ -83,24 +84,6 @@ class DeferRegressionTest(TestCase): self.assertEqual(results[0].child.name, "c1") self.assertEqual(results[0].second_child.name, "c2") - # Test for #12163 - Pickling error saving session with unsaved model - # instances. - SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' - - item = Item() - item._deferred = False - s = SessionStore(SESSION_KEY) - s.clear() - s["item"] = item - s.save() - - s = SessionStore(SESSION_KEY) - s.modified = True - s.save() - - i2 = s["item"] - self.assertFalse(i2._deferred) - # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), @@ -147,6 +130,27 @@ class DeferRegressionTest(TestCase): cache.get_app("defer_regress"), include_deferred=True)) ) + @override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer') + def test_ticket_12163(self): + # Test for #12163 - Pickling error saving session with unsaved model + # instances. + SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' + + item = Item() + item._deferred = False + s = SessionStore(SESSION_KEY) + s.clear() + s["item"] = item + s.save() + + s = SessionStore(SESSION_KEY) + s.modified = True + s.save() + + i2 = s["item"] + self.assertFalse(i2._deferred) + + def test_ticket_16409(self): # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), diff --git a/tests/queries/models.py b/tests/queries/models.py index 3a638b28676..a81ec9f029a 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -501,3 +501,29 @@ class OrderItem(models.Model): def __str__(self): return '%s' % self.pk + +class BaseUser(models.Model): + pass + +@python_2_unicode_compatible +class Task(models.Model): + title = models.CharField(max_length=10) + owner = models.ForeignKey(BaseUser, related_name='owner') + creator = models.ForeignKey(BaseUser, related_name='creator') + + def __str__(self): + return self.title + +@python_2_unicode_compatible +class Staff(models.Model): + name = models.CharField(max_length=10) + + def __str__(self): + return self.name + +@python_2_unicode_compatible +class StaffUser(BaseUser): + staff = models.OneToOneField(Staff, related_name='user') + + def __str__(self): + return self.staff diff --git a/tests/queries/tests.py b/tests/queries/tests.py index 91d4b17d0be..9e59a1aceea 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -25,7 +25,7 @@ from .models import ( OneToOneCategory, NullableName, ProxyCategory, SingleObject, RelatedObject, ModelA, ModelB, ModelC, ModelD, Responsibility, Job, JobResponsibilities, BaseA, FK1, Identifier, Program, Channel, Page, Paragraph, Chapter, Book, - MyObject, Order, OrderItem, SharedConnection) + MyObject, Order, OrderItem, SharedConnection, Task, Staff, StaffUser) class BaseQuerysetTest(TestCase): def assertValueQuerysetEqual(self, qs, values): @@ -2992,3 +2992,23 @@ class Ticket14056Tests(TestCase): SharedConnection.objects.order_by('-pointera__connection', 'pk'), expected_ordering, lambda x: x ) + +class Ticket20955Tests(TestCase): + def test_ticket_20955(self): + jack = Staff.objects.create(name='jackstaff') + jackstaff = StaffUser.objects.create(staff=jack) + jill = Staff.objects.create(name='jillstaff') + jillstaff = StaffUser.objects.create(staff=jill) + task = Task.objects.create(creator=jackstaff, owner=jillstaff, title="task") + task_get = Task.objects.get(pk=task.pk) + # Load data so that assertNumQueries doesn't complain about the get + # version's queries. + task_get.creator.staffuser.staff + task_get.owner.staffuser.staff + task_select_related = Task.objects.select_related( + 'creator__staffuser__staff', 'owner__staffuser__staff').get(pk=task.pk) + with self.assertNumQueries(0): + self.assertEqual(task_select_related.creator.staffuser.staff, + task_get.creator.staffuser.staff) + self.assertEqual(task_select_related.owner.staffuser.staff, + task_get.owner.staffuser.staff)