From b9590a69357917a3dbf2c7234774803bd43767ef Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Mon, 19 Aug 2013 18:42:48 -0400 Subject: [PATCH 01/34] Correctly format missing Pillow/PIL exceptions messages. refs #19934 --- django/utils/image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/utils/image.py b/django/utils/image.py index dd2fab6197..8df5850338 100644 --- a/django/utils/image.py +++ b/django/utils/image.py @@ -102,7 +102,7 @@ def _detect_image_library(): except ImportError as err: # Neither worked, so it's likely not installed. raise ImproperlyConfigured( - _("Neither Pillow nor PIL could be imported: %s" % err) + _("Neither Pillow nor PIL could be imported: %s") % err ) # ``Image.alpha_composite`` was added to Pillow in SHA: e414c6 & is not @@ -125,7 +125,7 @@ def _detect_image_library(): except ImportError as err: raise ImproperlyConfigured( _("The '_imaging' module for the PIL could not be " - "imported: %s" % err) + "imported: %s") % err ) # Try to import ImageFile as well. From ac0e41e2c4d5c3712327fb4fe30420ec7a17c8d2 Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Mon, 19 Aug 2013 19:42:53 -0400 Subject: [PATCH 02/34] Avoid importing the deprecated `django.utils.importlib` package. --- django/utils/formats.py | 2 ++ django/utils/module_loading.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/django/utils/formats.py b/django/utils/formats.py index 6b89b40ecd..11e2bef8ae 100644 --- a/django/utils/formats.py +++ b/django/utils/formats.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import # Avoid importing `importlib` from this package. + import decimal import datetime from importlib import import_module diff --git a/django/utils/module_loading.py b/django/utils/module_loading.py index 359982e6ba..9c8ea98d50 100644 --- a/django/utils/module_loading.py +++ b/django/utils/module_loading.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import # Avoid importing `importlib` from this package. + import imp from importlib import import_module import os From fdbf492946a030bdf394aae8896bbad9a446e0fd Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Mon, 19 Aug 2013 20:39:30 -0400 Subject: [PATCH 03/34] Fixed an aggregation test failure on MySQL. --- tests/aggregation/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/aggregation/tests.py b/tests/aggregation/tests.py index 7d2490a77c..ce7f4e9b9d 100644 --- a/tests/aggregation/tests.py +++ b/tests/aggregation/tests.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime from decimal import Decimal +import re from django.db import connection from django.db.models import Avg, Sum, Count, Max, Min @@ -640,5 +641,14 @@ class BaseAggregateTestCase(TestCase): self.assertEqual(len(captured_queries), 1) qstr = captured_queries[0]['sql'].lower() self.assertNotIn('for update', qstr) - self.assertNotIn('order by', qstr) + forced_ordering = connection.ops.force_no_ordering() + if forced_ordering: + # If the backend needs to force an ordering we make sure it's + # the only "ORDER BY" clause present in the query. + self.assertEqual( + re.findall(r'order by (\w+)', qstr), + [', '.join(forced_ordering).lower()] + ) + else: + self.assertNotIn('order by', qstr) self.assertEqual(qstr.count(' join '), 0) From e55ca60903adcfd525938335b1ad9dbb6fd96c3e Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Mon, 19 Aug 2013 23:14:21 -0400 Subject: [PATCH 04/34] Fixed #20943 -- Weakly reference senders when caching their associated receivers --- django/db/models/signals.py | 2 +- django/dispatch/dispatcher.py | 12 ++++++++---- tests/dispatch/tests/test_dispatcher.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/django/db/models/signals.py b/django/db/models/signals.py index 3e321893c1..07824421d8 100644 --- a/django/db/models/signals.py +++ b/django/db/models/signals.py @@ -13,6 +13,6 @@ pre_delete = Signal(providing_args=["instance", "using"], use_caching=True) post_delete = Signal(providing_args=["instance", "using"], use_caching=True) pre_syncdb = Signal(providing_args=["app", "create_models", "verbosity", "interactive", "db"]) -post_syncdb = Signal(providing_args=["class", "app", "created_models", "verbosity", "interactive", "db"], use_caching=True) +post_syncdb = Signal(providing_args=["class", "app", "created_models", "verbosity", "interactive", "db"]) m2m_changed = Signal(providing_args=["action", "instance", "reverse", "model", "pk_set", "using"], use_caching=True) diff --git a/django/dispatch/dispatcher.py b/django/dispatch/dispatcher.py index 65c5c408ff..a8cdc93b21 100644 --- a/django/dispatch/dispatcher.py +++ b/django/dispatch/dispatcher.py @@ -4,8 +4,10 @@ import threading from django.dispatch import saferef from django.utils.six.moves import xrange + WEAKREF_TYPES = (weakref.ReferenceType, saferef.BoundMethodWeakref) + def _make_id(target): if hasattr(target, '__func__'): return (id(target.__self__), id(target.__func__)) @@ -15,6 +17,7 @@ NONE_ID = _make_id(None) # A marker for caching NO_RECEIVERS = object() + class Signal(object): """ Base class for all signals @@ -42,7 +45,7 @@ class Signal(object): # distinct sender we cache the receivers that sender has in # 'sender_receivers_cache'. The cache is cleaned when .connect() or # .disconnect() is called and populated on send(). - self.sender_receivers_cache = {} + self.sender_receivers_cache = weakref.WeakKeyDictionary() if use_caching else {} def connect(self, receiver, sender=None, weak=True, dispatch_uid=None): """ @@ -116,7 +119,7 @@ class Signal(object): break else: self.receivers.append((lookup_key, receiver)) - self.sender_receivers_cache = {} + self.sender_receivers_cache.clear() def disconnect(self, receiver=None, sender=None, weak=True, dispatch_uid=None): """ @@ -151,7 +154,7 @@ class Signal(object): if r_key == lookup_key: del self.receivers[index] break - self.sender_receivers_cache = {} + self.sender_receivers_cache.clear() def has_listeners(self, sender=None): return bool(self._live_receivers(sender)) @@ -276,7 +279,8 @@ class Signal(object): for idx, (r_key, _) in enumerate(reversed(self.receivers)): if r_key == key: del self.receivers[last_idx - idx] - self.sender_receivers_cache = {} + self.sender_receivers_cache.clear() + def receiver(signal, **kwargs): """ diff --git a/tests/dispatch/tests/test_dispatcher.py b/tests/dispatch/tests/test_dispatcher.py index 5f7dca87cc..e25f60b0c7 100644 --- a/tests/dispatch/tests/test_dispatcher.py +++ b/tests/dispatch/tests/test_dispatcher.py @@ -2,6 +2,7 @@ import gc import sys import time import unittest +import weakref from django.dispatch import Signal, receiver @@ -35,6 +36,8 @@ class Callable(object): a_signal = Signal(providing_args=["val"]) b_signal = Signal(providing_args=["val"]) c_signal = Signal(providing_args=["val"]) +d_signal = Signal(providing_args=["val"], use_caching=True) + class DispatcherTests(unittest.TestCase): """Test suite for dispatcher (barely started)""" @@ -72,6 +75,24 @@ class DispatcherTests(unittest.TestCase): self.assertEqual(result, expected) self._testIsClean(a_signal) + def testCachedGarbagedCollected(self): + """ + Make sure signal caching sender receivers don't prevent garbage + collection of senders. + """ + class sender: + pass + wref = weakref.ref(sender) + d_signal.connect(receiver_1_arg) + d_signal.send(sender, val='garbage') + del sender + garbage_collect() + try: + self.assertIsNone(wref()) + finally: + # Disconnect after reference check since it flushes the tested cache. + d_signal.disconnect(receiver_1_arg) + def testMultipleRegistration(self): a = Callable() a_signal.connect(a) From b53ed351b34918e337cbf26773998dafc6f82f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 09:47:43 +0300 Subject: [PATCH 05/34] Fixed #14043 -- Made sure nullable o2o delete works as expected There was an old complaint about nullable one-to-one field cascading even when the o2o field was saved to None value before the deletion. Added an test to verify this doesn't happen. Also some PEP 8 cleanup. --- tests/one_to_one_regress/tests.py | 34 +++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/one_to_one_regress/tests.py b/tests/one_to_one_regress/tests.py index 0e20f19acb..55836d47e2 100644 --- a/tests/one_to_one_regress/tests.py +++ b/tests/one_to_one_regress/tests.py @@ -25,7 +25,7 @@ class OneToOneRegressionTests(TestCase): # The bug in #9023: if you access the one-to-one relation *before* # setting to None and deleting, the cascade happens anyway. self.p1.undergroundbar - bar.place.name='foo' + bar.place.name = 'foo' bar.place = None bar.save() self.p1.delete() @@ -40,12 +40,12 @@ class OneToOneRegressionTests(TestCase): Check that we create models via the m2m relation if the remote model has a OneToOneField. """ - f = Favorites(name = 'Fred') + f = Favorites(name='Fred') f.save() f.restaurants = [self.r1] self.assertQuerysetEqual( - f.restaurants.all(), - [''] + f.restaurants.all(), + [''] ) def test_reverse_object_cache(self): @@ -114,23 +114,23 @@ class OneToOneRegressionTests(TestCase): misbehaving. We test both (primary_key=True & False) cases here to prevent any reappearance of the problem. """ - t = Target.objects.create() + Target.objects.create() self.assertQuerysetEqual( - Target.objects.filter(pointer=None), - [''] + Target.objects.filter(pointer=None), + [''] ) self.assertQuerysetEqual( - Target.objects.exclude(pointer=None), - [] + Target.objects.exclude(pointer=None), + [] ) self.assertQuerysetEqual( - Target.objects.filter(pointer2=None), - [''] + Target.objects.filter(pointer2=None), + [''] ) self.assertQuerysetEqual( - Target.objects.exclude(pointer2=None), - [] + Target.objects.exclude(pointer2=None), + [] ) def test_reverse_object_does_not_exist_cache(self): @@ -235,3 +235,11 @@ class OneToOneRegressionTests(TestCase): b = UndergroundBar.objects.create() with self.assertNumQueries(0), self.assertRaises(ValueError): p.undergroundbar = b + + def test_nullable_o2o_delete(self): + u = UndergroundBar.objects.create(place=self.p1) + u.place_id = None + u.save() + self.p1.delete() + self.assertTrue(UndergroundBar.objects.filter(pk=u.pk).exists()) + self.assertIsNone(UndergroundBar.objects.get(pk=u.pk).place) From 905409855c6b69f613190fcc2d8bd8bf5e1580b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 10:32:18 +0300 Subject: [PATCH 06/34] Fixed #14056 -- Made sure LEFT JOIN aren't trimmed in ORDER BY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If LEFT JOINs are required for correct results, then trimming the join can lead to incorrect results. Consider case: TBL A: ID | TBL B: ID A_ID 1 1 1 2 Now A.order_by('b__a') did use a join to B, and B's a_id column. This was seen to contain the same value as A's id, and so the join was trimmed. But this wasn't correct as the join is LEFT JOIN, and for row A.id = 2 the B.a_id column is NULL. --- django/db/models/sql/compiler.py | 46 +++++++++----------------------- tests/queries/tests.py | 13 ++++++++- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 54b4e86245..caaeaefa6e 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -319,10 +319,10 @@ class SQLCompiler(object): for name in self.query.distinct_fields: parts = name.split(LOOKUP_SEP) - field, cols, alias, _, _ = self._setup_joins(parts, opts, None) - cols, alias = self._final_join_removal(cols, alias) - for col in cols: - result.append("%s.%s" % (qn(alias), qn2(col))) + _, targets, alias, joins, path, _ = self._setup_joins(parts, opts, None) + targets, alias, _ = self.query.trim_joins(targets, joins, path) + for target in targets: + result.append("%s.%s" % (qn(alias), qn2(target.column))) return result def get_ordering(self): @@ -421,7 +421,7 @@ class SQLCompiler(object): return result, params, group_by def find_ordering_name(self, name, opts, alias=None, default_order='ASC', - already_seen=None): + already_seen=None): """ Returns the table alias (the name might be ambiguous, the alias will not be) and column name for ordering by the given 'name' parameter. @@ -429,11 +429,11 @@ class SQLCompiler(object): """ name, order = get_order_dir(name, default_order) pieces = name.split(LOOKUP_SEP) - field, cols, alias, joins, opts = self._setup_joins(pieces, opts, alias) + field, targets, alias, joins, path, opts = self._setup_joins(pieces, opts, alias) # If we get to this point and the field is a relation to another model, # append the default ordering for that model. - if field.rel and len(joins) > 1 and opts.ordering: + if field.rel and path and opts.ordering: # Firstly, avoid infinite loops. if not already_seen: already_seen = set() @@ -445,10 +445,10 @@ class SQLCompiler(object): results = [] for item in opts.ordering: results.extend(self.find_ordering_name(item, opts, alias, - order, already_seen)) + order, already_seen)) return results - cols, alias = self._final_join_removal(cols, alias) - return [(alias, cols, order)] + targets, alias, _ = self.query.trim_joins(targets, joins, path) + return [(alias, [t.column for t in targets], order)] def _setup_joins(self, pieces, opts, alias): """ @@ -461,13 +461,12 @@ class SQLCompiler(object): """ if not alias: alias = self.query.get_initial_alias() - field, targets, opts, joins, _ = self.query.setup_joins( + field, targets, opts, joins, path = self.query.setup_joins( pieces, opts, alias) # We will later on need to promote those joins that were added to the # query afresh above. joins_to_promote = [j for j in joins if self.query.alias_refcount[j] < 2] alias = joins[-1] - cols = [target.column for target in targets] if not field.rel: # To avoid inadvertent trimming of a necessary alias, use the # refcount to show that we are referencing a non-relation field on @@ -478,28 +477,7 @@ class SQLCompiler(object): # Ordering or distinct must not affect the returned set, and INNER # JOINS for nullable fields could do this. self.query.promote_joins(joins_to_promote) - return field, cols, alias, joins, opts - - def _final_join_removal(self, cols, alias): - """ - A helper method for get_distinct and get_ordering. This method will - trim extra not-needed joins from the tail of the join chain. - - This is very similar to what is done in trim_joins, but we will - trim LEFT JOINS here. It would be a good idea to consolidate this - method and query.trim_joins(). - """ - if alias: - while 1: - join = self.query.alias_map[alias] - lhs_cols, rhs_cols = zip(*[(lhs_col, rhs_col) for lhs_col, rhs_col in join.join_cols]) - if set(cols) != set(rhs_cols): - break - - cols = [lhs_cols[rhs_cols.index(col)] for col in cols] - self.query.unref_alias(alias) - alias = join.lhs_alias - return cols, alias + return field, targets, alias, joins, path, opts def get_from_clause(self): """ diff --git a/tests/queries/tests.py b/tests/queries/tests.py index 4d9ffb353f..bb4e9eee8f 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -25,7 +25,7 @@ from .models import ( OneToOneCategory, NullableName, ProxyCategory, SingleObject, RelatedObject, ModelA, ModelB, ModelC, ModelD, Responsibility, Job, JobResponsibilities, BaseA, FK1, Identifier, Program, Channel, Page, Paragraph, Chapter, Book, - MyObject, Order, OrderItem) + MyObject, Order, OrderItem, SharedConnection) class BaseQuerysetTest(TestCase): def assertValueQuerysetEqual(self, qs, values): @@ -2977,3 +2977,14 @@ class RelatedLookupTypeTests(TestCase): self.assertQuerysetEqual( ObjectB.objects.filter(objecta__in=[wrong_type]), [ob], lambda x: x) + +class Ticket14056Tests(TestCase): + def test_ticket_14056(self): + s1 = SharedConnection.objects.create(data='s1') + s2 = SharedConnection.objects.create(data='s2') + s3 = SharedConnection.objects.create(data='s3') + PointerA.objects.create(connection=s2) + self.assertQuerysetEqual( + SharedConnection.objects.order_by('pointera__connection', 'pk'), + [s1, s3, s2], lambda x: x + ) From 8dc76c4b9108f49afc5db07a23f34ad259a7642b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 11:33:44 +0300 Subject: [PATCH 07/34] Fixed test failure caused by different NULL ordering between backends --- django/db/backends/__init__.py | 3 +++ django/db/backends/postgresql_psycopg2/base.py | 1 + tests/queries/models.py | 4 ++++ tests/queries/tests.py | 10 +++++++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 946c650c34..07d45c9175 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -595,6 +595,9 @@ class BaseDatabaseFeatures(object): # to remove any ordering? requires_explicit_null_ordering_when_grouping = False + # Does the backend order NULL values as largest or smallest? + nulls_order_largest = False + # Is there a 1000 item limit on query parameters? supports_1000_query_parameters = True diff --git a/django/db/backends/postgresql_psycopg2/base.py b/django/db/backends/postgresql_psycopg2/base.py index e676065578..f0a82c22d6 100644 --- a/django/db/backends/postgresql_psycopg2/base.py +++ b/django/db/backends/postgresql_psycopg2/base.py @@ -55,6 +55,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_tablespaces = True supports_transactions = True can_distinct_on_fields = True + nulls_order_largest = True class DatabaseWrapper(BaseDatabaseWrapper): diff --git a/tests/queries/models.py b/tests/queries/models.py index 71346d8be9..3a638b2867 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -262,9 +262,13 @@ class ReservedName(models.Model): return self.name # A simpler shared-foreign-key setup that can expose some problems. +@python_2_unicode_compatible class SharedConnection(models.Model): data = models.CharField(max_length=10) + def __str__(self): + return self.data + class PointerA(models.Model): connection = models.ForeignKey(SharedConnection) diff --git a/tests/queries/tests.py b/tests/queries/tests.py index bb4e9eee8f..91d4b17d0b 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -2984,7 +2984,11 @@ class Ticket14056Tests(TestCase): s2 = SharedConnection.objects.create(data='s2') s3 = SharedConnection.objects.create(data='s3') PointerA.objects.create(connection=s2) - self.assertQuerysetEqual( - SharedConnection.objects.order_by('pointera__connection', 'pk'), - [s1, s3, s2], lambda x: x + expected_ordering = ( + [s1, s3, s2] if connection.features.nulls_order_largest + else [s2, s1, s3] + ) + self.assertQuerysetEqual( + SharedConnection.objects.order_by('-pointera__connection', 'pk'), + expected_ordering, lambda x: x ) From 1ed77e7782069e938e896a0b8d0fd50156ed3680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 16:23:25 +0300 Subject: [PATCH 08/34] Fixed #20820 -- Model inheritance + m2m fixture loading regression Tests by Tim Graham, report from jeroen.pulles@redslider.net. --- django/db/models/fields/related.py | 11 ++++++++++- tests/fixtures_regress/fixtures/special-article.json | 10 ++++++++++ tests/fixtures_regress/models.py | 5 +++++ tests/fixtures_regress/tests.py | 11 +++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures_regress/fixtures/special-article.json diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index b84693eba4..ca7f5d7afa 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -989,7 +989,16 @@ class ForeignObject(RelatedField): @staticmethod def get_instance_value_for_fields(instance, fields): - return tuple([getattr(instance, field.attname) for field in fields]) + ret = [] + for field in fields: + # Gotcha: in some cases (like fixture loading) a model can have + # different values in parent_ptr_id and parent's id. So, use + # instance.pk (that is, parent_ptr_id) when asked for instance.id. + if field.primary_key: + ret.append(instance.pk) + else: + ret.append(getattr(instance, field.attname)) + return tuple(ret) def get_attname_column(self): attname, column = super(ForeignObject, self).get_attname_column() diff --git a/tests/fixtures_regress/fixtures/special-article.json b/tests/fixtures_regress/fixtures/special-article.json new file mode 100644 index 0000000000..435ceb7ca4 --- /dev/null +++ b/tests/fixtures_regress/fixtures/special-article.json @@ -0,0 +1,10 @@ +[ + { + "pk": 1, + "model": "fixtures_regress.specialarticle", + "fields": { + "title": "Article Title 1", + "channels": [] + } + } +] diff --git a/tests/fixtures_regress/models.py b/tests/fixtures_regress/models.py index 99096728a7..ab4fb8750c 100644 --- a/tests/fixtures_regress/models.py +++ b/tests/fixtures_regress/models.py @@ -70,6 +70,11 @@ class Article(models.Model): ordering = ('id',) +# Subclass of a model with a ManyToManyField for test_ticket_20820 +class SpecialArticle(Article): + pass + + # Models to regression test #11428 @python_2_unicode_compatible class Widget(models.Model): diff --git a/tests/fixtures_regress/tests.py b/tests/fixtures_regress/tests.py index f917b21642..e2985d3350 100644 --- a/tests/fixtures_regress/tests.py +++ b/tests/fixtures_regress/tests.py @@ -444,6 +444,17 @@ class TestFixtures(TestCase): self.assertTrue("No fixture 'this_fixture_doesnt_exist' in" in force_text(stdout_output.getvalue())) + def test_ticket_20820(self): + """ + Regression for ticket #20820 -- loaddata on a model that inherits + from a model with a M2M shouldn't blow up. + """ + management.call_command( + 'loaddata', + 'special-article.json', + verbosity=0, + ) + class NaturalKeyFixtureTests(TestCase): From 86f4459f9e3c035ec96578617605e93234bf2700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 17:48:02 +0300 Subject: [PATCH 09/34] Fixed invalid testing fixture --- tests/fixtures_regress/fixtures/special-article.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fixtures_regress/fixtures/special-article.json b/tests/fixtures_regress/fixtures/special-article.json index 435ceb7ca4..a36244acc1 100644 --- a/tests/fixtures_regress/fixtures/special-article.json +++ b/tests/fixtures_regress/fixtures/special-article.json @@ -1,4 +1,10 @@ [ + { + "pk": 1, + "model": "fixtures_regress.article", + "fields": {"title": "foof" + } + }, { "pk": 1, "model": "fixtures_regress.specialarticle", From 859e678b3d4df084c771e6344dd1a2b2ae716afe Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Tue, 20 Aug 2013 12:28:59 -0400 Subject: [PATCH 10/34] Oracle also treats NULLs as largests values when ordering. --- django/db/backends/oracle/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 2786d05f79..8b556b8449 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -90,6 +90,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): has_bulk_insert = True supports_tablespaces = True supports_sequence_reset = False + nulls_order_largest = True class DatabaseOperations(BaseDatabaseOperations): From 96346ed5adf90849ac5ebd10d74377ed2e0c061c Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Tue, 20 Aug 2013 19:03:33 +0200 Subject: [PATCH 11/34] Fixed #20933 -- Allowed loaddata to load fixtures from relative paths. --- django/core/management/commands/loaddata.py | 2 +- docs/howto/initial-data.txt | 4 ++-- tests/fixtures_regress/models.py | 6 ------ tests/fixtures_regress/tests.py | 17 ++++++++++++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index e7aebf6d2a..1997f2956b 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -183,7 +183,7 @@ class Command(BaseCommand): if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) - if os.path.isabs(fixture_name): + if os.path.sep in fixture_name: fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: diff --git a/docs/howto/initial-data.txt b/docs/howto/initial-data.txt index b86aaa834e..70d1ae18a6 100644 --- a/docs/howto/initial-data.txt +++ b/docs/howto/initial-data.txt @@ -90,8 +90,8 @@ fixtures. You can set the :setting:`FIXTURE_DIRS` setting to a list of additional directories where Django should look. When running :djadmin:`manage.py loaddata `, you can also -specify an absolute path to a fixture file, which overrides searching -the usual directories. +specify a path to a fixture file, which overrides searching the usual +directories. .. seealso:: diff --git a/tests/fixtures_regress/models.py b/tests/fixtures_regress/models.py index ab4fb8750c..4b33cef09b 100644 --- a/tests/fixtures_regress/models.py +++ b/tests/fixtures_regress/models.py @@ -39,12 +39,6 @@ class Stuff(models.Model): class Absolute(models.Model): name = models.CharField(max_length=40) - load_count = 0 - - def __init__(self, *args, **kwargs): - super(Absolute, self).__init__(*args, **kwargs) - Absolute.load_count += 1 - class Parent(models.Model): name = models.CharField(max_length=10) diff --git a/tests/fixtures_regress/tests.py b/tests/fixtures_regress/tests.py index e2985d3350..334aa6cadc 100644 --- a/tests/fixtures_regress/tests.py +++ b/tests/fixtures_regress/tests.py @@ -148,7 +148,22 @@ class TestFixtures(TestCase): load_absolute_path, verbosity=0, ) - self.assertEqual(Absolute.load_count, 1) + self.assertEqual(Absolute.objects.count(), 1) + + def test_relative_path(self): + directory = os.path.dirname(upath(__file__)) + relative_path = os.path.join('fixtures', 'absolute.json') + cwd = os.getcwd() + try: + os.chdir(directory) + management.call_command( + 'loaddata', + relative_path, + verbosity=0, + ) + finally: + os.chdir(cwd) + self.assertEqual(Absolute.objects.count(), 1) def test_unknown_format(self): """ From f9d1d5dc1377cb21b39452b0897e7a79a3d02844 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Tue, 20 Aug 2013 22:17:26 -0300 Subject: [PATCH 12/34] Fixed #18967 -- Don't base64-encode message/rfc822 attachments. Thanks Michael Farrell for the report and his work on the fix. --- django/core/mail/message.py | 44 ++++++++++++++++++++++++++++++++++--- docs/topics/email.txt | 14 ++++++++++-- tests/mail/tests.py | 33 ++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/django/core/mail/message.py b/django/core/mail/message.py index db9023a0bb..9796e59260 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -4,11 +4,13 @@ import mimetypes import os import random import time -from email import charset as Charset, encoders as Encoders +from email import charset as Charset, encoders as Encoders, message_from_string from email.generator import Generator +from email.message import Message from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase +from email.mime.message import MIMEMessage from email.header import Header from email.utils import formatdate, getaddresses, formataddr, parseaddr @@ -118,6 +120,27 @@ def sanitize_address(addr, encoding): return formataddr((nm, addr)) +class SafeMIMEMessage(MIMEMessage): + + def __setitem__(self, name, val): + # message/rfc822 attachments must be ASCII + name, val = forbid_multi_line_headers(name, val, 'ascii') + MIMEMessage.__setitem__(self, name, val) + + def as_string(self, unixfrom=False): + """Return the entire formatted message as a string. + Optional `unixfrom' when True, means include the Unix From_ envelope + header. + + This overrides the default as_string() implementation to not mangle + lines that begin with 'From '. See bug #13433 for details. + """ + fp = six.StringIO() + g = Generator(fp, mangle_from_=False) + g.flatten(self, unixfrom=unixfrom) + return fp.getvalue() + + class SafeMIMEText(MIMEText): def __init__(self, text, subtype, charset): @@ -137,7 +160,7 @@ class SafeMIMEText(MIMEText): lines that begin with 'From '. See bug #13433 for details. """ fp = six.StringIO() - g = Generator(fp, mangle_from_ = False) + g = Generator(fp, mangle_from_=False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -161,7 +184,7 @@ class SafeMIMEMultipart(MIMEMultipart): lines that begin with 'From '. See bug #13433 for details. """ fp = six.StringIO() - g = Generator(fp, mangle_from_ = False) + g = Generator(fp, mangle_from_=False) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -292,11 +315,26 @@ class EmailMessage(object): def _create_mime_attachment(self, content, mimetype): """ Converts the content, mimetype pair into a MIME attachment object. + + If the mimetype is message/rfc822, content may be an + email.Message or EmailMessage object, as well as a str. """ basetype, subtype = mimetype.split('/', 1) if basetype == 'text': encoding = self.encoding or settings.DEFAULT_CHARSET attachment = SafeMIMEText(content, subtype, encoding) + elif basetype == 'message' and subtype == 'rfc822': + # Bug #18967: per RFC2046 s5.2.1, message/rfc822 attachments + # must not be base64 encoded. + if isinstance(content, EmailMessage): + # convert content into an email.Message first + content = content.message() + elif not isinstance(content, Message): + # For compatibility with existing code, parse the message + # into a email.Message object if it is not one already. + content = message_from_string(content) + + attachment = SafeMIMEMessage(content, subtype) else: # Encode non-text attachments with base64. attachment = MIMEBase(basetype, subtype) diff --git a/docs/topics/email.txt b/docs/topics/email.txt index c007c2b856..ebbb0963f4 100644 --- a/docs/topics/email.txt +++ b/docs/topics/email.txt @@ -319,6 +319,18 @@ The class has the following methods: message.attach('design.png', img_data, 'image/png') + .. versionchanged:: 1.7 + + If you specify a ``mimetype`` of ``message/rfc822``, it will also accept + :class:`django.core.mail.EmailMessage` and :py:class:`email.message.Message`. + + In addition, ``message/rfc822`` attachments will no longer be + base64-encoded in violation of :rfc:`2046#section-5.2.1`, which can cause + issues with displaying the attachments in `Evolution`__ and `Thunderbird`__. + + __ https://bugzilla.gnome.org/show_bug.cgi?id=651197 + __ https://bugzilla.mozilla.org/show_bug.cgi?id=333880 + * ``attach_file()`` creates a new attachment using a file from your filesystem. Call it with the path of the file to attach and, optionally, the MIME type to use for the attachment. If the MIME type is omitted, it @@ -326,8 +338,6 @@ The class has the following methods: message.attach_file('/images/weather_map.png') -.. _DEFAULT_FROM_EMAIL: ../settings/#default-from-email - Sending alternative content types ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 0f85cc0c76..2ba428e359 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -331,6 +331,39 @@ class MailTests(TestCase): self.assertFalse(str('Content-Transfer-Encoding: quoted-printable') in s) self.assertTrue(str('Content-Transfer-Encoding: 8bit') in s) + def test_dont_base64_encode_message_rfc822(self): + # Ticket #18967 + # Shouldn't use base64 encoding for a child EmailMessage attachment. + # Create a child message first + child_msg = EmailMessage('Child Subject', 'Some body of child message', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + child_s = child_msg.message().as_string() + + # Now create a parent + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + + # Attach to parent as a string + parent_msg.attach(content=child_s, mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + + # Feature test: try attaching email.Message object directly to the mail. + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + parent_msg.attach(content=child_msg.message(), mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + + # Feature test: try attaching Django's EmailMessage object directly to the mail. + parent_msg = EmailMessage('Parent Subject', 'Some parent body', 'bounce@example.com', ['to@example.com'], headers={'From': 'from@example.com'}) + parent_msg.attach(content=child_msg, mimetype='message/rfc822') + parent_s = parent_msg.message().as_string() + + # Verify that the child message header is not base64 encoded + self.assertTrue(str('Child Subject') in parent_s) + class BaseEmailBackendTests(object): email_backend = None From deebb1a97767b69249a107848661456bf452d1f1 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Tue, 20 Aug 2013 14:59:23 -0300 Subject: [PATCH 13/34] Import test case classes from their public API module. --- django/contrib/messages/tests/test_mixins.py | 2 +- tests/deprecation/tests.py | 2 +- tests/runtests.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django/contrib/messages/tests/test_mixins.py b/django/contrib/messages/tests/test_mixins.py index 8eef4cb3dc..a24d580bd8 100644 --- a/django/contrib/messages/tests/test_mixins.py +++ b/django/contrib/messages/tests/test_mixins.py @@ -1,4 +1,4 @@ -from django.test.testcases import TestCase +from django.test import TestCase from django.contrib.messages.tests.urls import ContactFormViewWithMsg from django.core.urlresolvers import reverse diff --git a/tests/deprecation/tests.py b/tests/deprecation/tests.py index fda780c7e6..719bd635db 100644 --- a/tests/deprecation/tests.py +++ b/tests/deprecation/tests.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals import warnings -from django.test.testcases import SimpleTestCase +from django.test import SimpleTestCase from django.utils import six from django.utils.deprecation import RenameMethodsBase diff --git a/tests/runtests.py b/tests/runtests.py index adfc77b13b..5053b4f54c 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -97,7 +97,7 @@ def get_installed(): def setup(verbosity, test_labels): from django.conf import settings from django.db.models.loading import get_apps, load_app - from django.test.testcases import TransactionTestCase, TestCase + from django.test import TransactionTestCase, TestCase # Force declaring available_apps in TransactionTestCase for faster tests. def no_available_apps(self): From 83e434a2c2910d9e0540ea693d5f65e6550240c1 Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Tue, 20 Aug 2013 23:22:25 -0400 Subject: [PATCH 14/34] Documentation - Noted that OneToOneField doesn't respect unique. Added OneToOneField to the list of model fields for which the unique argument isn't valid. (OneToOneFields are inherently unique, and if the user supplies a value for unique it is ignored / overwritten.) --- docs/ref/models/fields.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index a9673ce3d2..6ef487e90f 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -281,8 +281,8 @@ you try to save a model with a duplicate value in a :attr:`~Field.unique` field, a :exc:`django.db.IntegrityError` will be raised by the model's :meth:`~django.db.models.Model.save` method. -This option is valid on all field types except :class:`ManyToManyField` and -:class:`FileField`. +This option is valid on all field types except :class:`ManyToManyField`, +:class:`OneToOneField`, and :class:`FileField`. Note that when ``unique`` is ``True``, you don't need to specify :attr:`~Field.db_index`, because ``unique`` implies the creation of an index. From b065aeb17f9daf395e22d4d5f9f49c0e2c7f4522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 20 Aug 2013 17:13:41 +0300 Subject: [PATCH 15/34] Fixed #20946 -- model inheritance + m2m failure Cleaned up the internal implementation of m2m fields by removing related.py _get_fk_val(). The _get_fk_val() was doing the wrong thing if asked for the foreign key value on foreign key to parent model's primary key when child model had different primary key field. --- django/db/models/fields/related.py | 38 ++++++++++++------------------ tests/model_inheritance/models.py | 6 +++++ tests/model_inheritance/tests.py | 16 ++++++++++++- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index ca7f5d7afa..78569042a5 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -503,8 +503,6 @@ def create_many_related_manager(superclass, rel): self.through = through self.prefetch_cache_name = prefetch_cache_name self.related_val = source_field.get_foreign_related_value(instance) - # Used for single column related auto created models - self._fk_val = self.related_val[0] if None in self.related_val: raise ValueError('"%r" needs to have a value for field "%s" before ' 'this many-to-many relationship can be used.' % @@ -517,18 +515,6 @@ def create_many_related_manager(superclass, rel): "a many-to-many relationship can be used." % instance.__class__.__name__) - def _get_fk_val(self, obj, field_name): - """ - Returns the correct value for this relationship's foreign key. This - might be something else than pk value when to_field is used. - """ - fk = self.through._meta.get_field(field_name) - if fk.rel.field_name and fk.rel.field_name != fk.rel.to._meta.pk.attname: - attname = fk.rel.get_related_field().get_attname() - return fk.get_prep_lookup('exact', getattr(obj, attname)) - else: - return obj.pk - def get_queryset(self): try: return self.instance._prefetched_objects_cache[self.prefetch_cache_name] @@ -626,11 +612,12 @@ def create_many_related_manager(superclass, rel): if not router.allow_relation(obj, self.instance): raise ValueError('Cannot add "%r": instance is on database "%s", value is on database "%s"' % (obj, self.instance._state.db, obj._state.db)) - fk_val = self._get_fk_val(obj, target_field_name) + fk_val = self.through._meta.get_field( + target_field_name).get_foreign_related_value(obj)[0] if fk_val is None: raise ValueError('Cannot add "%r": the value for field "%s" is None' % (obj, target_field_name)) - new_ids.add(self._get_fk_val(obj, target_field_name)) + new_ids.add(fk_val) elif isinstance(obj, Model): raise TypeError("'%s' instance expected, got %r" % (self.model._meta.object_name, obj)) else: @@ -638,7 +625,7 @@ def create_many_related_manager(superclass, rel): db = router.db_for_write(self.through, instance=self.instance) vals = self.through._default_manager.using(db).values_list(target_field_name, flat=True) vals = vals.filter(**{ - source_field_name: self._fk_val, + source_field_name: self.related_val[0], '%s__in' % target_field_name: new_ids, }) new_ids = new_ids - set(vals) @@ -652,7 +639,7 @@ def create_many_related_manager(superclass, rel): # Add the ones that aren't there already self.through._default_manager.using(db).bulk_create([ self.through(**{ - '%s_id' % source_field_name: self._fk_val, + '%s_id' % source_field_name: self.related_val[0], '%s_id' % target_field_name: obj_id, }) for obj_id in new_ids @@ -676,7 +663,9 @@ def create_many_related_manager(superclass, rel): old_ids = set() for obj in objs: if isinstance(obj, self.model): - old_ids.add(self._get_fk_val(obj, target_field_name)) + fk_val = self.through._meta.get_field( + target_field_name).get_foreign_related_value(obj)[0] + old_ids.add(fk_val) else: old_ids.add(obj) # Work out what DB we're operating on @@ -690,7 +679,7 @@ def create_many_related_manager(superclass, rel): model=self.model, pk_set=old_ids, using=db) # Remove the specified objects from the join table self.through._default_manager.using(db).filter(**{ - source_field_name: self._fk_val, + source_field_name: self.related_val[0], '%s__in' % target_field_name: old_ids }).delete() if self.reverse or source_field_name == self.source_field_name: @@ -994,10 +983,13 @@ class ForeignObject(RelatedField): # Gotcha: in some cases (like fixture loading) a model can have # different values in parent_ptr_id and parent's id. So, use # instance.pk (that is, parent_ptr_id) when asked for instance.id. + opts = instance._meta if field.primary_key: - ret.append(instance.pk) - else: - ret.append(getattr(instance, field.attname)) + possible_parent_link = opts.get_ancestor_link(field.model) + if not possible_parent_link or possible_parent_link.primary_key: + ret.append(instance.pk) + continue + ret.append(getattr(instance, field.attname)) return tuple(ret) def get_attname_column(self): diff --git a/tests/model_inheritance/models.py b/tests/model_inheritance/models.py index 106645d23c..020bb35bc7 100644 --- a/tests/model_inheritance/models.py +++ b/tests/model_inheritance/models.py @@ -162,3 +162,9 @@ class Mixin(object): class MixinModel(models.Model, Mixin): pass + +class Base(models.Model): + titles = models.ManyToManyField(Title) + +class SubBase(Base): + sub_id = models.IntegerField(primary_key=True) diff --git a/tests/model_inheritance/tests.py b/tests/model_inheritance/tests.py index b8ab0c8581..dab3088a41 100644 --- a/tests/model_inheritance/tests.py +++ b/tests/model_inheritance/tests.py @@ -10,7 +10,8 @@ from django.utils import six from .models import ( Chef, CommonInfo, ItalianRestaurant, ParkingLot, Place, Post, - Restaurant, Student, StudentWorker, Supplier, Worker, MixinModel) + Restaurant, Student, StudentWorker, Supplier, Worker, MixinModel, + Title, Base, SubBase) class ModelInheritanceTests(TestCase): @@ -357,3 +358,16 @@ class ModelInheritanceTests(TestCase): [Place.objects.get(pk=s.pk)], lambda x: x ) + + def test_custompk_m2m(self): + b = Base.objects.create() + b.titles.add(Title.objects.create(title="foof")) + s = SubBase.objects.create(sub_id=b.id) + b = Base.objects.get(pk=s.id) + self.assertNotEqual(b.pk, s.pk) + # Low-level test for related_val + self.assertEqual(s.titles.related_val, (s.id,)) + # Higher level test for correct query values (title foof not + # accidentally found). + self.assertQuerysetEqual( + s.titles.all(), []) From ececbe77ff573707d8f25084018e66ee07f820fd Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Mon, 19 Aug 2013 20:04:50 -0300 Subject: [PATCH 16/34] Fixed #12422 -- Don't override global email charset behavior for utf-8. Thanks simonb for the report, Claude Paroz and Susan Tan for their work on a fix. --- django/core/mail/message.py | 14 +++++- tests/mail/tests.py | 97 +++++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/django/core/mail/message.py b/django/core/mail/message.py index 9796e59260..a24817ac90 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -22,7 +22,8 @@ from django.utils import six # Don't BASE64-encode UTF-8 messages so that we avoid unwanted attention from # some spam filters. -Charset.add_charset('utf-8', Charset.SHORTEST, None, 'utf-8') +utf8_charset = Charset.Charset('utf-8') +utf8_charset.body_encoding = None # Python defaults to BASE64 # Default MIME type to use on attachments (if it is not explicitly given # and cannot be guessed). @@ -145,7 +146,16 @@ class SafeMIMEText(MIMEText): def __init__(self, text, subtype, charset): self.encoding = charset - MIMEText.__init__(self, text, subtype, charset) + if charset == 'utf-8': + # Unfortunately, Python doesn't support setting a Charset instance + # as MIMEText init parameter (http://bugs.python.org/issue16324). + # We do it manually and trigger re-encoding of the payload. + MIMEText.__init__(self, text, subtype, None) + del self['Content-Transfer-Encoding'] + self.set_payload(text, utf8_charset) + self.replace_header('Content-Type', 'text/%s; charset="%s"' % (subtype, charset)) + else: + MIMEText.__init__(self, text, subtype, charset) def __setitem__(self, name, val): name, val = forbid_multi_line_headers(name, val, self.encoding) diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 2ba428e359..71733d69ae 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import asyncore import email +from email.mime.text import MIMEText import os import shutil import smtpd @@ -20,11 +21,32 @@ from django.core.mail.message import BadHeaderError from django.test import TestCase from django.test.utils import override_settings from django.utils.encoding import force_str, force_text -from django.utils.six import PY3, StringIO +from django.utils.six import PY3, StringIO, string_types from django.utils.translation import ugettext_lazy -class MailTests(TestCase): +class HeadersCheckMixin(object): + + def assertMessageHasHeaders(self, message, headers): + """ + Check that :param message: has all :param headers: headers. + + :param message: can be an instance of an email.Message subclass or a + string with the contens of an email message. + :param headers: should be a set of (header-name, header-value) tuples. + """ + if isinstance(message, string_types): + just_headers = message.split('\n\n', 1)[0] + hlist = just_headers.split('\n') + pairs = [hl.split(':', 1) for hl in hlist] + msg_headers = {(n, v.lstrip()) for (n, v) in pairs} + else: + msg_headers = set(message.items()) + self.assertTrue(headers.issubset(msg_headers), msg='Message is missing ' + 'the following headers: %s' % (headers - msg_headers),) + + +class MailTests(HeadersCheckMixin, TestCase): """ Non-backend specific tests. """ @@ -93,7 +115,7 @@ class MailTests(TestCase): headers = {"date": "Fri, 09 Nov 2001 01:08:47 -0000", "Message-ID": "foo"} email = EmailMessage('subject', 'content', 'from@example.com', ['to@example.com'], headers=headers) - self.assertEqual(sorted(email.message().items()), [ + self.assertMessageHasHeaders(email.message(), { ('Content-Transfer-Encoding', '7bit'), ('Content-Type', 'text/plain; charset="utf-8"'), ('From', 'from@example.com'), @@ -102,7 +124,7 @@ class MailTests(TestCase): ('Subject', 'subject'), ('To', 'to@example.com'), ('date', 'Fri, 09 Nov 2001 01:08:47 -0000'), - ]) + }) def test_from_header(self): """ @@ -184,7 +206,13 @@ class MailTests(TestCase): email = EmailMessage('Subject', 'Firstname Sürname is a great guy.', 'from@example.com', ['other@example.com']) email.encoding = 'iso-8859-1' message = email.message() - self.assertTrue(message.as_string().startswith('Content-Type: text/plain; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\nSubject: Subject\nFrom: from@example.com\nTo: other@example.com')) + self.assertMessageHasHeaders(message, { + ('MIME-Version', '1.0'), + ('Content-Type', 'text/plain; charset="iso-8859-1"'), + ('Content-Transfer-Encoding', 'quoted-printable'), + ('Subject', 'Subject'), + ('From', 'from@example.com'), + ('To', 'other@example.com')}) self.assertEqual(message.get_payload(), 'Firstname S=FCrname is a great guy.') # Make sure MIME attachments also works correctly with other encodings than utf-8 @@ -193,8 +221,18 @@ class MailTests(TestCase): msg = EmailMultiAlternatives('Subject', text_content, 'from@example.com', ['to@example.com']) msg.encoding = 'iso-8859-1' msg.attach_alternative(html_content, "text/html") - self.assertEqual(msg.message().get_payload(0).as_string(), 'Content-Type: text/plain; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\n\nFirstname S=FCrname is a great guy.') - self.assertEqual(msg.message().get_payload(1).as_string(), 'Content-Type: text/html; charset="iso-8859-1"\nMIME-Version: 1.0\nContent-Transfer-Encoding: quoted-printable\n\n

Firstname S=FCrname is a great guy.

') + payload0 = msg.message().get_payload(0) + self.assertMessageHasHeaders(payload0, { + ('MIME-Version', '1.0'), + ('Content-Type', 'text/plain; charset="iso-8859-1"'), + ('Content-Transfer-Encoding', 'quoted-printable')}) + self.assertTrue(payload0.as_string().endswith('\n\nFirstname S=FCrname is a great guy.')) + payload1 = msg.message().get_payload(1) + self.assertMessageHasHeaders(payload1, { + ('MIME-Version', '1.0'), + ('Content-Type', 'text/html; charset="iso-8859-1"'), + ('Content-Transfer-Encoding', 'quoted-printable')}) + self.assertTrue(payload1.as_string().endswith('\n\n

Firstname S=FCrname is a great guy.

')) def test_attachments(self): """Regression test for #9367""" @@ -365,7 +403,31 @@ class MailTests(TestCase): self.assertTrue(str('Child Subject') in parent_s) -class BaseEmailBackendTests(object): +class PythonGlobalState(TestCase): + """ + Tests for #12422 -- Django smarts (#2472/#11212) with charset of utf-8 text + parts shouldn't pollute global email Python package charset registry when + django.mail.message is imported. + """ + + def test_utf8(self): + txt = MIMEText('UTF-8 encoded body', 'plain', 'utf-8') + self.assertTrue('Content-Transfer-Encoding: base64' in txt.as_string()) + + def test_7bit(self): + txt = MIMEText('Body with only ASCII characters.', 'plain', 'utf-8') + self.assertTrue('Content-Transfer-Encoding: base64' in txt.as_string()) + + def test_8bit_latin(self): + txt = MIMEText('Body with latin characters: àáä.', 'plain', 'utf-8') + self.assertTrue(str('Content-Transfer-Encoding: base64') in txt.as_string()) + + def test_8bit_non_latin(self): + txt = MIMEText('Body with non latin characters: А Б В Г Д Е Ж Ѕ З И І К Л М Н О П.', 'plain', 'utf-8') + self.assertTrue(str('Content-Transfer-Encoding: base64') in txt.as_string()) + + +class BaseEmailBackendTests(HeadersCheckMixin, object): email_backend = None def setUp(self): @@ -523,7 +585,15 @@ class BaseEmailBackendTests(object): email = EmailMessage('Subject', 'Content', 'from@example.com', ['to@example.com'], cc=['cc@example.com']) mail.get_connection().send_messages([email]) message = self.get_the_message() - self.assertStartsWith(message.as_string(), 'Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nSubject: Subject\nFrom: from@example.com\nTo: to@example.com\nCc: cc@example.com\nDate: ') + self.assertMessageHasHeaders(message, { + ('MIME-Version', '1.0'), + ('Content-Type', 'text/plain; charset="utf-8"'), + ('Content-Transfer-Encoding', '7bit'), + ('Subject', 'Subject'), + ('From', 'from@example.com'), + ('To', 'to@example.com'), + ('Cc', 'cc@example.com')}) + self.assertIn('\nDate: ', message.as_string()) def test_idn_send(self): """ @@ -681,7 +751,14 @@ class ConsoleBackendTests(BaseEmailBackendTests, TestCase): s = StringIO() connection = mail.get_connection('django.core.mail.backends.console.EmailBackend', stream=s) send_mail('Subject', 'Content', 'from@example.com', ['to@example.com'], connection=connection) - self.assertTrue(s.getvalue().startswith('Content-Type: text/plain; charset="utf-8"\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nSubject: Subject\nFrom: from@example.com\nTo: to@example.com\nDate: ')) + self.assertMessageHasHeaders(s.getvalue(), { + ('MIME-Version', '1.0'), + ('Content-Type', 'text/plain; charset="utf-8"'), + ('Content-Transfer-Encoding', '7bit'), + ('Subject', 'Subject'), + ('From', 'from@example.com'), + ('To', 'to@example.com')}) + self.assertIn('\nDate: ', s.getvalue()) class FakeSMTPChannel(smtpd.SMTPChannel): From a5cf5da50d93ccb75de2af6ee3b65d942076d0f8 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Wed, 21 Aug 2013 07:48:16 -0300 Subject: [PATCH 17/34] Switched mail tests to SimpleTestCase. --- tests/mail/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 71733d69ae..bb57ca37ff 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -18,7 +18,7 @@ from django.core.mail import (EmailMessage, mail_admins, mail_managers, EmailMultiAlternatives, send_mail, send_mass_mail) from django.core.mail.backends import console, dummy, locmem, filebased, smtp from django.core.mail.message import BadHeaderError -from django.test import TestCase +from django.test import SimpleTestCase from django.test.utils import override_settings from django.utils.encoding import force_str, force_text from django.utils.six import PY3, StringIO, string_types @@ -46,7 +46,7 @@ class HeadersCheckMixin(object): 'the following headers: %s' % (headers - msg_headers),) -class MailTests(HeadersCheckMixin, TestCase): +class MailTests(HeadersCheckMixin, SimpleTestCase): """ Non-backend specific tests. """ @@ -403,7 +403,7 @@ class MailTests(HeadersCheckMixin, TestCase): self.assertTrue(str('Child Subject') in parent_s) -class PythonGlobalState(TestCase): +class PythonGlobalState(SimpleTestCase): """ Tests for #12422 -- Django smarts (#2472/#11212) with charset of utf-8 text parts shouldn't pollute global email Python package charset registry when @@ -636,7 +636,7 @@ class BaseEmailBackendTests(HeadersCheckMixin, object): self.fail("close() unexpectedly raised an exception: %s" % e) -class LocmemBackendTests(BaseEmailBackendTests, TestCase): +class LocmemBackendTests(BaseEmailBackendTests, SimpleTestCase): email_backend = 'django.core.mail.backends.locmem.EmailBackend' def get_mailbox_content(self): @@ -666,7 +666,7 @@ class LocmemBackendTests(BaseEmailBackendTests, TestCase): send_mail('Subject\nMultiline', 'Content', 'from@example.com', ['to@example.com']) -class FileBackendTests(BaseEmailBackendTests, TestCase): +class FileBackendTests(BaseEmailBackendTests, SimpleTestCase): email_backend = 'django.core.mail.backends.filebased.EmailBackend' def setUp(self): @@ -723,7 +723,7 @@ class FileBackendTests(BaseEmailBackendTests, TestCase): connection.close() -class ConsoleBackendTests(BaseEmailBackendTests, TestCase): +class ConsoleBackendTests(BaseEmailBackendTests, SimpleTestCase): email_backend = 'django.core.mail.backends.console.EmailBackend' def setUp(self): @@ -826,7 +826,7 @@ class FakeSMTPServer(smtpd.SMTPServer, threading.Thread): self.join() -class SMTPBackendTests(BaseEmailBackendTests, TestCase): +class SMTPBackendTests(BaseEmailBackendTests, SimpleTestCase): email_backend = 'django.core.mail.backends.smtp.EmailBackend' @classmethod From f7290581fe2106c08d97215ab93e27cf6b27e100 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 20 Aug 2013 14:13:43 -0400 Subject: [PATCH 18/34] Fixed a regression with get_or_create and virtual fields. refs #20429 Thanks Simon Charette for the report and review. --- django/db/models/query.py | 20 ++++++++------------ tests/generic_relations/tests.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index 836d394e9b..67780a4991 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -411,7 +411,7 @@ class QuerySet(object): Returns a tuple of (object, created), where created is a boolean specifying whether an object was created. """ - lookup, params, _ = self._extract_model_params(defaults, **kwargs) + lookup, params = self._extract_model_params(defaults, **kwargs) self._for_write = True try: return self.get(**lookup), False @@ -425,7 +425,8 @@ class QuerySet(object): Returns a tuple (object, created), where created is a boolean specifying whether an object was created. """ - lookup, params, filtered_defaults = self._extract_model_params(defaults, **kwargs) + defaults = defaults or {} + lookup, params = self._extract_model_params(defaults, **kwargs) self._for_write = True try: obj = self.get(**lookup) @@ -433,12 +434,12 @@ class QuerySet(object): obj, created = self._create_object_from_params(lookup, params) if created: return obj, created - for k, v in six.iteritems(filtered_defaults): + for k, v in six.iteritems(defaults): setattr(obj, k, v) sid = transaction.savepoint(using=self.db) try: - obj.save(update_fields=filtered_defaults.keys(), using=self.db) + obj.save(using=self.db) transaction.savepoint_commit(sid, using=self.db) return obj, False except DatabaseError: @@ -469,22 +470,17 @@ class QuerySet(object): def _extract_model_params(self, defaults, **kwargs): """ Prepares `lookup` (kwargs that are valid model attributes), `params` - (for creating a model instance) and `filtered_defaults` (defaults - that are valid model attributes) based on given kwargs; for use by + (for creating a model instance) based on given kwargs; for use by get_or_create and update_or_create. """ defaults = defaults or {} - filtered_defaults = {} lookup = kwargs.copy() for f in self.model._meta.fields: - # Filter out fields that don't belongs to the model. if f.attname in lookup: lookup[f.name] = lookup.pop(f.attname) - if f.attname in defaults: - filtered_defaults[f.name] = defaults.pop(f.attname) params = dict((k, v) for k, v in kwargs.items() if LOOKUP_SEP not in k) - params.update(filtered_defaults) - return lookup, params, filtered_defaults + params.update(defaults) + return lookup, params def _earliest_or_latest(self, field_name=None, direction="-"): """ diff --git a/tests/generic_relations/tests.py b/tests/generic_relations/tests.py index 2b52ebac56..253eb76e32 100644 --- a/tests/generic_relations/tests.py +++ b/tests/generic_relations/tests.py @@ -263,6 +263,29 @@ class GenericRelationsTests(TestCase): formset = GenericFormSet(initial=initial_data) self.assertEqual(formset.forms[0].initial, initial_data[0]) + def test_get_or_create(self): + # get_or_create should work with virtual fields (content_object) + quartz = Mineral.objects.create(name="Quartz", hardness=7) + tag, created = TaggedItem.objects.get_or_create(tag="shiny", + defaults={'content_object': quartz}) + self.assertTrue(created) + self.assertEqual(tag.tag, "shiny") + self.assertEqual(tag.content_object.id, quartz.id) + + def test_update_or_create_defaults(self): + # update_or_create should work with virtual fields (content_object) + quartz = Mineral.objects.create(name="Quartz", hardness=7) + diamond = Mineral.objects.create(name="Diamond", hardness=7) + tag, created = TaggedItem.objects.update_or_create(tag="shiny", + defaults={'content_object': quartz}) + self.assertTrue(created) + self.assertEqual(tag.content_object.id, quartz.id) + + tag, created = TaggedItem.objects.update_or_create(tag="shiny", + defaults={'content_object': diamond}) + self.assertFalse(created) + self.assertEqual(tag.content_object.id, diamond.id) + class CustomWidget(forms.TextInput): pass From d3ed15b79d8b9288c4f6af07d5161f0727dd7669 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 21 Aug 2013 09:01:52 -0400 Subject: [PATCH 19/34] Fixed docstring typo, thanks minddust. --- django/template/loader_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py index 406775da9d..63ddbd4a6a 100644 --- a/django/template/loader_tags.py +++ b/django/template/loader_tags.py @@ -204,7 +204,7 @@ def do_extends(parser, token): uses the literal value "base" as the name of the parent template to extend, or ``{% extends variable %}`` uses the value of ``variable`` as either the name of the parent template to extend (if it evaluates to a string) or as - the parent tempate itelf (if it evaluates to a Template object). + the parent tempate itself (if it evaluates to a Template object). """ bits = token.split_contents() if len(bits) != 2: From 082b0638ef05391fa3004463bd7f49721cde3c1d Mon Sep 17 00:00:00 2001 From: evildmp Date: Tue, 20 Aug 2013 22:56:39 +0200 Subject: [PATCH 20/34] Added myself to the committers list. --- AUTHORS | 2 +- docs/internals/committers.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 059310d5c6..c343eeb67b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -46,6 +46,7 @@ The PRIMARY AUTHORS are (and/or have been): * Daniel Lindsley * Marc Tamlyn * Baptiste Mispelon + * Daniele Procida More information on the main contributors to Django can be found in docs/internals/committers.txt. @@ -478,7 +479,6 @@ answer newbie questions, and generally made Django that much better: polpak@yahoo.com Ross Poulton Mihai Preda - Daniele Procida Matthias Pronk Jyrki Pulliainen Thejaswi Puthraya diff --git a/docs/internals/committers.txt b/docs/internals/committers.txt index 6732f1561f..cc0a59e44a 100644 --- a/docs/internals/committers.txt +++ b/docs/internals/committers.txt @@ -537,6 +537,20 @@ Baptiste Mispelon .. _M2BPO: http://www.m2bpo.fr +`Daniele Procida`_ + Daniele works at Cardiff University `School of Medicine`_. He unexpectedly + became a Django developer on 29th April 2009. Since then he has relied + daily on Django's documentation, which has been a constant companion to + him. More recently he has been able to contribute back to the project by + helping improve the documentation itself. + + He is the author of `Arkestra`_ and `Don't be afraid to commit`_. + +.. _Daniele Procida: http://medicine.cf.ac.uk/person/mr-daniele-marco-procida/ +.. _School of Medicine: http://medicine.cf.ac.uk/ +.. _Arkestra: http://arkestra-project.org/ +.. _Don\'t be afraid to commit: https://dont-be-afraid-to-commit.readthedocs.org + Developers Emeritus =================== From 0073f1d94fad3f04ed77e63984cfd7827f78e580 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 21 Aug 2013 10:49:50 -0400 Subject: [PATCH 21/34] Fixed #20949 -- Typo #2 in docstring --- django/template/loader_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py index 63ddbd4a6a..d48f85eb35 100644 --- a/django/template/loader_tags.py +++ b/django/template/loader_tags.py @@ -204,7 +204,7 @@ def do_extends(parser, token): uses the literal value "base" as the name of the parent template to extend, or ``{% extends variable %}`` uses the value of ``variable`` as either the name of the parent template to extend (if it evaluates to a string) or as - the parent tempate itself (if it evaluates to a Template object). + the parent template itself (if it evaluates to a Template object). """ bits = token.split_contents() if len(bits) != 2: From bb011cf809359da3f717e35b5e70fec7897dd22f Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Wed, 21 Aug 2013 15:38:07 -0400 Subject: [PATCH 22/34] Documentation -- Corrected error about Model.full_clean() Although the ModelForm validation code was changed to call Model.full_clean(), the documentation still said otherwise. The offending phrase was removed. --- docs/ref/models/instances.txt | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 015393a408..6295eb0407 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -104,14 +104,9 @@ aren't present on your form from being validated since any errors raised could not be corrected by the user. Note that ``full_clean()`` will *not* be called automatically when you call -your model's :meth:`~Model.save()` method, nor as a result of -:class:`~django.forms.ModelForm` validation. In the case of -:class:`~django.forms.ModelForm` validation, :meth:`Model.clean_fields()`, -:meth:`Model.clean()`, and :meth:`Model.validate_unique()` are all called -individually. - -You'll need to call ``full_clean`` manually when you want to run one-step model -validation for your own manually created models. For example:: +your model's :meth:`~Model.save()` method. You'll need to call it manually +when you want to run one-step model validation for your own manually created +models. For example:: from django.core.exceptions import ValidationError try: From 8d65b6082c8bf5df25608d8733470879a8a61d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 22 Aug 2013 10:42:10 +0300 Subject: [PATCH 23/34] Fixed #20955 -- select_related regression In cases where the same connection (from model A to model B along the same field) was needed multiple times in a select_related query, the join setup code mistakenly reused an existing join. --- django/db/models/sql/compiler.py | 5 ++--- tests/queries/models.py | 26 ++++++++++++++++++++++++++ tests/queries/tests.py | 22 +++++++++++++++++++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index caaeaefa6e..6c8b850c84 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -664,9 +664,8 @@ class SQLCompiler(object): # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. new_nullable = True - table = model._meta.db_table - self.fill_related_selections(model._meta, table, cur_depth + 1, - next, restricted, new_nullable) + self.fill_related_selections(model._meta, alias, cur_depth + 1, + next, restricted, new_nullable) def deferred_to_columns(self): """ diff --git a/tests/queries/models.py b/tests/queries/models.py index 3a638b2867..a81ec9f029 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -501,3 +501,29 @@ class OrderItem(models.Model): def __str__(self): return '%s' % self.pk + +class BaseUser(models.Model): + pass + +@python_2_unicode_compatible +class Task(models.Model): + title = models.CharField(max_length=10) + owner = models.ForeignKey(BaseUser, related_name='owner') + creator = models.ForeignKey(BaseUser, related_name='creator') + + def __str__(self): + return self.title + +@python_2_unicode_compatible +class Staff(models.Model): + name = models.CharField(max_length=10) + + def __str__(self): + return self.name + +@python_2_unicode_compatible +class StaffUser(BaseUser): + staff = models.OneToOneField(Staff, related_name='user') + + def __str__(self): + return self.staff diff --git a/tests/queries/tests.py b/tests/queries/tests.py index 91d4b17d0b..9e59a1acee 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -25,7 +25,7 @@ from .models import ( OneToOneCategory, NullableName, ProxyCategory, SingleObject, RelatedObject, ModelA, ModelB, ModelC, ModelD, Responsibility, Job, JobResponsibilities, BaseA, FK1, Identifier, Program, Channel, Page, Paragraph, Chapter, Book, - MyObject, Order, OrderItem, SharedConnection) + MyObject, Order, OrderItem, SharedConnection, Task, Staff, StaffUser) class BaseQuerysetTest(TestCase): def assertValueQuerysetEqual(self, qs, values): @@ -2992,3 +2992,23 @@ class Ticket14056Tests(TestCase): SharedConnection.objects.order_by('-pointera__connection', 'pk'), expected_ordering, lambda x: x ) + +class Ticket20955Tests(TestCase): + def test_ticket_20955(self): + jack = Staff.objects.create(name='jackstaff') + jackstaff = StaffUser.objects.create(staff=jack) + jill = Staff.objects.create(name='jillstaff') + jillstaff = StaffUser.objects.create(staff=jill) + task = Task.objects.create(creator=jackstaff, owner=jillstaff, title="task") + task_get = Task.objects.get(pk=task.pk) + # Load data so that assertNumQueries doesn't complain about the get + # version's queries. + task_get.creator.staffuser.staff + task_get.owner.staffuser.staff + task_select_related = Task.objects.select_related( + 'creator__staffuser__staff', 'owner__staffuser__staff').get(pk=task.pk) + with self.assertNumQueries(0): + self.assertEqual(task_select_related.creator.staffuser.staff, + task_get.creator.staffuser.staff) + self.assertEqual(task_select_related.owner.staffuser.staff, + task_get.owner.staffuser.staff) From 8cd8742981020e315acc4e70bdf3613fcf68e3a8 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 22 Aug 2013 09:14:06 +0200 Subject: [PATCH 24/34] Moved translator comment just above the target string --- django/forms/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/forms/forms.py b/django/forms/forms.py index ec51507981..d8d08e18fe 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -526,9 +526,9 @@ class BoundField(object): """ contents = contents or self.label # Only add the suffix if the label does not end in punctuation. + label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix # Translators: If found as last label character, these punctuation # characters will prevent the default label_suffix to be appended to the label - label_suffix = label_suffix if label_suffix is not None else self.form.label_suffix if label_suffix and contents and contents[-1] not in _(':?.!'): contents = format_html('{0}{1}', contents, label_suffix) widget = self.field.widget From 297f5af222bde02a7cdd005da2e4b00ec81801de Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Thu, 22 Aug 2013 08:25:21 -0300 Subject: [PATCH 25/34] Made description of LANGUAGE_CODE setting more clear. --- docs/ref/settings.txt | 15 +++++++++++++-- docs/topics/i18n/translation.txt | 13 ++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 424f7d5795..738ae32cdb 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1290,11 +1290,22 @@ LANGUAGE_CODE Default: ``'en-us'`` -A string representing the language code for this installation. This should be -in standard :term:`language format`. For example, U.S. English +A string representing the language code for this installation. This should be in +standard :term:`language ID format `. For example, U.S. English is ``"en-us"``. See also the `list of language identifiers`_ and :doc:`/topics/i18n/index`. +:setting:`USE_I18N` must be active to this setting to have any effect. + +it serves two purposes: + +* If the locale middleware isn't in use, it decides which translation is served + to all users. +* If the locale middleware is active, it provides the fallback translation when + no translation exist for a given literal to the user preferred language. + +See :ref:`how-django-discovers-language-preference` for more details. + .. _list of language identifiers: http://www.i18nguy.com/unicode/language-identifiers.html .. setting:: LANGUAGE_COOKIE_NAME diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 6436e7dcf9..86f6637d77 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1550,14 +1550,17 @@ should be used -- installation-wide, for a particular user, or both. To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`. Django uses this language as the default translation -- the final attempt if no -other translator finds a translation. +better matching translation is found by one of the methods employed by the +locale middleware (see below). -If all you want to do is run Django with your native language, and a language -file is available for it, all you need to do is set :setting:`LANGUAGE_CODE`. +If all you want to do is run Django with your native language all you need to do +is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message +files ` and their compiled versions (``.mo``) exist. If you want to let each individual user specify which language he or she -prefers, use ``LocaleMiddleware``. ``LocaleMiddleware`` enables language -selection based on data from the request. It customizes content for each user. +prefers, the you also need to use use the ``LocaleMiddleware``. +``LocaleMiddleware`` enables language selection based on data from the request. +It customizes content for each user. To use ``LocaleMiddleware``, add ``'django.middleware.locale.LocaleMiddleware'`` to your :setting:`MIDDLEWARE_CLASSES` setting. Because middleware order From ce0e86cf761dc182ae6fb8c0d68964955d19ec75 Mon Sep 17 00:00:00 2001 From: Dima Kurguzov Date: Thu, 22 Aug 2013 11:55:14 +0400 Subject: [PATCH 26/34] Fixed #20956 -- Removed useless check in django.db.utils --- django/db/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/utils.py b/django/db/utils.py index a1a2c0b564..8dbfc117a5 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -173,7 +173,7 @@ class ConnectionHandler(object): conn.setdefault('AUTOCOMMIT', False) conn.setdefault('AUTOCOMMIT', True) conn.setdefault('ENGINE', 'django.db.backends.dummy') - if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']: + if conn['ENGINE'] == 'django.db.backends.': conn['ENGINE'] = 'django.db.backends.dummy' conn.setdefault('CONN_MAX_AGE', 0) conn.setdefault('OPTIONS', {}) From bac4d03ce68eae7fe13f1891ecdc015817c6d1d8 Mon Sep 17 00:00:00 2001 From: Marc Tamlyn Date: Tue, 20 Aug 2013 10:26:48 +0100 Subject: [PATCH 27/34] Fixed #20944 -- Removed inaccurate statement about View.dispatch(). --- docs/ref/class-based-views/base.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/ref/class-based-views/base.txt b/docs/ref/class-based-views/base.txt index 0db1e15ea9..f0543e6095 100644 --- a/docs/ref/class-based-views/base.txt +++ b/docs/ref/class-based-views/base.txt @@ -79,10 +79,6 @@ View you can override the ``head()`` method. See :ref:`supporting-other-http-methods` for an example. - The default implementation also sets ``request``, ``args`` and - ``kwargs`` as instance variables, so any method on the view can know - the full details of the request that was made to invoke the view. - .. method:: http_method_not_allowed(request, *args, **kwargs) If the view was called with a HTTP method it doesn't support, this From 2e926b041c36d46d921acba516a262e21ddaa60d Mon Sep 17 00:00:00 2001 From: Kevin Christopher Henry Date: Thu, 22 Aug 2013 04:39:31 -0400 Subject: [PATCH 28/34] Documentation -- Clarified use of 'view' in test client introduction. --- docs/topics/testing/overview.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index 746c78f41b..89b43c42c5 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -328,7 +328,8 @@ Some of the things you can do with the test client are: everything from low-level HTTP (result headers and status codes) to page content. -* Test that the correct view is executed for a given URL. +* See the chain of redirects (if any) and check the URL and status code at + each step. * Test that a given request is rendered by a given Django template, with a template context that contains certain values. @@ -337,8 +338,8 @@ Note that the test client is not intended to be a replacement for Selenium_ or other "in-browser" frameworks. Django's test client has a different focus. In short: -* Use Django's test client to establish that the correct view is being - called and that the view is collecting the correct context data. +* Use Django's test client to establish that the correct template is being + rendered and that the template is passed the correct context data. * Use in-browser frameworks like Selenium_ to test *rendered* HTML and the *behavior* of Web pages, namely JavaScript functionality. Django also From 768bbf3efe0c412bced1e865e90139a0f07dc613 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Thu, 22 Aug 2013 09:47:17 -0400 Subject: [PATCH 29/34] Revert "Fixed #20956 -- Removed useless check in django.db.utils" This reverts commit ce0e86cf761dc182ae6fb8c0d68964955d19ec75. The check is necessary if 'ENGINE' is an empty string. Thanks apollo13 for pointing this out. --- django/db/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/utils.py b/django/db/utils.py index 8dbfc117a5..a1a2c0b564 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -173,7 +173,7 @@ class ConnectionHandler(object): conn.setdefault('AUTOCOMMIT', False) conn.setdefault('AUTOCOMMIT', True) conn.setdefault('ENGINE', 'django.db.backends.dummy') - if conn['ENGINE'] == 'django.db.backends.': + if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']: conn['ENGINE'] = 'django.db.backends.dummy' conn.setdefault('CONN_MAX_AGE', 0) conn.setdefault('OPTIONS', {}) From 6af05e7a0f0e4604d6a67899acaa99d73ec0dfaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Wed, 14 Aug 2013 11:05:01 +0300 Subject: [PATCH 30/34] Fixed model.__eq__ and __hash__ for no pk value cases The __eq__ method now considers two instances without primary key value equal only when they have same id(). The __hash__ method raises TypeError for no primary key case. Fixed #18864, fixed #18250 Thanks to Tim Graham for docs review. --- django/db/models/base.py | 13 ++++++++++--- django/forms/models.py | 6 +++++- docs/ref/models/instances.txt | 19 +++++++++++++++++++ docs/releases/1.7.txt | 8 ++++++++ tests/basic/tests.py | 11 +++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/django/db/models/base.py b/django/db/models/base.py index 3e2ae8d425..a5b0f188b4 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -459,14 +459,21 @@ class Model(six.with_metaclass(ModelBase)): return '%s object' % self.__class__.__name__ def __eq__(self, other): - return (isinstance(other, Model) and - self._meta.concrete_model == other._meta.concrete_model and - self._get_pk_val() == other._get_pk_val()) + if not isinstance(other, Model): + return False + if self._meta.concrete_model != other._meta.concrete_model: + return False + my_pk = self._get_pk_val() + if my_pk is None: + return self is other + return my_pk == other._get_pk_val() def __ne__(self, other): return not self.__eq__(other) def __hash__(self): + if self._get_pk_val() is None: + raise TypeError("Model instances without primary key value are unhashable") return hash(self._get_pk_val()) def __reduce__(self): diff --git a/django/forms/models.py b/django/forms/models.py index a5b82e521d..4c6ee9c6ed 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -631,7 +631,11 @@ class BaseModelFormSet(BaseFormSet): seen_data = set() for form in valid_forms: # get data for each field of each of unique_check - row_data = tuple([form.cleaned_data[field] for field in unique_check if field in form.cleaned_data]) + row_data = (form.cleaned_data[field] + for field in unique_check if field in form.cleaned_data) + # Reduce Model instances to their primary key values + row_data = tuple(d._get_pk_val() if hasattr(d, '_get_pk_val') else d + for d in row_data) if row_data and not None in row_data: # if we've already seen it then we have a uniqueness failure if row_data in seen_data: diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 6295eb0407..da657a9a01 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -521,6 +521,25 @@ For example:: In previous versions only instances of the exact same class and same primary key value were considered equal. +``__hash__`` +------------ + +.. method:: Model.__hash__() + +The ``__hash__`` method is based on the instance's primary key value. It +is effectively hash(obj.pk). If the instance doesn't have a primary key +value then a ``TypeError`` will be raised (otherwise the ``__hash__`` +method would return different values before and after the instance is +saved, but changing the ``__hash__`` value of an instance `is forbidden +in Python`_). + +.. versionchanged:: 1.7 + + In previous versions instance's without primary key value were + hashable. + +.. _is forbidden in Python: http://docs.python.org/reference/datamodel.html#object.__hash__ + ``get_absolute_url`` -------------------- diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 774d9d3161..6480a2505f 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -199,6 +199,14 @@ Miscellaneous equal when primary keys match. Previously only instances of exact same class were considered equal on primary key match. +* The :meth:`django.db.models.Model.__eq__` method has changed such that + two ``Model`` instances without primary key values won't be considered + equal (unless they are the same instance). + +* The :meth:`django.db.models.Model.__hash__` will now raise ``TypeError`` + when called on an instance without a primary key value. This is done to + avoid mutable ``__hash__`` values in containers. + Features deprecated in 1.7 ========================== diff --git a/tests/basic/tests.py b/tests/basic/tests.py index 2b051621ef..611944902a 100644 --- a/tests/basic/tests.py +++ b/tests/basic/tests.py @@ -708,9 +708,20 @@ class ModelTest(TestCase): SelfRef.objects.get(selfref=sr) def test_eq(self): + self.assertEqual(Article(id=1), Article(id=1)) self.assertNotEqual(Article(id=1), object()) self.assertNotEqual(object(), Article(id=1)) + a = Article() + self.assertEqual(a, a) + self.assertNotEqual(Article(), a) + def test_hash(self): + # Value based on PK + self.assertEqual(hash(Article(id=1)), hash(1)) + with self.assertRaises(TypeError): + # No PK value -> unhashable (because save() would then change + # hash) + hash(Article()) class ConcurrentSaveTests(TransactionTestCase): From 65cf82bd08631a7aa8d9dd007b2527476fa3304f Mon Sep 17 00:00:00 2001 From: Rainer Koirikivi Date: Wed, 21 Aug 2013 19:22:22 +0300 Subject: [PATCH 31/34] Fixed #20934 -- Avoided NoReverseMatch in ModelAdmin.changelist_view The view tried to display links to a ModelAdmin's change_view, which resulted in NoReverseMatches if get_urls was overridden to remove the corresponding url. --- .../contrib/admin/templatetags/admin_list.py | 42 ++++++++++++------- tests/admin_views/tests.py | 13 ++++++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py index 6c3c3e8511..ba109d1454 100644 --- a/django/contrib/admin/templatetags/admin_list.py +++ b/django/contrib/admin/templatetags/admin_list.py @@ -9,6 +9,7 @@ from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE, ORDER_VAR, PAGE_VAR, SEARCH_VAR) from django.contrib.admin.templatetags.admin_static import static from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import NoReverseMatch from django.db import models from django.utils import formats from django.utils.html import escapejs, format_html @@ -216,25 +217,36 @@ def items_for_result(cl, result, form): row_class = mark_safe(' class="%s"' % ' '.join(row_classes)) # If list_display_links not defined, add the link tag to the first field if (first and not cl.list_display_links) or field_name in cl.list_display_links: - table_tag = {True:'th', False:'td'}[first] + table_tag = 'th' if first else 'td' first = False - url = cl.url_for_result(result) - url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url) - # Convert the pk to something that can be used in Javascript. - # Problem cases are long ints (23L) and non-ASCII strings. - if cl.to_field: - attr = str(cl.to_field) + + # Display link to the result's change_view if the url exists, else + # display just the result's representation. + try: + url = cl.url_for_result(result) + except NoReverseMatch: + link_or_text = result_repr else: - attr = pk - value = result.serializable_value(attr) - result_id = escapejs(value) - yield format_html('<{0}{1}>{4}', + url = add_preserved_filters({'preserved_filters': cl.preserved_filters, 'opts': cl.opts}, url) + # Convert the pk to something that can be used in Javascript. + # Problem cases are long ints (23L) and non-ASCII strings. + if cl.to_field: + attr = str(cl.to_field) + else: + attr = pk + value = result.serializable_value(attr) + result_id = escapejs(value) + link_or_text = format_html( + '{2}', + url, + format_html(' onclick="opener.dismissRelatedLookupPopup(window, '{0}'); return false;"', result_id) + if cl.is_popup else '', + result_repr) + + yield format_html('<{0}{1}>{2}', table_tag, row_class, - url, - format_html(' onclick="opener.dismissRelatedLookupPopup(window, '{0}'); return false;"', result_id) - if cl.is_popup else '', - result_repr, + link_or_text, table_tag) else: # By default the fields come from ModelAdmin.list_editable, but if we pull diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py index 8f83324a37..66a5735b6c 100644 --- a/tests/admin_views/tests.py +++ b/tests/admin_views/tests.py @@ -630,6 +630,19 @@ class AdminViewBasicTest(AdminViewBasicTestCase): with self.assertRaises(AttributeError): self.client.get('/test_admin/%s/admin_views/simple/' % self.urlbit) + def test_changelist_with_no_change_url(self): + """ + ModelAdmin.changelist_view shouldn't result in a NoReverseMatch if url + for change_view is removed from get_urls + + Regression test for #20934 + """ + UnchangeableObject.objects.create() + response = self.client.get('/test_admin/admin/admin_views/unchangeableobject/') + self.assertEqual(response.status_code, 200) + # Check the format of the shown object -- shouldn't contain a change link + self.assertContains(response, 'UnchangeableObject object', html=True) + @override_settings(PASSWORD_HASHERS=('django.contrib.auth.hashers.SHA1PasswordHasher',)) class AdminViewFormUrlTest(TestCase): From 9d1987d7679165ad3a7c2b713a8a488cc1421905 Mon Sep 17 00:00:00 2001 From: Lukasz Balcerzak Date: Sat, 18 May 2013 22:15:10 +0200 Subject: [PATCH 32/34] Fixed #19303 -- Fixed ModelAdmin.formfield_overrides on fields with choices --- django/contrib/admin/options.py | 4 ++++ tests/admin_widgets/tests.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index b475868598..805868306d 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -107,6 +107,7 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): validator.validate(cls, model) def __init__(self): + self._orig_formfield_overrides = self.formfield_overrides overrides = FORMFIELD_FOR_DBFIELD_DEFAULTS.copy() overrides.update(self.formfield_overrides) self.formfield_overrides = overrides @@ -123,6 +124,9 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)): # If the field specifies choices, we don't need to look for special # admin widgets - we just need to use a select widget of some kind. if db_field.choices: + # see #19303 for an explanation of self._orig_formfield_overrides + if db_field.__class__ in self._orig_formfield_overrides: + kwargs = dict(self._orig_formfield_overrides[db_field.__class__], **kwargs) return self.formfield_for_choice_field(db_field, request, **kwargs) # ForeignKey or ManyToManyFields diff --git a/tests/admin_widgets/tests.py b/tests/admin_widgets/tests.py index d2ecf46358..7d2f70f69b 100644 --- a/tests/admin_widgets/tests.py +++ b/tests/admin_widgets/tests.py @@ -132,6 +132,23 @@ class AdminFormfieldForDBFieldTests(TestCase): self.assertEqual(f2.widget.attrs['maxlength'], '20') self.assertEqual(f2.widget.attrs['size'], '10') + def testFormfieldOverridesWidgetInstancesForFieldsWithChoices(self): + """ + Test that widget is actually overridden for fields with choices. + (#194303) + """ + class MemberAdmin(admin.ModelAdmin): + formfield_overrides = { + CharField: {'widget': forms.TextInput} + } + ma = MemberAdmin(models.Member, admin.site) + name_field = models.Member._meta.get_field('name') + gender_field = models.Member._meta.get_field('gender') + name = ma.formfield_for_dbfield(name_field, request=None) + gender = ma.formfield_for_dbfield(gender_field, request=None) + self.assertIsInstance(name.widget, forms.TextInput) + self.assertIsInstance(gender.widget, forms.TextInput) + def testFieldWithChoices(self): self.assertFormfield(models.Member, 'gender', forms.Select) From b0ce6fe656873825271bacb55e55474fc346c1c6 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 21 Aug 2013 20:12:19 -0400 Subject: [PATCH 33/34] Fixed #20922 -- Allowed customizing the serializer used by contrib.sessions Added settings.SESSION_SERIALIZER which is the import path of a serializer to use for sessions. Thanks apollo13, carljm, shaib, akaariai, charettes, and dstufft for reviews. --- django/conf/global_settings.py | 1 + django/contrib/messages/storage/session.py | 17 +++- django/contrib/messages/tests/base.py | 1 + django/contrib/messages/tests/test_session.py | 4 +- django/contrib/sessions/backends/base.py | 21 ++--- .../sessions/backends/signed_cookies.py | 21 +---- django/contrib/sessions/models.py | 2 +- django/contrib/sessions/serializers.py | 20 +++++ django/contrib/sessions/tests.py | 34 ++++---- docs/ref/settings.txt | 24 ++++- docs/releases/1.6.txt | 23 +++++ docs/topics/http/sessions.txt | 87 +++++++++++++++++-- tests/defer_regress/tests.py | 40 +++++---- 13 files changed, 218 insertions(+), 77 deletions(-) create mode 100644 django/contrib/sessions/serializers.py diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 95deaa8d87..ab3cdab59e 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -475,6 +475,7 @@ SESSION_SAVE_EVERY_REQUEST = False # Whether to save the se SESSION_EXPIRE_AT_BROWSER_CLOSE = False # Whether a user's session cookie expires when the Web browser is closed. SESSION_ENGINE = 'django.contrib.sessions.backends.db' # The module to store session data SESSION_FILE_PATH = None # Directory to store session files if using the file session module. If None, the backend will use a sensible default. +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer' # class to serialize session data ######### # CACHE # diff --git a/django/contrib/messages/storage/session.py b/django/contrib/messages/storage/session.py index 225dfda289..c3e293c22e 100644 --- a/django/contrib/messages/storage/session.py +++ b/django/contrib/messages/storage/session.py @@ -1,4 +1,8 @@ +import json + from django.contrib.messages.storage.base import BaseStorage +from django.contrib.messages.storage.cookie import MessageEncoder, MessageDecoder +from django.utils import six class SessionStorage(BaseStorage): @@ -20,14 +24,23 @@ class SessionStorage(BaseStorage): always stores everything it is given, so return True for the all_retrieved flag. """ - return self.request.session.get(self.session_key), True + return self.deserialize_messages(self.request.session.get(self.session_key)), True def _store(self, messages, response, *args, **kwargs): """ Stores a list of messages to the request's session. """ if messages: - self.request.session[self.session_key] = messages + self.request.session[self.session_key] = self.serialize_messages(messages) else: self.request.session.pop(self.session_key, None) return [] + + def serialize_messages(self, messages): + encoder = MessageEncoder(separators=(',', ':')) + return encoder.encode(messages) + + def deserialize_messages(self, data): + if data and isinstance(data, six.string_types): + return json.loads(data, cls=MessageDecoder) + return data diff --git a/django/contrib/messages/tests/base.py b/django/contrib/messages/tests/base.py index 7011a5779a..5241436daf 100644 --- a/django/contrib/messages/tests/base.py +++ b/django/contrib/messages/tests/base.py @@ -61,6 +61,7 @@ class BaseTests(object): MESSAGE_TAGS = '', MESSAGE_STORAGE = '%s.%s' % (self.storage_class.__module__, self.storage_class.__name__), + SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer', ) self.settings_override.enable() diff --git a/django/contrib/messages/tests/test_session.py b/django/contrib/messages/tests/test_session.py index 2ce564b773..940e1c02d0 100644 --- a/django/contrib/messages/tests/test_session.py +++ b/django/contrib/messages/tests/test_session.py @@ -11,13 +11,13 @@ def set_session_data(storage, messages): Sets the messages into the backend request's session and remove the backend's loaded data cache. """ - storage.request.session[storage.session_key] = messages + storage.request.session[storage.session_key] = storage.serialize_messages(messages) if hasattr(storage, '_loaded_data'): del storage._loaded_data def stored_session_messages_count(storage): - data = storage.request.session.get(storage.session_key, []) + data = storage.deserialize_messages(storage.request.session.get(storage.session_key, [])) return len(data) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 759d7ac7ad..7f5e958a60 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -3,11 +3,6 @@ from __future__ import unicode_literals import base64 from datetime import datetime, timedelta import logging - -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle import string from django.conf import settings @@ -17,6 +12,7 @@ from django.utils.crypto import get_random_string from django.utils.crypto import salted_hmac from django.utils import timezone from django.utils.encoding import force_bytes, force_text +from django.utils.module_loading import import_by_path from django.contrib.sessions.exceptions import SuspiciousSession @@ -42,6 +38,7 @@ class SessionBase(object): self._session_key = session_key self.accessed = False self.modified = False + self.serializer = import_by_path(settings.SESSION_SERIALIZER) def __contains__(self, key): return key in self._session @@ -86,21 +83,21 @@ class SessionBase(object): return salted_hmac(key_salt, value).hexdigest() def encode(self, session_dict): - "Returns the given session dictionary pickled and encoded as a string." - pickled = pickle.dumps(session_dict, pickle.HIGHEST_PROTOCOL) - hash = self._hash(pickled) - return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii') + "Returns the given session dictionary serialized and encoded as a string." + serialized = self.serializer().dumps(session_dict) + hash = self._hash(serialized) + return base64.b64encode(hash.encode() + b":" + serialized).decode('ascii') def decode(self, session_data): encoded_data = base64.b64decode(force_bytes(session_data)) try: # could produce ValueError if there is no ':' - hash, pickled = encoded_data.split(b':', 1) - expected_hash = self._hash(pickled) + hash, serialized = encoded_data.split(b':', 1) + expected_hash = self._hash(serialized) if not constant_time_compare(hash.decode(), expected_hash): raise SuspiciousSession("Session data corrupted") else: - return pickle.loads(pickled) + return self.serializer().loads(serialized) except Exception as e: # ValueError, SuspiciousOperation, unpickling exceptions. If any of # these happen, just return an empty dictionary (an empty session). diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py index c2b7a3123f..77a6750ce4 100644 --- a/django/contrib/sessions/backends/signed_cookies.py +++ b/django/contrib/sessions/backends/signed_cookies.py @@ -1,26 +1,9 @@ -try: - from django.utils.six.moves import cPickle as pickle -except ImportError: - import pickle - from django.conf import settings from django.core import signing from django.contrib.sessions.backends.base import SessionBase -class PickleSerializer(object): - """ - Simple wrapper around pickle to be used in signing.dumps and - signing.loads. - """ - def dumps(self, obj): - return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) - - def loads(self, data): - return pickle.loads(data) - - class SessionStore(SessionBase): def load(self): @@ -31,7 +14,7 @@ class SessionStore(SessionBase): """ try: return signing.loads(self.session_key, - serializer=PickleSerializer, + serializer=self.serializer, # This doesn't handle non-default expiry dates, see #19201 max_age=settings.SESSION_COOKIE_AGE, salt='django.contrib.sessions.backends.signed_cookies') @@ -91,7 +74,7 @@ class SessionStore(SessionBase): session_cache = getattr(self, '_session_cache', {}) return signing.dumps(session_cache, compress=True, salt='django.contrib.sessions.backends.signed_cookies', - serializer=PickleSerializer) + serializer=self.serializer) @classmethod def clear_expired(cls): diff --git a/django/contrib/sessions/models.py b/django/contrib/sessions/models.py index 0179c358b3..3a6e31152f 100644 --- a/django/contrib/sessions/models.py +++ b/django/contrib/sessions/models.py @@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _ class SessionManager(models.Manager): def encode(self, session_dict): """ - Returns the given session dictionary pickled and encoded as a string. + Returns the given session dictionary serialized and encoded as a string. """ return SessionStore().encode(session_dict) diff --git a/django/contrib/sessions/serializers.py b/django/contrib/sessions/serializers.py new file mode 100644 index 0000000000..92a31c054b --- /dev/null +++ b/django/contrib/sessions/serializers.py @@ -0,0 +1,20 @@ +from django.core.signing import JSONSerializer as BaseJSONSerializer +try: + from django.utils.six.moves import cPickle as pickle +except ImportError: + import pickle + + +class PickleSerializer(object): + """ + Simple wrapper around pickle to be used in signing.dumps and + signing.loads. + """ + def dumps(self, obj): + return pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) + + def loads(self, data): + return pickle.loads(data) + + +JSONSerializer = BaseJSONSerializer diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index f2a35c544e..4caefe938c 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -285,21 +285,25 @@ class SessionTestsMixin(object): def test_actual_expiry(self): - # Regression test for #19200 - old_session_key = None - new_session_key = None - try: - self.session['foo'] = 'bar' - self.session.set_expiry(-timedelta(seconds=10)) - self.session.save() - old_session_key = self.session.session_key - # With an expiry date in the past, the session expires instantly. - new_session = self.backend(self.session.session_key) - new_session_key = new_session.session_key - self.assertNotIn('foo', new_session) - finally: - self.session.delete(old_session_key) - self.session.delete(new_session_key) + # this doesn't work with JSONSerializer (serializing timedelta) + with override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer'): + self.session = self.backend() # reinitialize after overriding settings + + # Regression test for #19200 + old_session_key = None + new_session_key = None + try: + self.session['foo'] = 'bar' + self.session.set_expiry(-timedelta(seconds=10)) + self.session.save() + old_session_key = self.session.session_key + # With an expiry date in the past, the session expires instantly. + new_session = self.backend(self.session.session_key) + new_session_key = new_session.session_key + self.assertNotIn('foo', new_session) + finally: + self.session.delete(old_session_key) + self.session.delete(new_session_key) class DatabaseSessionTests(SessionTestsMixin, TestCase): diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 738ae32cdb..0105b2ccf5 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2403,7 +2403,7 @@ SESSION_ENGINE Default: ``django.contrib.sessions.backends.db`` -Controls where Django stores session data. Valid values are: +Controls where Django stores session data. Included engines are: * ``'django.contrib.sessions.backends.db'`` * ``'django.contrib.sessions.backends.file'`` @@ -2446,6 +2446,28 @@ Whether to save the session data on every request. If this is ``False`` (default), then the session data will only be saved if it has been modified -- that is, if any of its dictionary values have been assigned or deleted. +.. setting:: SESSION_SERIALIZER + +SESSION_SERIALIZER +------------------ + +Default: ``'django.contrib.sessions.serializers.JSONSerializer'`` + +.. versionchanged:: 1.6 + + The default switched from + :class:`~django.contrib.sessions.serializers.PickleSerializer` to + :class:`~django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. + +Full import path of a serializer class to use for serializing session data. +Included serializers are: + +* ``'django.contrib.sessions.serializers.PickleSerializer'`` +* ``'django.contrib.sessions.serializers.JSONSerializer'`` + +See :ref:`session_serialization` for details, including a warning regarding +possible remote code execution when using +:class:`~django.contrib.sessions.serializers.PickleSerializer`. Sites ===== diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index c0f5c51194..556edddda1 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -727,6 +727,29 @@ the ``name`` argument so it doesn't conflict with the new url:: You can remove this url pattern after your app has been deployed with Django 1.6 for :setting:`PASSWORD_RESET_TIMEOUT_DAYS`. +Default session serialization switched to JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Historically, :mod:`django.contrib.sessions` used :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaulted to using :mod:`pickle` +in Django 1.5.3, but we've changed the default to JSON in 1.6. If you upgrade +and switch from pickle to JSON, sessions created before the upgrade will be +lost. While JSON serialization does not support all Python objects like +:mod:`pickle` does, we highly recommend using JSON-serialized sessions. See the +:ref:`session_serialization` documentation for more details. + Miscellaneous ~~~~~~~~~~~~~ diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 87b5972c2e..637991b3b5 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -128,8 +128,9 @@ and the :setting:`SECRET_KEY` setting. .. warning:: - **If the SECRET_KEY is not kept secret, this can lead to arbitrary remote - code execution.** + **If the SECRET_KEY is not kept secret and you are using the** + :class:`~django.contrib.sessions.serializers.PickleSerializer`, **this can + lead to arbitrary remote code execution.** An attacker in possession of the :setting:`SECRET_KEY` can not only generate falsified session data, which your site will trust, but also @@ -256,7 +257,9 @@ You can edit it multiple times. in 5 minutes. * If ``value`` is a ``datetime`` or ``timedelta`` object, the - session will expire at that specific date/time. + session will expire at that specific date/time. Note that ``datetime`` + and ``timedelta`` values are only serializable if you are using the + :class:`~django.contrib.sessions.serializers.PickleSerializer`. * If ``value`` is ``0``, the user's session cookie will expire when the user's Web browser is closed. @@ -301,6 +304,72 @@ You can edit it multiple times. Removes expired sessions from the session store. This class method is called by :djadmin:`clearsessions`. +.. _session_serialization: + +Session serialization +--------------------- + +.. versionchanged:: 1.6 + +Before version 1.6, Django defaulted to using :mod:`pickle` to serialize +session data before storing it in the backend. If you're using the :ref:`signed +cookie session backend` and :setting:`SECRET_KEY` is +known by an attacker, the attacker could insert a string into his session +which, when unpickled, executes arbitrary code on the server. The technique for +doing so is simple and easily available on the internet. Although the cookie +session storage signs the cookie-stored data to prevent tampering, a +:setting:`SECRET_KEY` leak immediately escalates to a remote code execution +vulnerability. + +This attack can be mitigated by serializing session data using JSON rather +than :mod:`pickle`. To facilitate this, Django 1.5.3 introduced a new setting, +:setting:`SESSION_SERIALIZER`, to customize the session serialization format. +For backwards compatibility, this setting defaults to +using :class:`django.contrib.sessions.serializers.PickleSerializer` in +Django 1.5.x, but, for security hardening, defaults to +:class:`django.contrib.sessions.serializers.JSONSerializer` in Django 1.6. +Even with the caveats described in :ref:`custom-serializers`, we highly +recommend sticking with JSON serialization *especially if you are using the +cookie backend*. + +Bundled Serializers +^^^^^^^^^^^^^^^^^^^ + +.. class:: serializers.JSONSerializer + + A wrapper around the JSON serializer from :mod:`django.core.signing`. Can + only serialize basic data types. See the :ref:`custom-serializers` section + for more details. + +.. class:: serializers.PickleSerializer + + Supports arbitrary Python objects, but, as described above, can lead to a + remote code execution vulnerability if :setting:`SECRET_KEY` becomes known + by an attacker. + +.. _custom-serializers: + +Write Your Own Serializer +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that unlike :class:`~django.contrib.sessions.serializers.PickleSerializer`, +the :class:`~django.contrib.sessions.serializers.JSONSerializer` cannot handle +arbitrary Python data types. As is often the case, there is a trade-off between +convenience and security. If you wish to store more advanced data types +including ``datetime`` and ``Decimal`` in JSON backed sessions, you will need +to write a custom serializer (or convert such values to a JSON serializable +object before storing them in ``request.session``). While serializing these +values is fairly straightforward +(``django.core.serializers.json.DateTimeAwareJSONEncoder`` may be helpful), +writing a decoder that can reliably get back the same thing that you put in is +more fragile. For example, you run the risk of returning a ``datetime`` that +was actually a string that just happened to be in the same format chosen for +``datetime``\s). + +Your serializer class must implement two methods, +``dumps(self, obj)`` and ``loads(self, data)``, to serialize and deserialize +the dictionary of session data, respectively. + Session object guidelines ------------------------- @@ -390,14 +459,15 @@ An API is available to manipulate session data outside of a view:: >>> from django.contrib.sessions.backends.db import SessionStore >>> import datetime >>> s = SessionStore() - >>> s['last_login'] = datetime.datetime(2005, 8, 20, 13, 35, 10) + >>> # stored as seconds since epoch since datetimes are not serializable in JSON. + >>> s['last_login'] = 1376587691 >>> s.save() >>> s.session_key '2b1189a188b44ad18c35e113ac6ceead' >>> s = SessionStore(session_key='2b1189a188b44ad18c35e113ac6ceead') >>> s['last_login'] - datetime.datetime(2005, 8, 20, 13, 35, 0) + 1376587691 In order to prevent session fixation attacks, sessions keys that don't exist are regenerated:: @@ -543,8 +613,11 @@ behavior: Technical details ================= -* The session dictionary should accept any pickleable Python object. See - the :mod:`pickle` module for more information. +* The session dictionary accepts any :mod:`json` serializable value when using + :class:`~django.contrib.sessions.serializers.JSONSerializer` or any + pickleable Python object when using + :class:`~django.contrib.sessions.serializers.PickleSerializer`. See the + :mod:`pickle` module for more information. * Session data is stored in a database table named ``django_session`` . diff --git a/tests/defer_regress/tests.py b/tests/defer_regress/tests.py index 619f65163c..ffb47a8133 100644 --- a/tests/defer_regress/tests.py +++ b/tests/defer_regress/tests.py @@ -7,6 +7,7 @@ from django.contrib.sessions.backends.db import SessionStore from django.db.models import Count from django.db.models.loading import cache from django.test import TestCase +from django.test.utils import override_settings from .models import ( ResolveThis, Item, RelatedItem, Child, Leaf, Proxy, SimpleItem, Feature, @@ -83,24 +84,6 @@ class DeferRegressionTest(TestCase): self.assertEqual(results[0].child.name, "c1") self.assertEqual(results[0].second_child.name, "c2") - # Test for #12163 - Pickling error saving session with unsaved model - # instances. - SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' - - item = Item() - item._deferred = False - s = SessionStore(SESSION_KEY) - s.clear() - s["item"] = item - s.save() - - s = SessionStore(SESSION_KEY) - s.modified = True - s.save() - - i2 = s["item"] - self.assertFalse(i2._deferred) - # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), @@ -147,6 +130,27 @@ class DeferRegressionTest(TestCase): cache.get_app("defer_regress"), include_deferred=True)) ) + @override_settings(SESSION_SERIALIZER='django.contrib.sessions.serializers.PickleSerializer') + def test_ticket_12163(self): + # Test for #12163 - Pickling error saving session with unsaved model + # instances. + SESSION_KEY = '2b1189a188b44ad18c35e1baac6ceead' + + item = Item() + item._deferred = False + s = SessionStore(SESSION_KEY) + s.clear() + s["item"] = item + s.save() + + s = SessionStore(SESSION_KEY) + s.modified = True + s.save() + + i2 = s["item"] + self.assertFalse(i2._deferred) + + def test_ticket_16409(self): # Regression for #16409 - make sure defer() and only() work with annotate() self.assertIsInstance( list(SimpleItem.objects.annotate(Count('feature')).defer('name')), From 57c82f909b212708a17edd11014be718bd02be3b Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Thu, 22 Aug 2013 12:14:56 -0300 Subject: [PATCH 34/34] Typos introduced in 297f5af222. --- docs/ref/settings.txt | 8 ++++---- docs/topics/i18n/translation.txt | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 0105b2ccf5..2f531803bc 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1291,18 +1291,18 @@ LANGUAGE_CODE Default: ``'en-us'`` A string representing the language code for this installation. This should be in -standard :term:`language ID format `. For example, U.S. English +standard :term:`language ID format `. For example, U.S. English is ``"en-us"``. See also the `list of language identifiers`_ and :doc:`/topics/i18n/index`. -:setting:`USE_I18N` must be active to this setting to have any effect. +:setting:`USE_I18N` must be active for this setting to have any effect. -it serves two purposes: +It serves two purposes: * If the locale middleware isn't in use, it decides which translation is served to all users. * If the locale middleware is active, it provides the fallback translation when - no translation exist for a given literal to the user preferred language. + no translation exist for a given literal to the user's preferred language. See :ref:`how-django-discovers-language-preference` for more details. diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 86f6637d77..120db8e5b0 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1550,15 +1550,15 @@ should be used -- installation-wide, for a particular user, or both. To set an installation-wide language preference, set :setting:`LANGUAGE_CODE`. Django uses this language as the default translation -- the final attempt if no -better matching translation is found by one of the methods employed by the +better matching translation is found through one of the methods employed by the locale middleware (see below). -If all you want to do is run Django with your native language all you need to do +If all you want is to run Django with your native language all you need to do is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message files ` and their compiled versions (``.mo``) exist. If you want to let each individual user specify which language he or she -prefers, the you also need to use use the ``LocaleMiddleware``. +prefers, then you also need to use use the ``LocaleMiddleware``. ``LocaleMiddleware`` enables language selection based on data from the request. It customizes content for each user.