From 3fce0d2a9162cf6e749a6de0b18890dea8955e89 Mon Sep 17 00:00:00 2001 From: Simon Meers Date: Mon, 20 Aug 2012 16:47:30 +1000 Subject: [PATCH 01/82] Fixed #18063 -- Avoid unicode in Model.__repr__ in python 2 Thanks guettli and mrmachine. --- django/db/models/base.py | 2 ++ tests/regressiontests/model_regress/tests.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/django/db/models/base.py b/django/db/models/base.py index fd7250cdb0..d7a9932388 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -404,6 +404,8 @@ class Model(six.with_metaclass(ModelBase, object)): u = six.text_type(self) except (UnicodeEncodeError, UnicodeDecodeError): u = '[Bad Unicode data]' + if not six.PY3: + u = u.encode('ascii', 'replace') return smart_str('<%s: %s>' % (self.__class__.__name__, u)) def __str__(self): diff --git a/tests/regressiontests/model_regress/tests.py b/tests/regressiontests/model_regress/tests.py index 6a45a83052..6a03b861e4 100644 --- a/tests/regressiontests/model_regress/tests.py +++ b/tests/regressiontests/model_regress/tests.py @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import, unicode_literals import datetime @@ -146,6 +148,14 @@ class ModelTests(TestCase): b = BrokenUnicodeMethod.objects.create(name="Jerry") self.assertEqual(repr(b), "") + def test_no_unicode_in_repr(self): + a = Article.objects.create( + headline="Watch for umlauts: üöä", pub_date=datetime.datetime.now()) + if six.PY3: + self.assertEqual(repr(a), '') + else: + self.assertEqual(repr(a), '') + @skipUnlessDBFeature("supports_timezones") def test_timezones(self): # Saving an updating with timezone-aware datetime Python objects. From bfc380baeaca359ac7b36eb35a2415e6a37e836c Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 20 Aug 2012 10:46:21 +0200 Subject: [PATCH 02/82] [py3] Prepared MySQL backend for Python 3 compatibility --- django/db/backends/mysql/compiler.py | 8 +++++++- django/db/backends/mysql/introspection.py | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/django/db/backends/mysql/compiler.py b/django/db/backends/mysql/compiler.py index 012bca5da6..d8e9b3a202 100644 --- a/django/db/backends/mysql/compiler.py +++ b/django/db/backends/mysql/compiler.py @@ -1,10 +1,16 @@ +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest + from django.db.models.sql import compiler + class SQLCompiler(compiler.SQLCompiler): def resolve_columns(self, row, fields=()): values = [] index_extra_select = len(self.query.extra_select) - for value, field in map(None, row[index_extra_select:], fields): + for value, field in zip_longest(row[index_extra_select:], fields): if (field and field.get_internal_type() in ("BooleanField", "NullBooleanField") and value in (0, 1)): value = bool(value) diff --git a/django/db/backends/mysql/introspection.py b/django/db/backends/mysql/introspection.py index 6aab0b99ab..041fbee1b2 100644 --- a/django/db/backends/mysql/introspection.py +++ b/django/db/backends/mysql/introspection.py @@ -1,8 +1,9 @@ +import re +from .base import FIELD_TYPE + from django.db.backends import BaseDatabaseIntrospection from django.utils import six -from MySQLdb import ProgrammingError, OperationalError -from MySQLdb.constants import FIELD_TYPE -import re + foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)") From 4e82d614002f3fb46a3504b3a4fa2dcbf7f53dab Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 20 Aug 2012 15:01:57 +0200 Subject: [PATCH 03/82] Added links in URLs doc for consistency. --- docs/topics/http/urls.txt | 6 ++++++ docs/topics/http/views.txt | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/docs/topics/http/urls.txt b/docs/topics/http/urls.txt index 4e75dfe55f..819ce24b91 100644 --- a/docs/topics/http/urls.txt +++ b/docs/topics/http/urls.txt @@ -314,6 +314,9 @@ that should be called if none of the URL patterns match. By default, this is ``'django.views.defaults.page_not_found'``. That default value should suffice. +See the documentation about :ref:`the 404 (HTTP Not Found) view +` for more information. + handler500 ---------- @@ -326,6 +329,9 @@ have runtime errors in view code. By default, this is ``'django.views.defaults.server_error'``. That default value should suffice. +See the documentation about :ref:`the 500 (HTTP Internal Server Error) view +` for more information. + Notes on capturing text in URLs =============================== diff --git a/docs/topics/http/views.txt b/docs/topics/http/views.txt index a8c85ddf6e..c4bd15e72e 100644 --- a/docs/topics/http/views.txt +++ b/docs/topics/http/views.txt @@ -127,6 +127,8 @@ called ``404.html`` and located in the top level of your template tree. Customizing error views ======================= +.. _http_not_found_view: + The 404 (page not found) view ----------------------------- @@ -167,6 +169,8 @@ Four things to note about 404 views: your 404 view will never be used, and your URLconf will be displayed instead, with some debug information. +.. _http_internal_server_error_view: + The 500 (server error) view ---------------------------- From 1288572d9203472be8aa28e956027899fec3ed92 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 20 Aug 2012 18:23:17 +0200 Subject: [PATCH 04/82] Made an example more readable in the URLs docs. --- docs/topics/http/urls.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/topics/http/urls.txt b/docs/topics/http/urls.txt index 819ce24b91..7297184ed3 100644 --- a/docs/topics/http/urls.txt +++ b/docs/topics/http/urls.txt @@ -574,10 +574,8 @@ For example:: (r'^blog/(?P\d{4})/$', 'year_archive', {'foo': 'bar'}), ) -In this example, for a request to ``/blog/2005/``, Django will call the -``blog.views.year_archive()`` view, passing it these keyword arguments:: - - year='2005', foo='bar' +In this example, for a request to ``/blog/2005/``, Django will call +``blog.views.year_archive(year='2005', foo='bar')``. This technique is used in the :doc:`syndication framework ` to pass metadata and From 610746f6cb89c13b364c27fc41ecc9592cbe1605 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 20 Aug 2012 21:21:00 +0200 Subject: [PATCH 05/82] Fixed #18023 -- Documented simplejson issues. Thanks Luke Plant for reporting the issue and Alex Ogier for thoroughly investigating it. --- docs/releases/1.5.txt | 62 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index 4f25919d79..6042569cea 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -172,6 +172,54 @@ If you were using the ``data`` parameter in a PUT request without a ``content_type``, you must encode your data before passing it to the test client and set the ``content_type`` argument. +.. _simplejson-incompatibilities: + +System version of :mod:`simplejson` no longer used +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:ref:`As explained below `, Django 1.5 deprecates +:mod:`django.utils.simplejson` in favor of Python 2.6's built-in :mod:`json` +module. In theory, this change is harmless. Unfortunately, because of +incompatibilities between versions of :mod:`simplejson`, it may trigger errors +in some circumstances. + +JSON-related features in Django 1.4 always used :mod:`django.utils.simplejson`. +This module was actually: + +- A system version of :mod:`simplejson`, if one was available (ie. ``import + simplejson`` works), if it was more recent than Django's built-in copy or it + had the C speedups, or +- The :mod:`json` module from the standard library, if it was available (ie. + Python 2.6 or greater), or +- A built-in copy of version 2.0.7 of :mod:`simplejson`. + +In Django 1.5, those features use Python's :mod:`json` module, which is based +on version 2.0.9 of :mod:`simplejson`. + +There are no known incompatibilities between Django's copy of version 2.0.7 and +Python's copy of version 2.0.9. However, there are some incompatibilities +between other versions of :mod:`simplejson`: + +- While the :mod:`simplejson` API is documented as always returning unicode + strings, the optional C implementation can return a byte string. This was + fixed in Python 2.7. +- :class:`simplejson.JSONEncoder` gained a ``namedtuple_as_object`` keyword + argument in version 2.2. + +More information on these incompatibilities is available in `ticket #18023`_. + +The net result is that, if you have installed :mod:`simplejson` and your code +uses Django's serialization internals directly -- for instance +:class:`django.core.serializers.json.DjangoJSONEncoder`, the switch from +:mod:`simplejson` to :mod:`json` could break your code. (In general, changes to +internals aren't documented; we're making an exception here.) + +At this point, the maintainers of Django believe that using :mod:`json` from +the standard library offers the strongest guarantee of backwards-compatibility. +They recommend to use it from now on. + +.. _ticket #18023: https://code.djangoproject.com/ticket/18023#comment:10 + String types of hasher method parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -282,13 +330,21 @@ Miscellaneous Features deprecated in 1.5 ========================== +.. _simplejson-deprecation: + ``django.utils.simplejson`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since Django 1.5 drops support for Python 2.5, we can now rely on the -:mod:`json` module being in Python's standard library -- so we've removed -our own copy of ``simplejson``. You can safely change any use of -:mod:`django.utils.simplejson` to :mod:`json`. +:mod:`json` module being available in Python's standard library, so we've +removed our own copy of :mod:`simplejson`. You should now import :mod:`json` +instead :mod:`django.utils.simplejson`. + +Unfortunately, this change might have unwanted side-effects, because of +incompatibilities between versions of :mod:`simplejson` -- see the +:ref:`backwards-incompatible changes ` section. +If you rely on features added to :mod:`simplejson` after it became Python's +:mod:`json`, you should import :mod:`simplejson` explicitly. ``itercompat.product`` ~~~~~~~~~~~~~~~~~~~~~~ From 54899d810dfb892558699ddbd5f724a358dc96e1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 20 Aug 2012 22:24:48 +0200 Subject: [PATCH 06/82] [py3] Fixed #18805 -- ported createsuperuser. Thanks sunsesh at gmail.com for the report. --- django/contrib/auth/management/__init__.py | 25 ++++++++++++++-------- django/contrib/auth/tests/management.py | 8 +++++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py index 83e1518387..23a053d985 100644 --- a/django/contrib/auth/management/__init__.py +++ b/django/contrib/auth/management/__init__.py @@ -9,6 +9,7 @@ import unicodedata from django.contrib.auth import models as auth_app from django.db.models import get_models, signals from django.contrib.auth.models import User +from django.utils import six from django.utils.six.moves import input @@ -84,17 +85,23 @@ def get_system_username(): :returns: The username as a unicode string, or an empty string if the username could not be determined. """ - default_locale = locale.getdefaultlocale()[1] - if default_locale: + try: + result = getpass.getuser() + except (ImportError, KeyError): + # KeyError will be raised by os.getpwuid() (called by getuser()) + # if there is no corresponding entry in the /etc/passwd file + # (a very restricted chroot environment, for example). + return '' + if not six.PY3: + default_locale = locale.getdefaultlocale()[1] + if not default_locale: + return '' try: - return getpass.getuser().decode(default_locale) - except (ImportError, KeyError, UnicodeDecodeError): - # KeyError will be raised by os.getpwuid() (called by getuser()) - # if there is no corresponding entry in the /etc/passwd file - # (a very restricted chroot environment, for example). + result = result.decode(default_locale) + except UnicodeDecodeError: # UnicodeDecodeError - preventive treatment for non-latin Windows. - pass - return '' + return '' + return result def get_default_username(check_db=True): diff --git a/django/contrib/auth/tests/management.py b/django/contrib/auth/tests/management.py index c98b7491c8..ac83086dc3 100644 --- a/django/contrib/auth/tests/management.py +++ b/django/contrib/auth/tests/management.py @@ -4,16 +4,20 @@ from django.contrib.auth import models, management from django.contrib.auth.management.commands import changepassword from django.core.management.base import CommandError from django.test import TestCase +from django.utils import six from django.utils.six import StringIO class GetDefaultUsernameTestCase(TestCase): def setUp(self): - self._getpass_getuser = management.get_system_username + self.old_get_system_username = management.get_system_username def tearDown(self): - management.get_system_username = self._getpass_getuser + management.get_system_username = self.old_get_system_username + + def test_actual_implementation(self): + self.assertIsInstance(management.get_system_username(), six.text_type) def test_simple(self): management.get_system_username = lambda: 'joe' From 62954ba04c86c4cea3b17a872ba6a0837730e17d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 20 Aug 2012 22:45:13 +0200 Subject: [PATCH 07/82] [py3] Fixed #17040 -- ported django.utils.crypto.constant_time_compare. This is a private API; adding a type check is acceptable. --- django/utils/crypto.py | 13 ++++++++++--- tests/regressiontests/utils/crypto.py | 12 +++++++++++- tests/regressiontests/utils/tests.py | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 70a07e7fde..e8fd2b1e2d 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -24,6 +24,7 @@ except NotImplementedError: from django.conf import settings from django.utils.encoding import smart_bytes +from django.utils import six from django.utils.six.moves import xrange @@ -81,15 +82,21 @@ def get_random_string(length=12, def constant_time_compare(val1, val2): """ - Returns True if the two strings are equal, False otherwise. + Returns True if the two bytestrings are equal, False otherwise. The time taken is independent of the number of characters that match. """ + if not (isinstance(val1, bytes) and isinstance(val2, bytes)): + raise TypeError("constant_time_compare only supports bytes") if len(val1) != len(val2): return False result = 0 - for x, y in zip(val1, val2): - result |= ord(x) ^ ord(y) + if six.PY3: + for x, y in zip(val1, val2): + result |= x ^ y + else: + for x, y in zip(val1, val2): + result |= ord(x) ^ ord(y) return result == 0 diff --git a/tests/regressiontests/utils/crypto.py b/tests/regressiontests/utils/crypto.py index 52a286cb27..447b2e5b80 100644 --- a/tests/regressiontests/utils/crypto.py +++ b/tests/regressiontests/utils/crypto.py @@ -6,7 +6,17 @@ import timeit import hashlib from django.utils import unittest -from django.utils.crypto import pbkdf2 +from django.utils.crypto import constant_time_compare, pbkdf2 + + +class TestUtilsCryptoMisc(unittest.TestCase): + + def test_constant_time_compare(self): + # It's hard to test for constant time, just test the result. + self.assertTrue(constant_time_compare(b'spam', b'spam')) + self.assertFalse(constant_time_compare(b'spam', b'eggs')) + with self.assertRaises(TypeError): + constant_time_compare('spam', 'spam') class TestUtilsCryptoPBKDF2(unittest.TestCase): diff --git a/tests/regressiontests/utils/tests.py b/tests/regressiontests/utils/tests.py index 91c0fd67e8..08f5f760cc 100644 --- a/tests/regressiontests/utils/tests.py +++ b/tests/regressiontests/utils/tests.py @@ -6,7 +6,7 @@ from __future__ import absolute_import from .archive import TestBzip2Tar, TestGzipTar, TestTar, TestZip from .baseconv import TestBaseConv from .checksums import TestUtilsChecksums -from .crypto import TestUtilsCryptoPBKDF2 +from .crypto import TestUtilsCryptoMisc, TestUtilsCryptoPBKDF2 from .datastructures import (DictWrapperTests, ImmutableListTests, MergeDictTests, MultiValueDictTests, SortedDictTests) from .dateformat import DateFormatTests From 831f2846dd9b3a93344e3c5cb3ea57d76317ae2c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Mon, 20 Aug 2012 14:27:28 -0700 Subject: [PATCH 08/82] Fixed #18819 -- fixed some typos in the auth docs --- docs/topics/auth.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/auth.txt b/docs/topics/auth.txt index 307691bd4a..a4e0f677b5 100644 --- a/docs/topics/auth.txt +++ b/docs/topics/auth.txt @@ -460,7 +460,7 @@ algorithm. Increasing the work factor ~~~~~~~~~~~~~~~~~~~~~~~~~~ -The PDKDF2 and bcrypt algorithms use a number of iterations or rounds of +The PBKDF2 and bcrypt algorithms use a number of iterations or rounds of hashing. This deliberately slows down attackers, making attacks against hashed passwords harder. However, as computing power increases, the number of iterations needs to be increased. We've chosen a reasonable default (and will @@ -468,7 +468,7 @@ increase it with each release of Django), but you may wish to tune it up or down, depending on your security needs and available processing power. To do so, you'll subclass the appropriate algorithm and override the ``iterations`` parameters. For example, to increase the number of iterations used by the -default PDKDF2 algorithm: +default PBKDF2 algorithm: 1. Create a subclass of ``django.contrib.auth.hashers.PBKDF2PasswordHasher``:: From e89bc39935afc5096e6a51a49874b2d30cbc2b5e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 21 Aug 2012 08:44:16 +0200 Subject: [PATCH 09/82] Reverted type check added in 62954ba04c. Refs #17040. --- django/utils/crypto.py | 6 ++---- tests/regressiontests/utils/crypto.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/django/utils/crypto.py b/django/utils/crypto.py index e8fd2b1e2d..1fdcc3082e 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -82,16 +82,14 @@ def get_random_string(length=12, def constant_time_compare(val1, val2): """ - Returns True if the two bytestrings are equal, False otherwise. + Returns True if the two strings are equal, False otherwise. The time taken is independent of the number of characters that match. """ - if not (isinstance(val1, bytes) and isinstance(val2, bytes)): - raise TypeError("constant_time_compare only supports bytes") if len(val1) != len(val2): return False result = 0 - if six.PY3: + if six.PY3 and isinstance(val1, bytes) and isinstance(val2, bytes): for x, y in zip(val1, val2): result |= x ^ y else: diff --git a/tests/regressiontests/utils/crypto.py b/tests/regressiontests/utils/crypto.py index 447b2e5b80..4c6b722ca9 100644 --- a/tests/regressiontests/utils/crypto.py +++ b/tests/regressiontests/utils/crypto.py @@ -15,8 +15,8 @@ class TestUtilsCryptoMisc(unittest.TestCase): # It's hard to test for constant time, just test the result. self.assertTrue(constant_time_compare(b'spam', b'spam')) self.assertFalse(constant_time_compare(b'spam', b'eggs')) - with self.assertRaises(TypeError): - constant_time_compare('spam', 'spam') + self.assertTrue(constant_time_compare('spam', 'spam')) + self.assertFalse(constant_time_compare('spam', 'eggs')) class TestUtilsCryptoPBKDF2(unittest.TestCase): From ab2f65bb7f9083b5fac777f5e3b7872c58837f4f Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 21 Aug 2012 11:20:22 +0200 Subject: [PATCH 10/82] Removed obsolete __members__ definitions This was useful for pre-Python 2.6 support. See commit c6e8e5d9. --- django/conf/__init__.py | 3 --- django/utils/functional.py | 1 - 2 files changed, 4 deletions(-) diff --git a/django/conf/__init__.py b/django/conf/__init__.py index e1c3fd1808..35f4ec8397 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -160,9 +160,6 @@ class UserSettingsHolder(BaseSettings): def __dir__(self): return list(self.__dict__) + dir(self.default_settings) - # For Python < 2.6: - __members__ = property(lambda self: self.__dir__()) - settings = LazySettings() diff --git a/django/utils/functional.py b/django/utils/functional.py index 085ec40b63..085a8fce59 100644 --- a/django/utils/functional.py +++ b/django/utils/functional.py @@ -238,7 +238,6 @@ class LazyObject(object): raise NotImplementedError # introspection support: - __members__ = property(lambda self: self.__dir__()) __dir__ = new_method_proxy(dir) From fd58d6c258058f8ac54e241b21b949bdbe50059b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 21 Aug 2012 14:43:37 +0200 Subject: [PATCH 11/82] Merge walk and find_files in makemessages command --- .../core/management/commands/makemessages.py | 53 ++++++------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 7bdd2472d3..81c4fdf8cc 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -47,31 +47,27 @@ def _popen(cmd): output, errors = p.communicate() return output, errors, p.returncode -def walk(root, topdown=True, onerror=None, followlinks=False, - ignore_patterns=None, verbosity=0, stdout=sys.stdout): +def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False): """ - A version of os.walk that can follow symlinks for Python < 2.6 + Helper function to get all files in the given root. """ - if ignore_patterns is None: - ignore_patterns = [] dir_suffix = '%s*' % os.sep norm_patterns = [p[:-len(dir_suffix)] if p.endswith(dir_suffix) else p for p in ignore_patterns] - for dirpath, dirnames, filenames in os.walk(root, topdown, onerror): - remove_dirs = [] - for dirname in dirnames: + all_files = [] + for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=symlinks): + for dirname in dirnames[:]: if is_ignored(os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns): - remove_dirs.append(dirname) - for dirname in remove_dirs: - dirnames.remove(dirname) - if verbosity > 1: - stdout.write('ignoring directory %s\n' % dirname) - yield (dirpath, dirnames, filenames) - if followlinks: - for d in dirnames: - p = os.path.join(dirpath, d) - if os.path.islink(p): - for link_dirpath, link_dirnames, link_filenames in walk(p): - yield (link_dirpath, link_dirnames, link_filenames) + dirnames.remove(dirname) + if verbosity > 1: + stdout.write('ignoring directory %s\n' % dirname) + for filename in filenames: + if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), ignore_patterns): + if verbosity > 1: + stdout.write('ignoring file %s in %s\n' % (filename, dirpath)) + else: + all_files.extend([(dirpath, filename)]) + all_files.sort() + return all_files def is_ignored(path, ignore_patterns): """ @@ -82,23 +78,6 @@ def is_ignored(path, ignore_patterns): return True return False -def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False): - """ - Helper function to get all files in the given root. - """ - all_files = [] - for (dirpath, dirnames, filenames) in walk(root, followlinks=symlinks, - ignore_patterns=ignore_patterns, verbosity=verbosity, stdout=stdout): - for filename in filenames: - norm_filepath = os.path.normpath(os.path.join(dirpath, filename)) - if is_ignored(norm_filepath, ignore_patterns): - if verbosity > 1: - stdout.write('ignoring file %s in %s\n' % (filename, dirpath)) - else: - all_files.extend([(dirpath, filename)]) - all_files.sort() - return all_files - def copy_plural_forms(msgs, locale, domain, verbosity, stdout=sys.stdout): """ Copies plural forms header contents from a Django catalog of locale to From 32ffcb21a00e08eb478cc287bc3254ee765d815d Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Tue, 21 Aug 2012 09:01:11 -0400 Subject: [PATCH 12/82] Fixed #17069 -- Added log filter example to docs. Added an example of filtering admin error emails (to exclude UnreadablePostErrors) to the docs. --- docs/topics/logging.txt | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/topics/logging.txt b/docs/topics/logging.txt index 9bdd706355..b54a9475ae 100644 --- a/docs/topics/logging.txt +++ b/docs/topics/logging.txt @@ -516,6 +516,35 @@ logging module. through the filter. Handling of that record will not proceed if the callback returns False. + For instance, to filter out :class:`~django.http.UnreadablePostError` + (raised when a user cancels an upload) from the admin emails, you would + create a filter function:: + + from django.http import UnreadablePostError + + def skip_unreadable_post(record): + if record.exc_info: + exc_type, exc_value = record.exc_info[:2] + if isinstance(exc_value, UnreadablePostError): + return False + return True + + and then add it to your logging config:: + + 'filters': { + 'skip_unreadable_posts': { + '()': 'django.utils.log.CallbackFilter', + 'callback': skip_unreadable_post, + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['skip_unreadable_posts'], + 'class': 'django.utils.log.AdminEmailHandler' + } + }, + .. class:: RequireDebugFalse() .. versionadded:: 1.4 From a193372753ad9d1d15ad5e2d6d06bbc07ca3f433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Tue, 21 Aug 2012 18:43:08 +0300 Subject: [PATCH 13/82] Fixed #17886 -- Fixed join promotion in ORed nullable queries The ORM generated a query with INNER JOIN instead of LEFT OUTER JOIN in a somewhat complicated case. The main issue was that there was a chain of nullable FK -> non-nullble FK, and the join promotion logic didn't see the need to promote the non-nullable FK even if the previous nullable FK was already promoted to LOUTER JOIN. This resulted in a query like a LOUTER b INNER c, which incorrectly prunes results. --- django/db/models/sql/query.py | 7 ++++++- tests/regressiontests/queries/models.py | 15 +++++++++++++++ tests/regressiontests/queries/tests.py | 25 ++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index c62c9ac23e..27a4ac9ce5 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -910,7 +910,12 @@ class Query(object): # Not all tables need to be joined to anything. No join type # means the later columns are ignored. join_type = None - elif promote or outer_if_first: + elif (promote or outer_if_first + or self.alias_map[lhs].join_type == self.LOUTER): + # We need to use LOUTER join if asked by promote or outer_if_first, + # or if the LHS table is left-joined in the query. Adding inner join + # to an existing outer join effectively cancels the effect of the + # outer join. join_type = self.LOUTER else: join_type = self.INNER diff --git a/tests/regressiontests/queries/models.py b/tests/regressiontests/queries/models.py index 45a48ee77c..f0178a0256 100644 --- a/tests/regressiontests/queries/models.py +++ b/tests/regressiontests/queries/models.py @@ -385,3 +385,18 @@ class NullableName(models.Model): class Meta: ordering = ['id'] + +class ModelD(models.Model): + name = models.TextField() + +class ModelC(models.Model): + name = models.TextField() + +class ModelB(models.Model): + name = models.TextField() + c = models.ForeignKey(ModelC) + +class ModelA(models.Model): + name = models.TextField() + b = models.ForeignKey(ModelB, null=True) + d = models.ForeignKey(ModelD) diff --git a/tests/regressiontests/queries/tests.py b/tests/regressiontests/queries/tests.py index 85ea4aa452..005aa9650b 100644 --- a/tests/regressiontests/queries/tests.py +++ b/tests/regressiontests/queries/tests.py @@ -23,7 +23,7 @@ from .models import (Annotation, Article, Author, Celebrity, Child, Cover, Ranking, Related, Report, ReservedName, Tag, TvChef, Valid, X, Food, Eaten, Node, ObjectA, ObjectB, ObjectC, CategoryItem, SimpleCategory, SpecialCategory, OneToOneCategory, NullableName, ProxyCategory, - SingleObject, RelatedObject) + SingleObject, RelatedObject, ModelA, ModelD) class BaseQuerysetTest(TestCase): @@ -2105,3 +2105,26 @@ class WhereNodeTest(TestCase): self.assertEqual(w.as_sql(qn, connection), (None, [])) w = WhereNode(children=[empty_w, NothingNode()], connector='OR') self.assertRaises(EmptyResultSet, w.as_sql, qn, connection) + +class NullJoinPromotionOrTest(TestCase): + def setUp(self): + d = ModelD.objects.create(name='foo') + ModelA.objects.create(name='bar', d=d) + + def test_ticket_17886(self): + # The first Q-object is generating the match, the rest of the filters + # should not remove the match even if they do not match anything. The + # problem here was that b__name generates a LOUTER JOIN, then + # b__c__name generates join to c, which the ORM tried to promote but + # failed as that join isn't nullable. + q_obj = ( + Q(d__name='foo')| + Q(b__name='foo')| + Q(b__c__name='foo') + ) + qset = ModelA.objects.filter(q_obj) + self.assertEqual(len(qset), 1) + # We generate one INNER JOIN to D. The join is direct and not nullable + # so we can use INNER JOIN for it. However, we can NOT use INNER JOIN + # for the b->c join, as a->b is nullable. + self.assertEqual(str(qset.query).count('INNER JOIN'), 1) From 4353a6163c00d4193307dae533e3e6ae9b44d18f Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 21 Aug 2012 21:52:25 +0200 Subject: [PATCH 14/82] Fixed #18196 -- Improved loaddata error messages. --- django/core/management/commands/loaddata.py | 12 +++++++++--- django/test/signals.py | 9 ++++++++- tests/regressiontests/fixtures_regress/tests.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 1896e53cee..30cf740cdf 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -196,6 +196,10 @@ class Command(BaseCommand): loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True + except Exception as e: + if not isinstance(e, CommandError): + e.args = ("Problem installing fixture '%s': %s" % (full_path, e),) + raise finally: fixture.close() @@ -209,7 +213,11 @@ class Command(BaseCommand): # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in models] - connection.check_constraints(table_names=table_names) + try: + connection.check_constraints(table_names=table_names) + except Exception as e: + e.args = ("Problem installing fixtures: %s" % e,) + raise except (SystemExit, KeyboardInterrupt): raise @@ -217,8 +225,6 @@ class Command(BaseCommand): if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) - if not isinstance(e, CommandError): - e.args = ("Problem installing fixture '%s': %s" % (full_path, e),) raise # If we found even one object in a fixture, we need to reset the diff --git a/django/test/signals.py b/django/test/signals.py index 052b7dfa5c..5b0a9a19ca 100644 --- a/django/test/signals.py +++ b/django/test/signals.py @@ -4,7 +4,6 @@ import time from django.conf import settings from django.db import connections from django.dispatch import receiver, Signal -from django.template import context from django.utils import timezone template_rendered = Signal(providing_args=["template", "context"]) @@ -48,9 +47,17 @@ def update_connections_time_zone(**kwargs): @receiver(setting_changed) def clear_context_processors_cache(**kwargs): if kwargs['setting'] == 'TEMPLATE_CONTEXT_PROCESSORS': + from django.template import context context._standard_context_processors = None +@receiver(setting_changed) +def clear_serializers_cache(**kwargs): + if kwargs['setting'] == 'SERIALIZATION_MODULES': + from django.core import serializers + serializers._serializers = {} + + @receiver(setting_changed) def language_changed(**kwargs): if kwargs['setting'] in ('LOCALE_PATHS', 'LANGUAGE_CODE'): diff --git a/tests/regressiontests/fixtures_regress/tests.py b/tests/regressiontests/fixtures_regress/tests.py index 89d754c58e..b3d0f42465 100644 --- a/tests/regressiontests/fixtures_regress/tests.py +++ b/tests/regressiontests/fixtures_regress/tests.py @@ -126,6 +126,20 @@ class TestFixtures(TestCase): commit=False, ) + @override_settings(SERIALIZATION_MODULES={'unkn': 'unexistent.path'}) + def test_unimportable_serializer(self): + """ + Test that failing serializer import raises the proper error + """ + with self.assertRaisesRegexp(ImportError, + "No module named unexistent.path"): + management.call_command( + 'loaddata', + 'bad_fixture1.unkn', + verbosity=0, + commit=False, + ) + def test_invalid_data(self): """ Test for ticket #4371 -- Loading a fixture file with invalid data From 3fd89d99036696ba08dd2dd7e20a5b375f85d23b Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 21 Aug 2012 17:32:53 -0400 Subject: [PATCH 15/82] Fixed #14885 - Clarified that ModelForm cleaning may not fully complete if the form is invalid. Thanks Ben Sturmfels for the patch. --- docs/topics/forms/modelforms.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/forms/modelforms.txt b/docs/topics/forms/modelforms.txt index 0ca37745c7..8159c8850c 100644 --- a/docs/topics/forms/modelforms.txt +++ b/docs/topics/forms/modelforms.txt @@ -200,7 +200,9 @@ The first time you call ``is_valid()`` or access the ``errors`` attribute of a well as :ref:`model validation `. This has the side-effect of cleaning the model you pass to the ``ModelForm`` constructor. For instance, calling ``is_valid()`` on your form will convert any date fields on your model -to actual date objects. +to actual date objects. If form validation fails, only some of the updates +may be applied. For this reason, you'll probably want to avoid reusing the +model instance. The ``save()`` method From 13d47c3f338e1e9a5da943b97b5334c0523d2e2c Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 21 Aug 2012 16:29:16 -0400 Subject: [PATCH 16/82] Fixed #18637 - Updated some documentation for aspects of models that are ModelForm specific, not admin specific. Thanks Ben Sturmfels for the patch. --- docs/ref/models/fields.txt | 90 +++++++++++++++++++------------------- docs/topics/db/models.txt | 28 ++++++------ 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index a43163c5e9..29cde19c78 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -70,8 +70,8 @@ If ``True``, the field is allowed to be blank. Default is ``False``. Note that this is different than :attr:`~Field.null`. :attr:`~Field.null` is purely database-related, whereas :attr:`~Field.blank` is validation-related. If -a field has ``blank=True``, validation on Django's admin site will allow entry -of an empty value. If a field has ``blank=False``, the field will be required. +a field has ``blank=True``, form validation will allow entry of an empty value. +If a field has ``blank=False``, the field will be required. .. _field-choices: @@ -81,14 +81,11 @@ of an empty value. If a field has ``blank=False``, the field will be required. .. attribute:: Field.choices An iterable (e.g., a list or tuple) of 2-tuples to use as choices for this -field. +field. If this is given, the default form widget will be a select box with +these choices instead of the standard text field. -If this is given, Django's admin will use a select box instead of the standard -text field and will limit choices to the choices given. - -A choices list is an iterable of 2-tuples; the first element in each -tuple is the actual value to be stored, and the second element is the -human-readable name. For example:: +The first element in each tuple is the actual value to be stored, and the +second element is the human-readable name. For example:: YEAR_IN_SCHOOL_CHOICES = ( ('FR', 'Freshman'), @@ -203,8 +200,8 @@ callable it will be called every time a new object is created. .. attribute:: Field.editable -If ``False``, the field will not be editable in the admin or via forms -automatically generated from the model class. Default is ``True``. +If ``False``, the field will not be displayed in the admin or any other +:class:`~django.forms.ModelForm`. Default is ``True``. ``error_messages`` ------------------ @@ -224,11 +221,11 @@ the `Field types`_ section below. .. attribute:: Field.help_text -Extra "help" text to be displayed under the field on the object's admin form. -It's useful for documentation even if your object doesn't have an admin form. +Extra "help" text to be displayed with the form widget. It's useful for +documentation even if your field isn't used on a form. -Note that this value is *not* HTML-escaped when it's displayed in the admin -interface. This lets you include HTML in :attr:`~Field.help_text` if you so +Note that this value is *not* HTML-escaped in automatically-generated +forms. This lets you include HTML in :attr:`~Field.help_text` if you so desire. For example:: help_text="Please use the following format: YYYY-MM-DD." @@ -259,7 +256,7 @@ Only one primary key is allowed on an object. If ``True``, this field must be unique throughout the table. -This is enforced at the database level and at the Django admin-form level. If +This is enforced at the database level and by model validation. If you try to save a model with a duplicate value in a :attr:`~Field.unique` field, a :exc:`django.db.IntegrityError` will be raised by the model's :meth:`~django.db.models.Model.save` method. @@ -279,7 +276,7 @@ For example, if you have a field ``title`` that has ``unique_for_date="pub_date"``, then Django wouldn't allow the entry of two records with the same ``title`` and ``pub_date``. -This is enforced at the Django admin-form level but not at the database level. +This is enforced by model validation but not at the database level. ``unique_for_month`` -------------------- @@ -337,7 +334,7 @@ otherwise. See :ref:`automatic-primary-key-fields`. A 64 bit integer, much like an :class:`IntegerField` except that it is guaranteed to fit numbers from -9223372036854775808 to 9223372036854775807. The -admin represents this as an ```` (a single-line input). +default form widget for this field is a :class:`~django.forms.TextInput`. ``BooleanField`` @@ -347,7 +344,8 @@ admin represents this as an ```` (a single-line input). A true/false field. -The admin represents this as a checkbox. +The default form widget for this field is a +:class:`~django.forms.CheckboxInput`. If you need to accept :attr:`~Field.null` values then use :class:`NullBooleanField` instead. @@ -361,7 +359,7 @@ A string field, for small- to large-sized strings. For large amounts of text, use :class:`~django.db.models.TextField`. -The admin represents this as an ```` (a single-line input). +The default form widget for this field is a :class:`~django.forms.TextInput`. :class:`CharField` has one extra required argument: @@ -414,9 +412,10 @@ optional arguments: for creation of timestamps. Note that the current date is *always* used; it's not just a default value that you can override. -The admin represents this as an ```` with a JavaScript -calendar, and a shortcut for "Today". Includes an additional ``invalid_date`` -error message key. +The default form widget for this field is a +:class:`~django.forms.TextInput`. The admin adds a JavaScript calendar, +and a shortcut for "Today". Includes an additional ``invalid_date`` error +message key. .. note:: As currently implemented, setting ``auto_now`` or ``auto_now_add`` to @@ -431,8 +430,9 @@ error message key. A date and time, represented in Python by a ``datetime.datetime`` instance. Takes the same extra arguments as :class:`DateField`. -The admin represents this as two ```` fields, with -JavaScript shortcuts. +The default form widget for this field is a single +:class:`~django.forms.TextInput`. The admin uses two separate +:class:`~django.forms.TextInput` widgets with JavaScript shortcuts. ``DecimalField`` ---------------- @@ -461,7 +461,7 @@ decimal places:: models.DecimalField(..., max_digits=19, decimal_places=10) -The admin represents this as an ```` (a single-line input). +The default form widget for this field is a :class:`~django.forms.TextInput`. .. note:: @@ -539,8 +539,8 @@ Also has one optional argument: Optional. A storage object, which handles the storage and retrieval of your files. See :doc:`/topics/files` for details on how to provide this object. -The admin represents this field as an ```` (a file-upload -widget). +The default form widget for this field is a +:class:`~django.forms.widgets.FileInput`. Using a :class:`FileField` or an :class:`ImageField` (see below) in a model takes a few steps: @@ -725,7 +725,7 @@ can change the maximum length using the :attr:`~CharField.max_length` argument. A floating-point number represented in Python by a ``float`` instance. -The admin represents this as an ```` (a single-line input). +The default form widget for this field is a :class:`~django.forms.TextInput`. .. _floatfield_vs_decimalfield: @@ -776,16 +776,16 @@ length using the :attr:`~CharField.max_length` argument. .. class:: IntegerField([**options]) -An integer. The admin represents this as an ```` (a -single-line input). +An integer. The default form widget for this field is a +:class:`~django.forms.TextInput`. ``IPAddressField`` ------------------ .. class:: IPAddressField([**options]) -An IP address, in string format (e.g. "192.0.2.30"). The admin represents this -as an ```` (a single-line input). +An IP address, in string format (e.g. "192.0.2.30"). The default form widget +for this field is a :class:`~django.forms.TextInput`. ``GenericIPAddressField`` ------------------------- @@ -795,8 +795,8 @@ as an ```` (a single-line input). .. versionadded:: 1.4 An IPv4 or IPv6 address, in string format (e.g. ``192.0.2.30`` or -``2a02:42fe::4``). The admin represents this as an ```` -(a single-line input). +``2a02:42fe::4``). The default form widget for this field is a +:class:`~django.forms.TextInput`. The IPv6 address normalization follows :rfc:`4291#section-2.2` section 2.2, including using the IPv4 format suggested in paragraph 3 of that section, like @@ -823,8 +823,8 @@ are converted to lowercase. .. class:: NullBooleanField([**options]) Like a :class:`BooleanField`, but allows ``NULL`` as one of the options. Use -this instead of a :class:`BooleanField` with ``null=True``. The admin represents -this as a ``