From ecd855658960d32efeb9cd97cad1ed076520e696 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Thu, 28 Nov 2013 01:37:21 +0700 Subject: [PATCH 001/158] Fixed typo in release notes. --- docs/releases/1.7.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index f8cdcf1f3b..8d130f9585 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -389,7 +389,7 @@ Management Commands * The :djadmin:`runserver` command received several improvements: - # On BSD systems, including OS X, the development server will reload + * On BSD systems, including OS X, the development server will reload immediately when a file is changed. Previously, it polled the filesystem for changes every second. That caused a small delay before reloads and reduced battery life on laptops. From df6760f12c2f08287e2aa7b5ddee6e567ab83220 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 13 Nov 2013 07:38:03 -0500 Subject: [PATCH 002/158] Added a warning regarding risks in serving user uploaded media. Thanks Preston Holmes for the draft text. --- docs/ref/settings.txt | 6 ++++ docs/topics/http/file-uploads.txt | 6 ++++ docs/topics/security.txt | 50 ++++++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index b9a7fd1a81..c99a7e4347 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1481,6 +1481,12 @@ to a non-empty value. Example: ``"http://media.example.com/"`` +.. warning:: + + There are security risks if you are accepting uploaded content from + untrusted users! See the security guide's topic on + :ref:`user-uploaded-content-security` for mitigation details. + .. warning:: :setting:`MEDIA_URL` and :setting:`STATIC_URL` must have different diff --git a/docs/topics/http/file-uploads.txt b/docs/topics/http/file-uploads.txt index d88524ee20..94415d03c3 100644 --- a/docs/topics/http/file-uploads.txt +++ b/docs/topics/http/file-uploads.txt @@ -10,6 +10,12 @@ When Django handles a file upload, the file data ends up placed in `). This document explains how files are stored on disk and in memory, and how to customize the default behavior. +.. warning:: + + There are security risks if you are accepting uploaded content from + untrusted users! See the security guide's topic on + :ref:`user-uploaded-content-security` for mitigation details. + Basic file uploads ================== diff --git a/docs/topics/security.txt b/docs/topics/security.txt index 5200edb95f..1ae5ddf78e 100644 --- a/docs/topics/security.txt +++ b/docs/topics/security.txt @@ -203,6 +203,52 @@ be deployed such that untrusted users don't have access to any subdomains, :mod:`django.contrib.sessions` also has limitations. See :ref:`the session topic guide section on security ` for details. +.. _user-uploaded-content-security: + +User-uploaded content +===================== + +.. note:: + Consider :ref:`serving static files from a cloud service or CDN + ` to avoid some of these issues. + +* If your site accepts file uploads, it is strongly advised that you limit + these uploads in your Web server configuration to a reasonable + size in order to prevent denial of service (DOS) attacks. In Apache, this + can be easily set using the LimitRequestBody_ directive. + +* If you are serving your own static files, be sure that handlers like Apache's + ``mod_php``, which would execute static files as code, are disabled. You don't + want users to be able to execute arbitrary code by uploading and requesting a + specially crafted file. + +* Django's media upload handling poses some vulnerabilities when that media is + served in ways that do not follow security best practices. Specifically, an + HTML file can be uploaded as an image if that file contains a valid PNG + header followed by malicious HTML. This file will pass verification of the + libraries that Django uses for :class:`~django.db.models.ImageField` image + processing (PIL or Pillow). When this file is subsequently displayed to a + user, it may be displayed as HTML depending on the type and configuration of + your web server. + + No bulletproof technical solution exists at the framework level to safely + validate all user uploaded file content, however, there are some other steps + you can take to mitigate these attacks: + + 1. One class of attacks can be prevented by always serving user uploaded + content from a distinct Top Level Domain (TLD). This prevents any + exploit blocked by `same-origin policy`_ protections such as cross site + scripting. For example, if your site runs on ``example.com``, you would + want to serve uploaded content (the :setting:`MEDIA_URL` setting) from + something like ``usercontent-example.com``. It's *not* sufficient to + serve content from a subdomain like ``usercontent.example.com``. + + 2. Beyond this, applications may choose to define a whitelist of allowable + file extensions for user uploaded files and configure the web server + to only serve such files. + +.. _same-origin policy: http://en.wikipedia.org/wiki/Same-origin_policy + .. _additional-security-topics: Additional security topics @@ -219,10 +265,6 @@ security protection of the Web server, operating system and other components. * Django does not throttle requests to authenticate users. To protect against brute-force attacks against the authentication system, you may consider deploying a Django plugin or Web server module to throttle these requests. -* If your site accepts file uploads, it is strongly advised that you limit - these uploads in your Web server configuration to a reasonable - size in order to prevent denial of service (DOS) attacks. In Apache, this - can be easily set using the LimitRequestBody_ directive. * Keep your :setting:`SECRET_KEY` a secret. * It is a good idea to limit the accessibility of your caching system and database using a firewall. From a5b94d9d4ea7f85f68bbbb7520d0d5475fbb846c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 27 Nov 2013 16:42:39 -0600 Subject: [PATCH 003/158] A handle of flake8 fixes --- django/contrib/gis/db/backends/postgis/schema.py | 2 -- django/contrib/gis/db/models/fields.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/django/contrib/gis/db/backends/postgis/schema.py b/django/contrib/gis/db/backends/postgis/schema.py index 46e438acaa..181826789b 100644 --- a/django/contrib/gis/db/backends/postgis/schema.py +++ b/django/contrib/gis/db/backends/postgis/schema.py @@ -1,6 +1,4 @@ -from django.conf import settings from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor -from django.utils.functional import cached_property class PostGISSchemaEditor(DatabaseSchemaEditor): diff --git a/django/contrib/gis/db/models/fields.py b/django/contrib/gis/db/models/fields.py index b5cdc72c29..734f805c8d 100644 --- a/django/contrib/gis/db/models/fields.py +++ b/django/contrib/gis/db/models/fields.py @@ -112,9 +112,9 @@ class GeometryField(Field): kwargs['srid'] = self.srid if self.dim != 2: kwargs['dim'] = self.dim - if self.spatial_index != True: + if self.spatial_index is not True: kwargs['spatial_index'] = self.spatial_index - if self.geography != False: + if self.geography is not False: kwargs['geography'] = self.geography return name, path, args, kwargs From 077af42139db84d88f293ab5eadc989a9169dce1 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Thu, 28 Nov 2013 00:53:10 +0100 Subject: [PATCH 004/158] Fixed #21515 -- Corrected example of template.Context in documentation. Thanks to trac user oubiga for the report. --- docs/ref/templates/api.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ref/templates/api.txt b/docs/ref/templates/api.txt index b3e49b6773..6c22017943 100644 --- a/docs/ref/templates/api.txt +++ b/docs/ref/templates/api.txt @@ -306,10 +306,12 @@ If you ``pop()`` too much, it'll raise >>> c = Context() >>> c['foo'] = 'first level' >>> c.push() + {} >>> c['foo'] = 'second level' >>> c['foo'] 'second level' >>> c.pop() + {'foo': 'second level'} >>> c['foo'] 'first level' >>> c['foo'] = 'overwritten' From d1df395f3ae768e495a105db2f85352c44ba1c28 Mon Sep 17 00:00:00 2001 From: Vajrasky Kok Date: Wed, 27 Nov 2013 23:01:18 +0800 Subject: [PATCH 005/158] Fixed #21517 -- Added unit test for non-autoincrement primary key with value 0. --- tests/custom_pk/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/custom_pk/tests.py b/tests/custom_pk/tests.py index 22369747a9..449e724acc 100644 --- a/tests/custom_pk/tests.py +++ b/tests/custom_pk/tests.py @@ -153,6 +153,13 @@ class CustomPKTests(TestCase): with transaction.atomic(): Employee.objects.create(employee_code=123, first_name="Fred", last_name="Jones") + def test_zero_non_autoincrement_pk(self): + Employee.objects.create( + employee_code=0, first_name="Frank", last_name="Jones" + ) + employee = Employee.objects.get(pk=0) + self.assertEqual(employee.employee_code, 0) + def test_custom_field_pk(self): # Regression for #10785 -- Custom fields can be used for primary keys. new_bar = Bar.objects.create() From 7477a4ffde4781f4e84503e66d7f775074089887 Mon Sep 17 00:00:00 2001 From: Christopher Medrela Date: Tue, 26 Nov 2013 10:43:46 +0100 Subject: [PATCH 006/158] Fixed E125 pep8 warnings --- django/contrib/admin/options.py | 2 +- django/contrib/admin/sites.py | 2 +- django/contrib/admin/validation.py | 2 +- django/contrib/auth/decorators.py | 2 +- django/contrib/flatpages/forms.py | 4 ++-- django/contrib/formtools/wizard/views.py | 2 +- django/contrib/gis/db/backends/postgis/operations.py | 4 ++-- django/contrib/gis/db/models/sql/where.py | 2 +- django/contrib/gis/models.py | 2 +- django/contrib/gis/utils/layermapping.py | 2 +- django/contrib/sessions/tests.py | 2 +- django/contrib/sitemaps/__init__.py | 2 +- django/db/backends/__init__.py | 2 +- django/db/backends/mysql/compiler.py | 2 +- django/db/models/deletion.py | 4 ++-- django/db/models/fields/__init__.py | 2 +- django/db/models/query.py | 4 ++-- django/db/models/sql/query.py | 8 ++++---- django/db/models/sql/subqueries.py | 2 +- django/db/models/sql/where.py | 4 ++-- django/forms/fields.py | 4 ++-- django/forms/formsets.py | 6 +++--- django/forms/models.py | 6 +++--- django/http/request.py | 2 +- django/middleware/common.py | 2 +- django/template/defaultfilters.py | 2 +- django/template/loader_tags.py | 2 +- django/test/runner.py | 2 +- django/test/testcases.py | 8 ++++---- django/utils/_os.py | 4 ++-- django/utils/feedgenerator.py | 6 +++--- django/utils/html.py | 2 +- django/views/debug.py | 6 +++--- django/views/generic/list.py | 2 +- extras/csrf_migration_helper.py | 4 ++-- tests/defer/tests.py | 2 +- tests/runtests.py | 10 +++++----- tests/wsgi/tests.py | 8 ++++---- 38 files changed, 67 insertions(+), 67 deletions(-) diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index 2f39c6a896..51e8a1b2e6 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -613,7 +613,7 @@ class ModelAdmin(BaseModelAdmin): } defaults.update(kwargs) if (defaults.get('fields') is None - and not modelform_defines_fields(defaults.get('form'))): + and not modelform_defines_fields(defaults.get('form'))): defaults['fields'] = forms.ALL_FIELDS return modelform_factory(self.model, **defaults) diff --git a/django/contrib/admin/sites.py b/django/contrib/admin/sites.py index 2dac947fbc..e620154312 100644 --- a/django/contrib/admin/sites.py +++ b/django/contrib/admin/sites.py @@ -169,7 +169,7 @@ class AdminSite(object): raise ImproperlyConfigured("Put 'django.contrib.contenttypes' in " "your INSTALLED_APPS setting in order to use the admin application.") if not ('django.contrib.auth.context_processors.auth' in settings.TEMPLATE_CONTEXT_PROCESSORS or - 'django.core.context_processors.auth' in settings.TEMPLATE_CONTEXT_PROCESSORS): + 'django.core.context_processors.auth' in settings.TEMPLATE_CONTEXT_PROCESSORS): raise ImproperlyConfigured("Put 'django.contrib.auth.context_processors.auth' " "in your TEMPLATE_CONTEXT_PROCESSORS setting in order to use the admin application.") diff --git a/django/contrib/admin/validation.py b/django/contrib/admin/validation.py index b2cb0b4181..9079678639 100644 --- a/django/contrib/admin/validation.py +++ b/django/contrib/admin/validation.py @@ -155,7 +155,7 @@ class BaseValidator(object): for field, val in cls.prepopulated_fields.items(): f = get_field(cls, model, 'prepopulated_fields', field) if isinstance(f, (models.DateTimeField, models.ForeignKey, - models.ManyToManyField)): + models.ManyToManyField)): raise ImproperlyConfigured("'%s.prepopulated_fields['%s']' " "is either a DateTimeField, ForeignKey or " "ManyToManyField. This isn't allowed." diff --git a/django/contrib/auth/decorators.py b/django/contrib/auth/decorators.py index 4935213d0b..6a976b48dc 100644 --- a/django/contrib/auth/decorators.py +++ b/django/contrib/auth/decorators.py @@ -29,7 +29,7 @@ def user_passes_test(test_func, login_url=None, redirect_field_name=REDIRECT_FIE login_scheme, login_netloc = urlparse(resolved_login_url)[:2] current_scheme, current_netloc = urlparse(path)[:2] if ((not login_scheme or login_scheme == current_scheme) and - (not login_netloc or login_netloc == current_netloc)): + (not login_netloc or login_netloc == current_netloc)): path = request.get_full_path() from django.contrib.auth.views import redirect_to_login return redirect_to_login( diff --git a/django/contrib/flatpages/forms.py b/django/contrib/flatpages/forms.py index 7e3a39e88d..8d85922d9d 100644 --- a/django/contrib/flatpages/forms.py +++ b/django/contrib/flatpages/forms.py @@ -23,8 +23,8 @@ class FlatpageForm(forms.ModelForm): code='missing_leading_slash', ) if (settings.APPEND_SLASH and - 'django.middleware.common.CommonMiddleware' in settings.MIDDLEWARE_CLASSES and - not url.endswith('/')): + 'django.middleware.common.CommonMiddleware' in settings.MIDDLEWARE_CLASSES and + not url.endswith('/')): raise forms.ValidationError( ugettext("URL is missing a trailing slash."), code='missing_trailing_slash', diff --git a/django/contrib/formtools/wizard/views.py b/django/contrib/formtools/wizard/views.py index f19cfc76f3..2b4a86d115 100644 --- a/django/contrib/formtools/wizard/views.py +++ b/django/contrib/formtools/wizard/views.py @@ -123,7 +123,7 @@ class WizardView(TemplateView): @classmethod def get_initkwargs(cls, form_list=None, initial_dict=None, - instance_dict=None, condition_dict=None, *args, **kwargs): + instance_dict=None, condition_dict=None, *args, **kwargs): """ Creates a dict with all needed parameters for the form wizard instances. diff --git a/django/contrib/gis/db/backends/postgis/operations.py b/django/contrib/gis/db/backends/postgis/operations.py index 86608afa35..40ca09f0fa 100644 --- a/django/contrib/gis/db/backends/postgis/operations.py +++ b/django/contrib/gis/db/backends/postgis/operations.py @@ -369,7 +369,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): dist_param = value if (not geography and geodetic and lookup_type != 'dwithin' - and option == 'spheroid'): + and option == 'spheroid'): # using distance_spheroid requires the spheroid of the field as # a parameter. return [f._spheroid, dist_param] @@ -467,7 +467,7 @@ class PostGISOperations(DatabaseOperations, BaseSpatialOperations): def two_to_three(np): return np >= 2 and np <= 3 if (lookup_type in self.distance_functions and - lookup_type != 'dwithin'): + lookup_type != 'dwithin'): return two_to_three(num_param) else: return exactly_two(num_param) diff --git a/django/contrib/gis/db/models/sql/where.py b/django/contrib/gis/db/models/sql/where.py index 3a504274b6..1e750ebd89 100644 --- a/django/contrib/gis/db/models/sql/where.py +++ b/django/contrib/gis/db/models/sql/where.py @@ -39,7 +39,7 @@ class GeoWhereNode(WhereNode): if isinstance(data, (list, tuple)): obj, lookup_type, value = data if (isinstance(obj, Constraint) and - isinstance(obj.field, GeometryField)): + isinstance(obj.field, GeometryField)): data = (GeoConstraint(obj), lookup_type, value) return super(GeoWhereNode, self)._prepare_data(data) diff --git a/django/contrib/gis/models.py b/django/contrib/gis/models.py index e379e82a7b..466efcee9a 100644 --- a/django/contrib/gis/models.py +++ b/django/contrib/gis/models.py @@ -1,7 +1,7 @@ from django.db import connection if (hasattr(connection.ops, 'spatial_version') and - not connection.ops.mysql): + not connection.ops.mysql): # Getting the `SpatialRefSys` and `GeometryColumns` # models for the default spatial backend. These # aliases are provided for backwards-compatibility. diff --git a/django/contrib/gis/utils/layermapping.py b/django/contrib/gis/utils/layermapping.py index 3ba087b967..c834cb03b0 100644 --- a/django/contrib/gis/utils/layermapping.py +++ b/django/contrib/gis/utils/layermapping.py @@ -339,7 +339,7 @@ class LayerMapping(object): otherwise the proper exception is raised. """ if (isinstance(ogr_field, OFTString) and - isinstance(model_field, (models.CharField, models.TextField))): + isinstance(model_field, (models.CharField, models.TextField))): if self.encoding: # The encoding for OGR data sources may be specified here # (e.g., 'cp437' for Census Bureau boundary files). diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index af89f0048e..66af3015bb 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -141,7 +141,7 @@ class SessionTestsMixin(object): def test_save(self): if (hasattr(self.session, '_cache') and 'DummyCache' in - settings.CACHES[settings.SESSION_CACHE_ALIAS]['BACKEND']): + settings.CACHES[settings.SESSION_CACHE_ALIAS]['BACKEND']): raise unittest.SkipTest("Session saving tests require a real cache backend") self.session.save() self.assertTrue(self.session.exists(self.session.session_key)) diff --git a/django/contrib/sitemaps/__init__.py b/django/contrib/sitemaps/__init__.py index db31043bfa..4396708098 100644 --- a/django/contrib/sitemaps/__init__.py +++ b/django/contrib/sitemaps/__init__.py @@ -94,7 +94,7 @@ class Sitemap(object): if all_items_lastmod: all_items_lastmod = lastmod is not None if (all_items_lastmod and - (latest_lastmod is None or lastmod > latest_lastmod)): + (latest_lastmod is None or lastmod > latest_lastmod)): latest_lastmod = lastmod url_info = { 'item': item, diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index a23751caf5..a2a1757ad3 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -153,7 +153,7 @@ class BaseDatabaseWrapper(object): """ self.validate_thread_sharing() if (self.use_debug_cursor or - (self.use_debug_cursor is None and settings.DEBUG)): + (self.use_debug_cursor is None and settings.DEBUG)): cursor = self.make_debug_cursor(self._cursor()) else: cursor = utils.CursorWrapper(self._cursor(), self) diff --git a/django/db/backends/mysql/compiler.py b/django/db/backends/mysql/compiler.py index 609573442c..85c045fff2 100644 --- a/django/db/backends/mysql/compiler.py +++ b/django/db/backends/mysql/compiler.py @@ -8,7 +8,7 @@ class SQLCompiler(compiler.SQLCompiler): index_extra_select = len(self.query.extra_select) for value, field in zip_longest(row[index_extra_select:], fields): if (field and field.get_internal_type() in ("BooleanField", "NullBooleanField") and - value in (0, 1)): + value in (0, 1)): value = bool(value) values.append(value) return row[:index_extra_select] + tuple(values) diff --git a/django/db/models/deletion.py b/django/db/models/deletion.py index 90e515cce5..8b48815a0d 100644 --- a/django/db/models/deletion.py +++ b/django/db/models/deletion.py @@ -135,7 +135,7 @@ class Collector(object): # Foreign keys pointing to this model, both from m2m and other # models. for related in opts.get_all_related_objects( - include_hidden=True, include_proxy_eq=True): + include_hidden=True, include_proxy_eq=True): if related.field.rel.on_delete is not DO_NOTHING: return False # GFK deletes @@ -145,7 +145,7 @@ class Collector(object): return True def collect(self, objs, source=None, nullable=False, collect_related=True, - source_attr=None, reverse_dependency=False): + source_attr=None, reverse_dependency=False): """ Adds 'objs' to the collection of objects to be deleted as well as all parent instances. 'objs' must be a homogenous iterable collection of diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index dec2e2dd9e..1fa230e1c2 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -1281,7 +1281,7 @@ class IntegerField(Field): def get_prep_lookup(self, lookup_type, value): if ((lookup_type == 'gte' or lookup_type == 'lt') - and isinstance(value, float)): + and isinstance(value, float)): value = math.ceil(value) return super(IntegerField, self).get_prep_lookup(lookup_type, value) diff --git a/django/db/models/query.py b/django/db/models/query.py index 1737626d16..58180e1cf9 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -392,7 +392,7 @@ class QuerySet(object): fields = self.model._meta.local_fields with transaction.commit_on_success_unless_managed(using=self.db): if (connection.features.can_combine_inserts_with_and_without_auto_increment_pk - and self.model._meta.has_auto_field): + and self.model._meta.has_auto_field): self._batched_insert(objs, fields, batch_size) else: objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs) @@ -1494,7 +1494,7 @@ class RawQuerySet(object): annotated model instances. """ def __init__(self, raw_query, model=None, query=None, params=None, - translations=None, using=None, hints=None): + translations=None, using=None, hints=None): self.raw_query = raw_query self.model = model self._db = using diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index e0593ef0a7..5f24e488cb 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -994,9 +994,9 @@ class Query(object): raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % ( aggregate.name, field_name, field_name)) elif ((len(field_list) > 1) or - (field_list[0] not in [i.name for i in opts.fields]) or - self.group_by is None or - not is_summary): + (field_list[0] not in [i.name for i in opts.fields]) or + self.group_by is None or + not is_summary): # If: # - the field descriptor has more than one part (foo__bar), or # - the field descriptor is referencing an m2m/m2o field, or @@ -1906,7 +1906,7 @@ class Query(object): # is_nullable() is needed to the compiler stage, but that is not easy # to do currently. if ((connections[DEFAULT_DB_ALIAS].features.interprets_empty_strings_as_nulls) - and field.empty_strings_allowed): + and field.empty_strings_allowed): return True else: return field.null diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py index 79db1ad99a..e9e292e787 100644 --- a/django/db/models/sql/subqueries.py +++ b/django/db/models/sql/subqueries.py @@ -61,7 +61,7 @@ class DeleteQuery(Query): innerq_used_tables = [t for t in innerq.tables if innerq.alias_refcount[t]] if ((not innerq_used_tables or innerq_used_tables == self.tables) - and not len(innerq.having)): + and not len(innerq.having)): # There is only the base table in use in the query, and there is # no aggregate filtering going on. self.where = innerq.where diff --git a/django/db/models/sql/where.py b/django/db/models/sql/where.py index 7b71580370..44a4ce9d1d 100644 --- a/django/db/models/sql/where.py +++ b/django/db/models/sql/where.py @@ -69,7 +69,7 @@ class WhereNode(tree.Node): # and empty values need special handling. Other types could be used # here in the future (using Python types is suggested for consistency). if (isinstance(value, datetime.datetime) - or (isinstance(obj.field, DateTimeField) and lookup_type != 'isnull')): + or (isinstance(obj.field, DateTimeField) and lookup_type != 'isnull')): value_annotation = datetime.datetime elif hasattr(value, 'value_annotation'): value_annotation = value.value_annotation @@ -207,7 +207,7 @@ class WhereNode(tree.Node): params = field_params + params if (len(params) == 1 and params[0] == '' and lookup_type == 'exact' - and connection.features.interprets_empty_strings_as_nulls): + and connection.features.interprets_empty_strings_as_nulls): lookup_type = 'isnull' value_annotation = True diff --git a/django/forms/fields.py b/django/forms/fields.py index 56d0e316f2..6f88fb2662 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -1102,8 +1102,8 @@ class FilePathField(ChoiceField): continue full_file = os.path.join(self.path, f) if (((self.allow_files and os.path.isfile(full_file)) or - (self.allow_folders and os.path.isdir(full_file))) and - (self.match is None or self.match_re.search(f))): + (self.allow_folders and os.path.isdir(full_file))) and + (self.match is None or self.match_re.search(f))): self.choices.append((full_file, f)) except OSError: pass diff --git a/django/forms/formsets.py b/django/forms/formsets.py index 3759d3381d..fd1fd92773 100644 --- a/django/forms/formsets.py +++ b/django/forms/formsets.py @@ -325,15 +325,15 @@ class BaseFormSet(object): self._errors.append(form.errors) try: if (self.validate_max and - self.total_form_count() - len(self.deleted_forms) > self.max_num) or \ - self.management_form.cleaned_data[TOTAL_FORM_COUNT] > self.absolute_max: + self.total_form_count() - len(self.deleted_forms) > self.max_num) or \ + self.management_form.cleaned_data[TOTAL_FORM_COUNT] > self.absolute_max: raise ValidationError(ungettext( "Please submit %d or fewer forms.", "Please submit %d or fewer forms.", self.max_num) % self.max_num, code='too_many_forms', ) if (self.validate_min and - self.total_form_count() - len(self.deleted_forms) < self.min_num): + self.total_form_count() - len(self.deleted_forms) < self.min_num): raise ValidationError(ungettext( "Please submit %d or more forms.", "Please submit %d or more forms.", self.min_num) % self.min_num, diff --git a/django/forms/models.py b/django/forms/models.py index e9bd338064..b14ea7a265 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -524,7 +524,7 @@ def modelform_factory(model, form=ModelForm, fields=None, exclude=None, # be difficult to debug for code that needs updating, so we produce the # warning here too. if (getattr(Meta, 'fields', None) is None and - getattr(Meta, 'exclude', None) is None): + getattr(Meta, 'exclude', None) is None): warnings.warn("Calling modelform_factory without defining 'fields' or " "'exclude' explicitly is deprecated", DeprecationWarning, stacklevel=2) @@ -675,7 +675,7 @@ class BaseModelFormSet(BaseFormSet): for form in valid_forms: # see if we have data for both fields if (form.cleaned_data and form.cleaned_data[field] is not None - and form.cleaned_data[unique_for] is not None): + and form.cleaned_data[unique_for] is not None): # if it's a date lookup we need to get the data for all the fields if lookup == 'date': date = form.cleaned_data[unique_for] @@ -815,7 +815,7 @@ def modelformset_factory(model, form=ModelForm, formfield_callback=None, if meta is None: meta = type(str('Meta'), (object,), {}) if (getattr(meta, 'fields', fields) is None and - getattr(meta, 'exclude', exclude) is None): + getattr(meta, 'exclude', exclude) is None): warnings.warn("Calling modelformset_factory without defining 'fields' or " "'exclude' explicitly is deprecated", DeprecationWarning, stacklevel=2) diff --git a/django/http/request.py b/django/http/request.py index 1aefe6bf22..6bce1f718d 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -66,7 +66,7 @@ class HttpRequest(object): """Returns the HTTP host using the environment or request headers.""" # We try three options, in order of decreasing preference. if settings.USE_X_FORWARDED_HOST and ( - 'HTTP_X_FORWARDED_HOST' in self.META): + 'HTTP_X_FORWARDED_HOST' in self.META): host = self.META['HTTP_X_FORWARDED_HOST'] elif 'HTTP_HOST' in self.META: host = self.META['HTTP_HOST'] diff --git a/django/middleware/common.py b/django/middleware/common.py index 51fcf1b2af..6ff8a8d6e7 100644 --- a/django/middleware/common.py +++ b/django/middleware/common.py @@ -122,7 +122,7 @@ class CommonMiddleware(object): etag = '"%s"' % hashlib.md5(response.content).hexdigest() if etag is not None: if (200 <= response.status_code < 300 - and request.META.get('HTTP_IF_NONE_MATCH') == etag): + and request.META.get('HTTP_IF_NONE_MATCH') == etag): cookies = response.cookies response = http.HttpResponseNotModified() response.cookies = cookies diff --git a/django/template/defaultfilters.py b/django/template/defaultfilters.py index 1bd54404b3..6dfff99bfc 100644 --- a/django/template/defaultfilters.py +++ b/django/template/defaultfilters.py @@ -40,7 +40,7 @@ def stringfilter(func): args = list(args) args[0] = force_text(args[0]) if (isinstance(args[0], SafeData) and - getattr(_dec._decorated_function, 'is_safe', False)): + getattr(_dec._decorated_function, 'is_safe', False)): return mark_safe(func(*args, **kwargs)) return func(*args, **kwargs) diff --git a/django/template/loader_tags.py b/django/template/loader_tags.py index 740b503cc5..31ba3475d3 100644 --- a/django/template/loader_tags.py +++ b/django/template/loader_tags.py @@ -70,7 +70,7 @@ class BlockNode(Node): def super(self): render_context = self.context.render_context if (BLOCK_CONTEXT_KEY in render_context and - render_context[BLOCK_CONTEXT_KEY].get_block(self.name) is not None): + render_context[BLOCK_CONTEXT_KEY].get_block(self.name) is not None): return mark_safe(self.render(self.context)) return '' diff --git a/django/test/runner.py b/django/test/runner.py index 44adaab141..21e4a81604 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -272,7 +272,7 @@ def setup_databases(verbosity, interactive, **kwargs): mirrors = [] for signature, (db_name, aliases) in dependency_ordered( - test_databases.items(), dependencies): + test_databases.items(), dependencies): test_db_name = None # Actually create the database for the first connection for alias in aliases: diff --git a/django/test/testcases.py b/django/test/testcases.py index bb40e5c4fd..64e2e3c591 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -298,7 +298,7 @@ class SimpleTestCase(unittest.TestCase): # If the response supports deferred rendering and hasn't been rendered # yet, then ensure that it does get rendered before proceeding further. if (hasattr(response, 'render') and callable(response.render) - and not response.is_rendered): + and not response.is_rendered): response.render() if msg_prefix: @@ -1043,7 +1043,7 @@ class LiveServerThread(threading.Thread): (self.host, port), QuietWSGIRequestHandler) except socket.error as e: if (index + 1 < len(self.possible_ports) and - e.errno == errno.EADDRINUSE): + e.errno == errno.EADDRINUSE): # This port is already in use, so we go on and try with # the next one in the list. continue @@ -1097,7 +1097,7 @@ class LiveServerTestCase(TransactionTestCase): # If using in-memory sqlite databases, pass the connections to # the server thread. if (conn.vendor == 'sqlite' - and conn.settings_dict['NAME'] == ':memory:'): + and conn.settings_dict['NAME'] == ':memory:'): # Explicitly enable thread-shareability for this connection conn.allow_thread_sharing = True connections_override[conn.alias] = conn @@ -1154,7 +1154,7 @@ class LiveServerTestCase(TransactionTestCase): # Restore sqlite connections' non-sharability for conn in connections.all(): if (conn.vendor == 'sqlite' - and conn.settings_dict['NAME'] == ':memory:'): + and conn.settings_dict['NAME'] == ':memory:'): conn.allow_thread_sharing = False @classmethod diff --git a/django/utils/_os.py b/django/utils/_os.py index 11fc5afe49..1cf250e09e 100644 --- a/django/utils/_os.py +++ b/django/utils/_os.py @@ -74,8 +74,8 @@ def safe_join(base, *paths): # b) The final path must be the same as the base path. # c) The base path must be the most root path (meaning either "/" or "C:\\") if (not normcase(final_path).startswith(normcase(base_path + sep)) and - normcase(final_path) != normcase(base_path) and - dirname(normcase(base_path)) != normcase(base_path)): + normcase(final_path) != normcase(base_path) and + dirname(normcase(base_path)) != normcase(base_path)): raise ValueError('The joined path (%s) is located outside of the base ' 'path component (%s)' % (final_path, base_path)) return final_path diff --git a/django/utils/feedgenerator.py b/django/utils/feedgenerator.py index 00d982025a..52c0ecca7f 100644 --- a/django/utils/feedgenerator.py +++ b/django/utils/feedgenerator.py @@ -113,9 +113,9 @@ class SyndicationFeed(object): self.items = [] def add_item(self, title, link, description, author_email=None, - author_name=None, author_link=None, pubdate=None, comments=None, - unique_id=None, unique_id_is_permalink=None, enclosure=None, - categories=(), item_copyright=None, ttl=None, updateddate=None, **kwargs): + author_name=None, author_link=None, pubdate=None, comments=None, + unique_id=None, unique_id_is_permalink=None, enclosure=None, + categories=(), item_copyright=None, ttl=None, updateddate=None, **kwargs): """ Adds an item to the feed. All args are expected to be Python Unicode objects except pubdate and updateddate, which are datetime.datetime diff --git a/django/utils/html.py b/django/utils/html.py index 3ad549de19..daced9a221 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -238,7 +238,7 @@ def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False): lead = lead + opening # Keep parentheses at the end only if they're balanced. if (middle.endswith(closing) - and middle.count(closing) == middle.count(opening) + 1): + and middle.count(closing) == middle.count(opening) + 1): middle = middle[:-len(closing)] trail = closing + trail diff --git a/django/views/debug.py b/django/views/debug.py index e3f399ef9d..66453fe2c6 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -189,7 +189,7 @@ class SafeExceptionReporterFilter(ExceptionReporterFilter): sensitive_variables = None while current_frame is not None: if (current_frame.f_code.co_name == 'sensitive_variables_wrapper' - and 'sensitive_variables_wrapper' in current_frame.f_locals): + and 'sensitive_variables_wrapper' in current_frame.f_locals): # The sensitive_variables decorator was used, so we take note # of the sensitive variables' names. wrapper = current_frame.f_locals['sensitive_variables_wrapper'] @@ -218,7 +218,7 @@ class SafeExceptionReporterFilter(ExceptionReporterFilter): cleansed[name] = self.cleanse_special_types(request, value) if (tb_frame.f_code.co_name == 'sensitive_variables_wrapper' - and 'sensitive_variables_wrapper' in tb_frame.f_locals): + and 'sensitive_variables_wrapper' in tb_frame.f_locals): # For good measure, obfuscate the decorated function's arguments in # the sensitive_variables decorator's frame, in case the variables # associated with those arguments were meant to be obfuscated from @@ -287,7 +287,7 @@ class ExceptionReporter(object): 'templates': template_list, }) if (settings.TEMPLATE_DEBUG and - hasattr(self.exc_value, 'django_template_source')): + hasattr(self.exc_value, 'django_template_source')): self.get_template_exception_info() frames = self.get_traceback_frames() diff --git a/django/views/generic/list.py b/django/views/generic/list.py index b2dfd38ece..0df46ec8f4 100644 --- a/django/views/generic/list.py +++ b/django/views/generic/list.py @@ -150,7 +150,7 @@ class BaseListView(MultipleObjectMixin, View): # it's better to do a cheap query than to load the unpaginated # queryset in memory. if (self.get_paginate_by(self.object_list) is not None - and hasattr(self.object_list, 'exists')): + and hasattr(self.object_list, 'exists')): is_empty = not self.object_list.exists() else: is_empty = len(self.object_list) == 0 diff --git a/extras/csrf_migration_helper.py b/extras/csrf_migration_helper.py index 38dee93735..9e7b04b85b 100755 --- a/extras/csrf_migration_helper.py +++ b/extras/csrf_migration_helper.py @@ -145,11 +145,11 @@ def get_template_dirs(): from django.conf import settings dirs = set() if ('django.template.loaders.filesystem.load_template_source' in settings.TEMPLATE_LOADERS - or 'django.template.loaders.filesystem.Loader' in settings.TEMPLATE_LOADERS): + or 'django.template.loaders.filesystem.Loader' in settings.TEMPLATE_LOADERS): dirs.update(map(unicode, settings.TEMPLATE_DIRS)) if ('django.template.loaders.app_directories.load_template_source' in settings.TEMPLATE_LOADERS - or 'django.template.loaders.app_directories.Loader' in settings.TEMPLATE_LOADERS): + or 'django.template.loaders.app_directories.Loader' in settings.TEMPLATE_LOADERS): from django.template.loaders.app_directories import app_template_dirs dirs.update(app_template_dirs) return dirs diff --git a/tests/defer/tests.py b/tests/defer/tests.py index b66b173f7a..266a851ffe 100644 --- a/tests/defer/tests.py +++ b/tests/defer/tests.py @@ -11,7 +11,7 @@ class DeferTests(TestCase): count = 0 for field in obj._meta.fields: if isinstance(obj.__class__.__dict__.get(field.attname), - DeferredAttribute): + DeferredAttribute): count += 1 self.assertEqual(count, num) diff --git a/tests/runtests.py b/tests/runtests.py index 08eaf6ed0e..f37c0e9dda 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -69,11 +69,11 @@ def get_test_modules(): for modpath, dirpath in discovery_paths: for f in os.listdir(dirpath): if ('.' in f or - # Python 3 byte code dirs (PEP 3147) - f == '__pycache__' or - f.startswith('sql') or - os.path.basename(f) in SUBDIRS_TO_SKIP or - os.path.isfile(f)): + # Python 3 byte code dirs (PEP 3147) + f == '__pycache__' or + f.startswith('sql') or + os.path.basename(f) in SUBDIRS_TO_SKIP or + os.path.isfile(f)): continue modules.append((modpath, f)) return modules diff --git a/tests/wsgi/tests.py b/tests/wsgi/tests.py index 57972d2e9a..90397bb742 100644 --- a/tests/wsgi/tests.py +++ b/tests/wsgi/tests.py @@ -93,15 +93,15 @@ class GetInternalWSGIApplicationTest(unittest.TestCase): @override_settings(WSGI_APPLICATION="wsgi.noexist.app") def test_bad_module(self): with six.assertRaisesRegex(self, - ImproperlyConfigured, - r"^WSGI application 'wsgi.noexist.app' could not be loaded; Error importing.*"): + ImproperlyConfigured, + r"^WSGI application 'wsgi.noexist.app' could not be loaded; Error importing.*"): get_internal_wsgi_application() @override_settings(WSGI_APPLICATION="wsgi.wsgi.noexist") def test_bad_name(self): with six.assertRaisesRegex(self, - ImproperlyConfigured, - r"^WSGI application 'wsgi.wsgi.noexist' could not be loaded; Module.*"): + ImproperlyConfigured, + r"^WSGI application 'wsgi.wsgi.noexist' could not be loaded; Module.*"): get_internal_wsgi_application() From c8b637d88e279412f52388b72c2bb581060b4ee9 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 28 Nov 2013 10:41:20 -0600 Subject: [PATCH 007/158] All the E125 errors hvae been eradicated. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d45eff4c36..393d0dc094 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ install-script = scripts/rpm-install.sh [flake8] exclude=./django/utils/dictconfig.py,./django/contrib/comments/*,./django/utils/unittest.py,./django/utils/lru_cache.py,./tests/comment_tests/*,./django/test/_doctest.py,./django/utils/six.py,./django/conf/app_template/* -ignore=E124,E125,E127,E128,E501,W601 +ignore=E124,E127,E128,E501,W601 [metadata] license-file = LICENSE From 34b8a385588a051814f77481c875047c8bd23b37 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 23 Nov 2013 10:53:09 +0100 Subject: [PATCH 008/158] Fixed #21496 -- Fixed crash when GeometryField uses TextInput Thanks Rhett Garber for the report and initial patch. --- django/contrib/gis/forms/fields.py | 25 +++++++++++++---------- django/contrib/gis/tests/test_geoforms.py | 13 ++++++++++++ docs/releases/1.6.1.txt | 1 + 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/django/contrib/gis/forms/fields.py b/django/contrib/gis/forms/fields.py index 151f66c39c..d0eee9324e 100644 --- a/django/contrib/gis/forms/fields.py +++ b/django/contrib/gis/forms/fields.py @@ -44,10 +44,16 @@ class GeometryField(forms.Field): if not isinstance(value, GEOSGeometry): try: value = GEOSGeometry(value) - if not value.srid: - value.srid = self.widget.map_srid except (GEOSException, ValueError, TypeError): raise forms.ValidationError(self.error_messages['invalid_geom'], code='invalid_geom') + + # Try to set the srid + if not value.srid: + try: + value.srid = self.widget.map_srid + except AttributeError: + if self.srid: + value.srid = self.srid return value def clean(self, value): @@ -66,15 +72,12 @@ class GeometryField(forms.Field): raise forms.ValidationError(self.error_messages['invalid_geom_type'], code='invalid_geom_type') # Transforming the geometry if the SRID was set. - if self.srid: - if not geom.srid: - # Should match that of the field if not given. - geom.srid = self.srid - elif self.srid != -1 and self.srid != geom.srid: - try: - geom.transform(self.srid) - except GEOSException: - raise forms.ValidationError(self.error_messages['transform_error'], code='transform_error') + if self.srid and self.srid != -1 and self.srid != geom.srid: + try: + geom.transform(self.srid) + except GEOSException: + raise forms.ValidationError( + self.error_messages['transform_error'], code='transform_error') return geom diff --git a/django/contrib/gis/tests/test_geoforms.py b/django/contrib/gis/tests/test_geoforms.py index ca28fb503c..bb2f65a962 100644 --- a/django/contrib/gis/tests/test_geoforms.py +++ b/django/contrib/gis/tests/test_geoforms.py @@ -76,6 +76,19 @@ class GeometryFieldTest(SimpleTestCase): for wkt in ('POINT(5)', 'MULTI POLYGON(((0 0, 0 1, 1 1, 1 0, 0 0)))', 'BLAH(0 0, 1 1)'): self.assertRaises(forms.ValidationError, fld.to_python, wkt) + def test_field_with_text_widget(self): + class PointForm(forms.Form): + pt = forms.PointField(srid=4326, widget=forms.TextInput) + + form = PointForm() + cleaned_pt = form.fields['pt'].clean('POINT(5 23)') + self.assertEqual(cleaned_pt, GEOSGeometry('POINT(5 23)')) + self.assertEqual(4326, cleaned_pt.srid) + + point = GEOSGeometry('SRID=4326;POINT(5 23)') + form = PointForm(data={'pt': 'POINT(5 23)'}, initial={'pt': point}) + self.assertFalse(form.has_changed()) + @skipUnless(HAS_GDAL and HAS_SPATIALREFSYS, "SpecializedFieldTest needs gdal support and a spatial database") diff --git a/docs/releases/1.6.1.txt b/docs/releases/1.6.1.txt index 91c6a9e261..f7c76afbb1 100644 --- a/docs/releases/1.6.1.txt +++ b/docs/releases/1.6.1.txt @@ -39,3 +39,4 @@ Bug fixes importing ``get_wsgi_application`` (#21486). * Fixed test client ``logout()`` method when using the cookie-based session backend (#21448). +* Fixed a crash when a ``GeometryField`` uses a non-geometric widget (#21496). From 91fce675a453764cb233e53c0460826600c828fa Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Fri, 29 Nov 2013 03:01:08 +0700 Subject: [PATCH 009/158] Use 'update_fields' in RelatedManager.clear() when bulk=False. Thanks Simon Charette for the suggestion. Refs #21169. --- django/db/models/fields/related.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 4a945e0f45..1466270e5f 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -492,7 +492,7 @@ def create_foreign_related_manager(superclass, rel_field, rel_model): with transaction.commit_on_success_unless_managed(using=db, savepoint=False): for obj in queryset: setattr(obj, rel_field.name, None) - obj.save() + obj.save(update_fields=[rel_field.name]) _clear.alters_data = True return RelatedManager From 42ac13800952e74e3969b850fa635b400948c5ee Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Thu, 28 Nov 2013 21:42:24 -0500 Subject: [PATCH 010/158] Fixed a deprecation warning introduced by 96dd48c83f. --- django/db/migrations/autodetector.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django/db/migrations/autodetector.py b/django/db/migrations/autodetector.py index 656ce3e939..083351a6f1 100644 --- a/django/db/migrations/autodetector.py +++ b/django/db/migrations/autodetector.py @@ -1,11 +1,13 @@ -import re +import importlib import os +import re import sys -from django.utils import datetime_safe, importlib -from django.utils.six.moves import input + from django.db.migrations import operations from django.db.migrations.migration import Migration from django.db.models.loading import cache +from django.utils import datetime_safe +from django.utils.six.moves import input class MigrationAutodetector(object): From 7e2d61a9724644d6d1c7ce9361d9fd5be3e2ab86 Mon Sep 17 00:00:00 2001 From: Vajrasky Kok Date: Tue, 5 Nov 2013 18:02:54 +0800 Subject: [PATCH 011/158] Fixed #21380 -- Added a way to set different permission for static directories. Previously when collecting static files, the directories would receive permissions from the global umask. Now the default permission comes from FILE_UPLOAD_DIRECTORY_PERMISSIONS and there's an option to specify the permissions by subclassing any of the static files storage classes and setting the directory_permissions_mode parameter. --- .../management/commands/collectstatic.py | 6 ---- django/core/files/storage.py | 13 +++++--- docs/ref/contrib/staticfiles.txt | 18 ++++++---- docs/ref/files/storage.txt | 13 +++++++- docs/ref/settings.txt | 14 +++++--- docs/releases/1.7.txt | 7 ++-- tests/staticfiles_tests/tests.py | 33 +++++++++++++++++-- 7 files changed, 77 insertions(+), 27 deletions(-) diff --git a/django/contrib/staticfiles/management/commands/collectstatic.py b/django/contrib/staticfiles/management/commands/collectstatic.py index c1e9fa811b..67fa6229c4 100644 --- a/django/contrib/staticfiles/management/commands/collectstatic.py +++ b/django/contrib/staticfiles/management/commands/collectstatic.py @@ -294,12 +294,6 @@ Type 'yes' to continue, or 'no' to cancel: """ self.log("Pretending to copy '%s'" % source_path, level=1) else: self.log("Copying '%s'" % source_path, level=1) - if self.local: - full_path = self.storage.path(prefixed_path) - try: - os.makedirs(os.path.dirname(full_path)) - except OSError: - pass with source_storage.open(path) as source_file: self.storage.save(prefixed_path, source_file) if not prefixed_path in self.copied_files: diff --git a/django/core/files/storage.py b/django/core/files/storage.py index aae3d7e984..e8db352d3e 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -149,7 +149,8 @@ class FileSystemStorage(Storage): Standard filesystem storage """ - def __init__(self, location=None, base_url=None, file_permissions_mode=None): + def __init__(self, location=None, base_url=None, file_permissions_mode=None, + directory_permissions_mode=None): if location is None: location = settings.MEDIA_ROOT self.base_location = location @@ -161,6 +162,10 @@ class FileSystemStorage(Storage): file_permissions_mode if file_permissions_mode is not None else settings.FILE_UPLOAD_PERMISSIONS ) + self.directory_permissions_mode = ( + directory_permissions_mode if directory_permissions_mode is not None + else settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS + ) def _open(self, name, mode='rb'): return File(open(self.path(name), mode)) @@ -175,12 +180,12 @@ class FileSystemStorage(Storage): directory = os.path.dirname(full_path) if not os.path.exists(directory): try: - if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: + if self.directory_permissions_mode is not None: # os.makedirs applies the global umask, so we reset it, - # for consistency with FILE_UPLOAD_PERMISSIONS behavior. + # for consistency with file_permissions_mode behavior. old_umask = os.umask(0) try: - os.makedirs(directory, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) + os.makedirs(directory, self.directory_permissions_mode) finally: os.umask(old_umask) else: diff --git a/docs/ref/contrib/staticfiles.txt b/docs/ref/contrib/staticfiles.txt index c65533fa0f..3202321269 100644 --- a/docs/ref/contrib/staticfiles.txt +++ b/docs/ref/contrib/staticfiles.txt @@ -60,16 +60,19 @@ by the :class:`~django.contrib.staticfiles.storage.CachedStaticFilesStorage` by default. By default, collected files receive permissions from -:setting:`FILE_UPLOAD_PERMISSIONS`. If you would like different permissions for -these files, you can subclass either of the :ref:`static files storage -classes ` and specify the ``file_permissions_mode`` -parameter. For example:: +:setting:`FILE_UPLOAD_PERMISSIONS` and collected directories receive permissions +from :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS`. If you would like different +permissions for these files and/or directories, you can subclass either of the +:ref:`static files storage classes ` and specify the +``file_permissions_mode`` and/or ``directory_permissions_mode`` parameters, +respectively. For example:: from django.contrib.staticfiles import storage class MyStaticFilesStorage(storage.StaticFilesStorage): def __init__(self, *args, **kwargs): kwargs['file_permissions_mode'] = 0o640 + kwargs['directory_permissions_mode'] = 0o760 super(CustomStaticFilesStorage, self).__init__(*args, **kwargs) Then set the :setting:`STATICFILES_STORAGE` setting to @@ -77,9 +80,10 @@ Then set the :setting:`STATICFILES_STORAGE` setting to .. versionadded:: 1.7 - The ability to override ``file_permissions_mode`` is new in Django 1.7. - Previously the file permissions always used - :setting:`FILE_UPLOAD_PERMISSIONS`. + The ability to override ``file_permissions_mode`` and + ``directory_permissions_mode`` is new in Django 1.7. Previously the file + permissions always used :setting:`FILE_UPLOAD_PERMISSIONS` and the directory + permissions always used :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS`. .. highlight:: console diff --git a/docs/ref/files/storage.txt b/docs/ref/files/storage.txt index bd8fa56787..6d367b70fc 100644 --- a/docs/ref/files/storage.txt +++ b/docs/ref/files/storage.txt @@ -29,7 +29,7 @@ Django provides two convenient ways to access the current storage class: The FileSystemStorage Class --------------------------- -.. class:: FileSystemStorage([location=None, base_url=None, file_permissions_mode=None]) +.. class:: FileSystemStorage([location=None, base_url=None, file_permissions_mode=None, directory_permissions_mode=None]) The :class:`~django.core.files.storage.FileSystemStorage` class implements basic file storage on a local filesystem. It inherits from @@ -46,6 +46,17 @@ The FileSystemStorage Class The ``file_permissions_mode`` attribute was added. Previously files always received :setting:`FILE_UPLOAD_PERMISSIONS` permissions. + .. attribute:: directory_permissions_mode + + The file system permissions that the directory will receive when it is + saved. Defaults to :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS`. + + .. versionadded:: 1.7 + + The ``directory_permissions_mode`` attribute was added. Previously + directories always received + :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` permissions. + .. note:: The ``FileSystemStorage.delete()`` method will not raise diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index c99a7e4347..a1207fb0f8 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1135,9 +1135,15 @@ FILE_UPLOAD_DIRECTORY_PERMISSIONS Default: ``None`` -The numeric mode to apply to directories created in the process of -uploading files. This value mirrors the functionality and caveats of -the :setting:`FILE_UPLOAD_PERMISSIONS` setting. +The numeric mode to apply to directories created in the process of uploading +files. + +This setting also determines the default permissions for collected static +directories when using the :djadmin:`collectstatic` management command. See +:djadmin:`collectstatic` for details on overriding it. + +This value mirrors the functionality and caveats of the +:setting:`FILE_UPLOAD_PERMISSIONS` setting. .. setting:: FILE_UPLOAD_PERMISSIONS @@ -1157,7 +1163,7 @@ system's standard umask. This setting also determines the default permissions for collected static files when using the :djadmin:`collectstatic` management command. See -:djadmin:`collectstatic` for details on overridding it. +:djadmin:`collectstatic` for details on overriding it. .. warning:: diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 8d130f9585..9cb907aded 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -256,10 +256,11 @@ Minor features ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * The :ref:`static files storage classes ` may be - subclassed to override the permissions that collected static files receive by - setting the + subclassed to override the permissions that collected static files and + directories receive by setting the :attr:`~django.core.files.storage.FileSystemStorage.file_permissions_mode` - parameter. See :djadmin:`collectstatic` for example usage. + and :attr:`~django.core.files.storage.FileSystemStorage.directory_permissions_mode` + parameters. See :djadmin:`collectstatic` for example usage. :mod:`django.contrib.syndication` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/staticfiles_tests/tests.py b/tests/staticfiles_tests/tests.py index 39ce49817c..e7c1383373 100644 --- a/tests/staticfiles_tests/tests.py +++ b/tests/staticfiles_tests/tests.py @@ -823,6 +823,7 @@ class CustomStaticFilesStorage(storage.StaticFilesStorage): """ def __init__(self, *args, **kwargs): kwargs['file_permissions_mode'] = 0o640 + kwargs['directory_permissions_mode'] = 0o740 super(CustomStaticFilesStorage, self).__init__(*args, **kwargs) @@ -839,21 +840,49 @@ class TestStaticFilePermissions(BaseCollectionTestCase, StaticFilesTestCase): 'link': False, 'dry_run': False} + def setUp(self): + self.umask = 0o027 + self.old_umask = os.umask(self.umask) + super(TestStaticFilePermissions, self).setUp() + + def tearDown(self): + os.umask(self.old_umask) + super(TestStaticFilePermissions, self).tearDown() + # Don't run collectstatic command in this test class. def run_collectstatic(self, **kwargs): pass - @override_settings(FILE_UPLOAD_PERMISSIONS=0o655) + @override_settings(FILE_UPLOAD_PERMISSIONS=0o655, + FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765) + def test_collect_static_files_permissions(self): + collectstatic.Command().execute(**self.command_params) + test_file = os.path.join(settings.STATIC_ROOT, "test.txt") + test_dir = os.path.join(settings.STATIC_ROOT, "subdir") + file_mode = os.stat(test_file)[0] & 0o777 + dir_mode = os.stat(test_dir)[0] & 0o777 + self.assertEqual(file_mode, 0o655) + self.assertEqual(dir_mode, 0o765) + + @override_settings(FILE_UPLOAD_PERMISSIONS=None, + FILE_UPLOAD_DIRECTORY_PERMISSIONS=None) def test_collect_static_files_default_permissions(self): collectstatic.Command().execute(**self.command_params) test_file = os.path.join(settings.STATIC_ROOT, "test.txt") + test_dir = os.path.join(settings.STATIC_ROOT, "subdir") file_mode = os.stat(test_file)[0] & 0o777 - self.assertEqual(file_mode, 0o655) + dir_mode = os.stat(test_dir)[0] & 0o777 + self.assertEqual(file_mode, 0o666 & ~self.umask) + self.assertEqual(dir_mode, 0o777 & ~self.umask) @override_settings(FILE_UPLOAD_PERMISSIONS=0o655, + FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765, STATICFILES_STORAGE='staticfiles_tests.tests.CustomStaticFilesStorage') def test_collect_static_files_subclass_of_static_storage(self): collectstatic.Command().execute(**self.command_params) test_file = os.path.join(settings.STATIC_ROOT, "test.txt") + test_dir = os.path.join(settings.STATIC_ROOT, "subdir") file_mode = os.stat(test_file)[0] & 0o777 + dir_mode = os.stat(test_dir)[0] & 0o777 self.assertEqual(file_mode, 0o640) + self.assertEqual(dir_mode, 0o740) From f563c339ca2eed81706ab17726c79a6f00d7c553 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Tue, 12 Nov 2013 00:56:01 +0700 Subject: [PATCH 012/158] Fixed #20867 -- Added the Form.add_error() method. Refs #20199 #16986. Thanks @akaariai, @bmispelon, @mjtamlyn, @timgraham for the reviews. --- django/core/exceptions.py | 90 ++++++++++++++++----------- django/db/models/base.py | 2 +- django/forms/forms.py | 52 ++++++++++++++-- django/forms/models.py | 43 ++++++------- docs/ref/forms/api.txt | 20 ++++++ docs/ref/forms/validation.txt | 28 +++++++++ docs/releases/1.7.txt | 3 + tests/forms_tests/tests/test_forms.py | 50 +++++++++++++-- 8 files changed, 214 insertions(+), 74 deletions(-) diff --git a/django/core/exceptions.py b/django/core/exceptions.py index efec22850b..87b173b9b0 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -77,64 +77,78 @@ class ValidationError(Exception): """An error while validating data.""" def __init__(self, message, code=None, params=None): """ - ValidationError can be passed any object that can be printed (usually - a string), a list of objects or a dictionary. + The `message` argument can be a single error, a list of errors, or a + dictionary that maps field names to lists of errors. What we define as + an "error" can be either a simple string or an instance of + ValidationError with its message attribute set, and what we define as + list or dictionary can be an actual `list` or `dict` or an instance + of ValidationError with its `error_list` or `error_dict` attribute set. """ + if isinstance(message, ValidationError): + if hasattr(message, 'error_dict'): + message = message.error_dict + elif not hasattr(message, 'message'): + message = message.error_list + else: + message, code, params = message.message, message.code, message.params + if isinstance(message, dict): - self.error_dict = message + self.error_dict = {} + for field, messages in message.items(): + if not isinstance(messages, ValidationError): + messages = ValidationError(messages) + self.error_dict[field] = messages.error_list + elif isinstance(message, list): - self.error_list = message + self.error_list = [] + for message in message: + # Normalize plain strings to instances of ValidationError. + if not isinstance(message, ValidationError): + message = ValidationError(message) + self.error_list.extend(message.error_list) + else: + self.message = message self.code = code self.params = params - self.message = message self.error_list = [self] @property def message_dict(self): - message_dict = {} - for field, messages in self.error_dict.items(): - message_dict[field] = [] - for message in messages: - if isinstance(message, ValidationError): - message_dict[field].extend(message.messages) - else: - message_dict[field].append(force_text(message)) - return message_dict + return dict(self) @property def messages(self): if hasattr(self, 'error_dict'): - message_list = reduce(operator.add, self.error_dict.values()) - else: - message_list = self.error_list - - messages = [] - for message in message_list: - if isinstance(message, ValidationError): - params = message.params - message = message.message - if params: - message %= params - message = force_text(message) - messages.append(message) - return messages - - def __str__(self): - if hasattr(self, 'error_dict'): - return repr(self.message_dict) - return repr(self.messages) - - def __repr__(self): - return 'ValidationError(%s)' % self + return reduce(operator.add, dict(self).values()) + return list(self) def update_error_dict(self, error_dict): if hasattr(self, 'error_dict'): if error_dict: - for k, v in self.error_dict.items(): - error_dict.setdefault(k, []).extend(v) + for field, errors in self.error_dict.items(): + error_dict.setdefault(field, []).extend(errors) else: error_dict = self.error_dict else: error_dict[NON_FIELD_ERRORS] = self.error_list return error_dict + + def __iter__(self): + if hasattr(self, 'error_dict'): + for field, errors in self.error_dict.items(): + yield field, list(ValidationError(errors)) + else: + for error in self.error_list: + message = error.message + if error.params: + message %= error.params + yield force_text(message) + + def __str__(self): + if hasattr(self, 'error_dict'): + return repr(dict(self)) + return repr(list(self)) + + def __repr__(self): + return 'ValidationError(%s)' % self diff --git a/django/db/models/base.py b/django/db/models/base.py index ce3f095055..015fc9ff3c 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -987,7 +987,7 @@ class Model(six.with_metaclass(ModelBase)): def clean_fields(self, exclude=None): """ - Cleans all fields and raises a ValidationError containing message_dict + Cleans all fields and raises a ValidationError containing a dict of all validation errors if any occur. """ if exclude is None: diff --git a/django/forms/forms.py b/django/forms/forms.py index 15bbff229b..350b8b8993 100644 --- a/django/forms/forms.py +++ b/django/forms/forms.py @@ -290,6 +290,51 @@ class BaseForm(object): prefix = self.add_prefix(fieldname) return field.widget.value_from_datadict(self.data, self.files, prefix) + def add_error(self, field, error): + """ + Update the content of `self._errors`. + + The `field` argument is the name of the field to which the errors + should be added. If its value is None the errors will be treated as + NON_FIELD_ERRORS. + + The `error` argument can be a single error, a list of errors, or a + dictionary that maps field names to lists of errors. What we define as + an "error" can be either a simple string or an instance of + ValidationError with its message attribute set and what we define as + list or dictionary can be an actual `list` or `dict` or an instance + of ValidationError with its `error_list` or `error_dict` attribute set. + + If `error` is a dictionary, the `field` argument *must* be None and + errors will be added to the fields that correspond to the keys of the + dictionary. + """ + if not isinstance(error, ValidationError): + # Normalize to ValidationError and let its constructor + # do the hard work of making sense of the input. + error = ValidationError(error) + + if hasattr(error, 'error_dict'): + if field is not None: + raise TypeError( + "The argument `field` must be `None` when the `error` " + "argument contains errors for multiple fields." + ) + else: + error = dict(error) + else: + error = {field or NON_FIELD_ERRORS: list(error)} + + for field, error_list in error.items(): + if field not in self.errors: + if field != NON_FIELD_ERRORS and field not in self.fields: + raise ValueError( + "'%s' has no field named '%s'." % (self.__class__.__name__, field)) + self._errors[field] = self.error_class() + self._errors[field].extend(error_list) + if field in self.cleaned_data: + del self.cleaned_data[field] + def full_clean(self): """ Cleans all of self.data and populates self._errors and @@ -303,6 +348,7 @@ class BaseForm(object): # changed from the initial data, short circuit any validation. if self.empty_permitted and not self.has_changed(): return + self._clean_fields() self._clean_form() self._post_clean() @@ -324,15 +370,13 @@ class BaseForm(object): value = getattr(self, 'clean_%s' % name)() self.cleaned_data[name] = value except ValidationError as e: - self._errors[name] = self.error_class(e.messages) - if name in self.cleaned_data: - del self.cleaned_data[name] + self.add_error(name, e) def _clean_form(self): try: cleaned_data = self.clean() except ValidationError as e: - self._errors[NON_FIELD_ERRORS] = self.error_class(e.messages) + self.add_error(None, e) else: if cleaned_data is not None: self.cleaned_data = cleaned_data diff --git a/django/forms/models.py b/django/forms/models.py index b14ea7a265..5c2c77cbf2 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -326,27 +326,6 @@ class BaseModelForm(BaseForm): super(BaseModelForm, self).__init__(data, files, auto_id, prefix, object_data, error_class, label_suffix, empty_permitted) - def _update_errors(self, errors): - for field, messages in errors.error_dict.items(): - if field not in self.fields: - continue - field = self.fields[field] - for message in messages: - if isinstance(message, ValidationError): - if message.code in field.error_messages: - message.message = field.error_messages[message.code] - - message_dict = errors.message_dict - for k, v in message_dict.items(): - if k != NON_FIELD_ERRORS: - self._errors.setdefault(k, self.error_class()).extend(v) - # Remove the data from the cleaned_data dict since it was invalid - if k in self.cleaned_data: - del self.cleaned_data[k] - if NON_FIELD_ERRORS in message_dict: - messages = message_dict[NON_FIELD_ERRORS] - self._errors.setdefault(NON_FIELD_ERRORS, self.error_class()).extend(messages) - def _get_validation_exclusions(self): """ For backwards-compatibility, several types of fields need to be @@ -393,6 +372,20 @@ class BaseModelForm(BaseForm): self._validate_unique = True return self.cleaned_data + def _update_errors(self, errors): + # Override any validation error messages defined at the model level + # with those defined on the form fields. + for field, messages in errors.error_dict.items(): + if field not in self.fields: + continue + field = self.fields[field] + for message in messages: + if (isinstance(message, ValidationError) and + message.code in field.error_messages): + message.message = field.error_messages[message.code] + + self.add_error(None, errors) + def _post_clean(self): opts = self._meta # Update the model instance with self.cleaned_data. @@ -407,13 +400,12 @@ class BaseModelForm(BaseForm): # object being referred to may not yet fully exist (#12749). # However, these fields *must* be included in uniqueness checks, # so this can't be part of _get_validation_exclusions(). - for f_name, field in self.fields.items(): + for name, field in self.fields.items(): if isinstance(field, InlineForeignKeyField): - exclude.append(f_name) + exclude.append(name) try: - self.instance.full_clean(exclude=exclude, - validate_unique=False) + self.instance.full_clean(exclude=exclude, validate_unique=False) except ValidationError as e: self._update_errors(e) @@ -695,6 +687,7 @@ class BaseModelFormSet(BaseFormSet): del form.cleaned_data[field] # mark the data as seen seen_data.add(data) + if errors: raise ValidationError(errors) diff --git a/docs/ref/forms/api.txt b/docs/ref/forms/api.txt index c15f748308..c06a836bef 100644 --- a/docs/ref/forms/api.txt +++ b/docs/ref/forms/api.txt @@ -117,6 +117,26 @@ The validation routines will only get called once, regardless of how many times you access :attr:`~Form.errors` or call :meth:`~Form.is_valid`. This means that if validation has side effects, those side effects will only be triggered once. +.. method:: Form.add_error(field, error) + +.. versionadded:: 1.7 + +This method allows adding errors to specific fields from within the +``Form.clean()`` method, or from outside the form altogether; for instance +from a view. This is a better alternative to fiddling directly with +``Form._errors`` as described in :ref:`modifying-field-errors`. + +The ``field`` argument is the name of the field to which the errors +should be added. If its value is ``None`` the error will be treated as +a non-field error as returned by ``Form.non_field_errors()``. + +The ``error`` argument can be a simple string, or preferably an instance of +``ValidationError``. See :ref:`raising-validation-error` for best practices +when defining form errors. + +Note that ``Form.add_error()`` automatically removes the relevant field from +``cleaned_data``. + Behavior of unbound forms ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/ref/forms/validation.txt b/docs/ref/forms/validation.txt index ea294c4108..3c7715dfcb 100644 --- a/docs/ref/forms/validation.txt +++ b/docs/ref/forms/validation.txt @@ -464,3 +464,31 @@ Secondly, once we have decided that the combined data in the two fields we are considering aren't valid, we must remember to remove them from the ``cleaned_data``. `cleaned_data`` is present even if the form doesn't validate, but it contains only field values that did validate. + +.. versionchanged:: 1.7 + +In lieu of manipulating ``_errors`` directly, it's now possible to add errors +to specific fields with :meth:`django.forms.Form.add_error()`:: + + from django import forms + + class ContactForm(forms.Form): + # Everything as before. + ... + + def clean(self): + cleaned_data = super(ContactForm, self).clean() + cc_myself = cleaned_data.get("cc_myself") + subject = cleaned_data.get("subject") + + if cc_myself and subject and "help" not in subject: + msg = u"Must put 'help' in subject when cc'ing yourself." + self.add_error('cc_myself', msg) + self.add_error('subject', msg) + +The second argument of ``add_error()`` can be a simple string, or preferably +an instance of ``ValidationError``. See :ref:`raising-validation-error` for +more details. + +Unlike the ``_errors`` approach, ``add_error()` automatically removes the field +from ``cleaned_data``. diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 9cb907aded..0767659540 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -350,6 +350,9 @@ Forms * It's now possible to opt-out from a ``Form`` field declared in a parent class by shadowing it with a non-``Field`` value. +* The new :meth:`~django.forms.Form.add_error()` method allows adding errors + to specific form fields. + Internationalization ^^^^^^^^^^^^^^^^^^^^ diff --git a/tests/forms_tests/tests/test_forms.py b/tests/forms_tests/tests/test_forms.py index b30742106b..9fec9ddcaa 100644 --- a/tests/forms_tests/tests/test_forms.py +++ b/tests/forms_tests/tests/test_forms.py @@ -657,25 +657,49 @@ class FormsTestCase(TestCase): self.assertEqual(f.cleaned_data['password2'], 'foo') # Another way of doing multiple-field validation is by implementing the - # Form's clean() method. If you do this, any ValidationError raised by that - # method will not be associated with a particular field; it will have a - # special-case association with the field named '__all__'. - # Note that in Form.clean(), you have access to self.cleaned_data, a dictionary of - # all the fields/values that have *not* raised a ValidationError. Also note - # Form.clean() is required to return a dictionary of all clean data. + # Form's clean() method. Usually ValidationError raised by that method + # will not be associated with a particular field and will have a + # special-case association with the field named '__all__'. It's + # possible to associate the errors to particular field with the + # Form.add_error() method or by passing a dictionary that maps each + # field to one or more errors. + # + # Note that in Form.clean(), you have access to self.cleaned_data, a + # dictionary of all the fields/values that have *not* raised a + # ValidationError. Also note Form.clean() is required to return a + # dictionary of all clean data. class UserRegistration(Form): username = CharField(max_length=10) password1 = CharField(widget=PasswordInput) password2 = CharField(widget=PasswordInput) def clean(self): + # Test raising a ValidationError as NON_FIELD_ERRORS. if self.cleaned_data.get('password1') and self.cleaned_data.get('password2') and self.cleaned_data['password1'] != self.cleaned_data['password2']: raise ValidationError('Please make sure your passwords match.') + # Test raising ValidationError that targets multiple fields. + errors = {} + if self.cleaned_data.get('password1') == 'FORBIDDEN_VALUE': + errors['password1'] = 'Forbidden value.' + if self.cleaned_data.get('password2') == 'FORBIDDEN_VALUE': + errors['password2'] = ['Forbidden value.'] + if errors: + raise ValidationError(errors) + + # Test Form.add_error() + if self.cleaned_data.get('password1') == 'FORBIDDEN_VALUE2': + self.add_error(None, 'Non-field error 1.') + self.add_error('password1', 'Forbidden value 2.') + if self.cleaned_data.get('password2') == 'FORBIDDEN_VALUE2': + self.add_error('password2', 'Forbidden value 2.') + raise ValidationError('Non-field error 2.') + return self.cleaned_data f = UserRegistration(auto_id=False) self.assertEqual(f.errors, {}) + f = UserRegistration({}, auto_id=False) self.assertHTMLEqual(f.as_table(), """Username:
  • This field is required.
Password1:
  • This field is required.
@@ -683,6 +707,7 @@ class FormsTestCase(TestCase): self.assertEqual(f.errors['username'], ['This field is required.']) self.assertEqual(f.errors['password1'], ['This field is required.']) self.assertEqual(f.errors['password2'], ['This field is required.']) + f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'bar'}, auto_id=False) self.assertEqual(f.errors['__all__'], ['Please make sure your passwords match.']) self.assertHTMLEqual(f.as_table(), """
  • Please make sure your passwords match.
@@ -693,12 +718,25 @@ class FormsTestCase(TestCase):
  • Username:
  • Password1:
  • Password2:
    • """) + f = UserRegistration({'username': 'adrian', 'password1': 'foo', 'password2': 'foo'}, auto_id=False) self.assertEqual(f.errors, {}) self.assertEqual(f.cleaned_data['username'], 'adrian') self.assertEqual(f.cleaned_data['password1'], 'foo') self.assertEqual(f.cleaned_data['password2'], 'foo') + f = UserRegistration({'username': 'adrian', 'password1': 'FORBIDDEN_VALUE', 'password2': 'FORBIDDEN_VALUE'}, auto_id=False) + self.assertEqual(f.errors['password1'], ['Forbidden value.']) + self.assertEqual(f.errors['password2'], ['Forbidden value.']) + + f = UserRegistration({'username': 'adrian', 'password1': 'FORBIDDEN_VALUE2', 'password2': 'FORBIDDEN_VALUE2'}, auto_id=False) + self.assertEqual(f.errors['__all__'], ['Non-field error 1.', 'Non-field error 2.']) + self.assertEqual(f.errors['password1'], ['Forbidden value 2.']) + self.assertEqual(f.errors['password2'], ['Forbidden value 2.']) + + with six.assertRaisesRegex(self, ValueError, "has no field named"): + f.add_error('missing_field', 'Some error.') + def test_dynamic_construction(self): # It's possible to construct a Form dynamically by adding to the self.fields # dictionary in __init__(). Don't forget to call Form.__init__() within the From b72b85af153848ef5f1f07f5c1e5a81e3563b015 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Tue, 12 Nov 2013 21:18:09 +0700 Subject: [PATCH 013/158] Removed Form._errors from the docs in favor of the add_error API. --- docs/ref/forms/api.txt | 3 +- docs/ref/forms/validation.txt | 103 ++++------------------------------ 2 files changed, 11 insertions(+), 95 deletions(-) diff --git a/docs/ref/forms/api.txt b/docs/ref/forms/api.txt index c06a836bef..69667643b4 100644 --- a/docs/ref/forms/api.txt +++ b/docs/ref/forms/api.txt @@ -123,8 +123,7 @@ if validation has side effects, those side effects will only be triggered once. This method allows adding errors to specific fields from within the ``Form.clean()`` method, or from outside the form altogether; for instance -from a view. This is a better alternative to fiddling directly with -``Form._errors`` as described in :ref:`modifying-field-errors`. +from a view. The ``field`` argument is the name of the field to which the errors should be added. If its value is ``None`` the error will be treated as diff --git a/docs/ref/forms/validation.txt b/docs/ref/forms/validation.txt index 3c7715dfcb..a07cb93183 100644 --- a/docs/ref/forms/validation.txt +++ b/docs/ref/forms/validation.txt @@ -86,9 +86,8 @@ overridden: be associated with any field in particular. They go into a special "field" (called ``__all__``), which you can access via the ``non_field_errors()`` method if you need to. If you want to attach - errors to a specific field in the form, you will need to access the - ``_errors`` attribute on the form, which is - :ref:`described later `. + errors to a specific field in the form, you need to call + :meth:`~django.forms.Form.add_error()`. Also note that there are special considerations when overriding the ``clean()`` method of a ``ModelForm`` subclass. (see the @@ -202,52 +201,6 @@ with ``code``\s and ``params`` but a list of strings will also work:: _('Error 2'), ]) -.. _modifying-field-errors: - -Form subclasses and modifying field errors ------------------------------------------- - -Sometimes, in a form's ``clean()`` method, you will want to add an error -message to a particular field in the form. This won't always be appropriate -and the more typical situation is to raise a ``ValidationError`` from -``Form.clean()``, which is turned into a form-wide error that is available -through the ``Form.non_field_errors()`` method. - -When you really do need to attach the error to a particular field, you should -store (or amend) a key in the ``Form._errors`` attribute. This attribute is an -instance of a ``django.forms.utils.ErrorDict`` class. Essentially, though, it's -just a dictionary. There is a key in the dictionary for each field in the form -that has an error. Each value in the dictionary is a -``django.forms.utils.ErrorList`` instance, which is a list that knows how to -display itself in different ways. So you can treat ``_errors`` as a dictionary -mapping field names to lists. - -If you want to add a new error to a particular field, you should check whether -the key already exists in ``self._errors`` or not. If not, create a new entry -for the given key, holding an empty ``ErrorList`` instance. In either case, -you can then append your error message to the list for the field name in -question and it will be displayed when the form is displayed. - -There is an example of modifying ``self._errors`` in the following section. - -.. admonition:: What's in a name? - - You may be wondering why is this attribute called ``_errors`` and not - ``errors``. Normal Python practice is to prefix a name with an underscore - if it's not for external usage. In this case, you are subclassing the - ``Form`` class, so you are essentially writing new internals. In effect, - you are given permission to access some of the internals of ``Form``. - - Of course, any code outside your form should never access ``_errors`` - directly. The data is available to external code through the ``errors`` - property, which populates ``_errors`` before returning it). - - Another reason is purely historical: the attribute has been called - ``_errors`` since the early days of the forms module and changing it now - (particularly since ``errors`` is used for the read-only property name) - would be inconvenient for a number of reasons. You can use whichever - explanation makes you feel more comfortable. The result is the same. - Using validation in practice ---------------------------- @@ -366,6 +319,13 @@ write a cleaning method that operates on the ``recipients`` field, like so:: # not. return data +Sometimes you may want to add an error message to a particular field from the +form's ``clean()`` method, in which case you can use +:meth:`~django.forms.Form.add_error()`. Note that this won't always be +appropriate and the more typical situation is to raise a ``ValidationError`` +from , which is turned into a form-wide error that is available through the +``Form.non_field_errors()`` method. + Cleaning and validating fields that depend on each other ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -431,47 +391,6 @@ sample) looks like this:: from django import forms - class ContactForm(forms.Form): - # Everything as before. - ... - - def clean(self): - cleaned_data = super(ContactForm, self).clean() - cc_myself = cleaned_data.get("cc_myself") - subject = cleaned_data.get("subject") - - if cc_myself and subject and "help" not in subject: - # We know these are not in self._errors now (see discussion - # below). - msg = u"Must put 'help' in subject when cc'ing yourself." - self._errors["cc_myself"] = self.error_class([msg]) - self._errors["subject"] = self.error_class([msg]) - - # These fields are no longer valid. Remove them from the - # cleaned data. - del cleaned_data["cc_myself"] - del cleaned_data["subject"] - -As you can see, this approach requires a bit more effort, not withstanding the -extra design effort to create a sensible form display. The details are worth -noting, however. Firstly, earlier we mentioned that you might need to check if -the field name keys already exist in the ``_errors`` dictionary. In this case, -since we know the fields exist in ``self.cleaned_data``, they must have been -valid when cleaned as individual fields, so there will be no corresponding -entries in ``_errors``. - -Secondly, once we have decided that the combined data in the two fields we are -considering aren't valid, we must remember to remove them from the -``cleaned_data``. `cleaned_data`` is present even if the form doesn't -validate, but it contains only field values that did validate. - -.. versionchanged:: 1.7 - -In lieu of manipulating ``_errors`` directly, it's now possible to add errors -to specific fields with :meth:`django.forms.Form.add_error()`:: - - from django import forms - class ContactForm(forms.Form): # Everything as before. ... @@ -488,7 +407,5 @@ to specific fields with :meth:`django.forms.Form.add_error()`:: The second argument of ``add_error()`` can be a simple string, or preferably an instance of ``ValidationError``. See :ref:`raising-validation-error` for -more details. - -Unlike the ``_errors`` approach, ``add_error()` automatically removes the field +more details. Note that ``add_error()` automatically removes the field from ``cleaned_data``. From 9af7e18f3579df18625b9eda70735790f23aeb96 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Nov 2013 16:57:14 -0600 Subject: [PATCH 014/158] Fixed an unescisarily gendered pronoun in a docstring --- django/contrib/auth/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/auth/views.py b/django/contrib/auth/views.py index 5e11030f03..013ed9f3e2 100644 --- a/django/contrib/auth/views.py +++ b/django/contrib/auth/views.py @@ -98,7 +98,7 @@ def logout(request, next_page=None, def logout_then_login(request, login_url=None, current_app=None, extra_context=None): """ - Logs out the user if he is logged in. Then redirects to the log-in page. + Logs out the user if they are logged in. Then redirects to the log-in page. """ if not login_url: login_url = settings.LOGIN_URL From 50a8ab7cd1e611e6422a148becaec02218577d67 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 30 Nov 2013 10:53:08 +0100 Subject: [PATCH 015/158] Enabled makemessages to support several translation directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Rémy Hubscher, Ramiro Morales, Unai Zalakain and Tim Graham for the reviews. Also fixes #16084. --- .../core/management/commands/makemessages.py | 156 +++++++++++------- docs/man/django-admin.1 | 3 +- docs/ref/django-admin.txt | 4 +- docs/releases/1.7.txt | 5 + docs/topics/i18n/translation.txt | 28 ++-- tests/i18n/project_dir/__init__.py | 4 + .../project_dir/app_no_locale/__init__.py | 0 .../i18n/project_dir/app_no_locale/models.py | 3 + .../project_dir/app_with_locale/__init__.py | 0 .../app_with_locale/locale/.gitkeep | 0 .../project_dir/app_with_locale/models.py | 3 + .../i18n/project_dir/project_locale/.gitkeep | 0 tests/i18n/test_extraction.py | 46 ++++++ 13 files changed, 172 insertions(+), 80 deletions(-) create mode 100644 tests/i18n/project_dir/__init__.py create mode 100644 tests/i18n/project_dir/app_no_locale/__init__.py create mode 100644 tests/i18n/project_dir/app_no_locale/models.py create mode 100644 tests/i18n/project_dir/app_with_locale/__init__.py create mode 100644 tests/i18n/project_dir/app_with_locale/locale/.gitkeep create mode 100644 tests/i18n/project_dir/app_with_locale/models.py create mode 100644 tests/i18n/project_dir/project_locale/.gitkeep diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 25ca250c4a..4994aaf6ba 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -29,25 +29,28 @@ def check_programs(*programs): @total_ordering class TranslatableFile(object): - def __init__(self, dirpath, file_name): + def __init__(self, dirpath, file_name, locale_dir): self.file = file_name self.dirpath = dirpath + self.locale_dir = locale_dir def __repr__(self): return "" % os.sep.join([self.dirpath, self.file]) def __eq__(self, other): - return self.dirpath == other.dirpath and self.file == other.file + return self.path == other.path def __lt__(self, other): - if self.dirpath == other.dirpath: - return self.file < other.file - return self.dirpath < other.dirpath + return self.path < other.path - def process(self, command, potfile, domain, keep_pot=False): + @property + def path(self): + return os.path.join(self.dirpath, self.file) + + def process(self, command, domain): """ - Extract translatable literals from self.file for :param domain: - creating or updating the :param potfile: POT file. + Extract translatable literals from self.file for :param domain:, + creating or updating the POT file. Uses the xgettext GNU gettext utility. """ @@ -127,8 +130,6 @@ class TranslatableFile(object): if status != STATUS_OK: if is_templatized: os.unlink(work_file) - if not keep_pot and os.path.exists(potfile): - os.unlink(potfile) raise CommandError( "errors happened while running xgettext on %s\n%s" % (self.file, errors)) @@ -136,6 +137,8 @@ class TranslatableFile(object): # Print warnings command.stdout.write(errors) if msgs: + # Write/append messages to pot file + potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain)) if is_templatized: # Remove '.py' suffix if os.name == 'nt': @@ -147,6 +150,7 @@ class TranslatableFile(object): new = '#: ' + orig_file[2:] msgs = msgs.replace(old, new) write_pot_file(potfile, msgs) + if is_templatized: os.unlink(work_file) @@ -242,64 +246,94 @@ class Command(NoArgsCommand): % get_text_list(list(self.extensions), 'and')) self.invoked_for_django = False + self.locale_paths = [] + self.default_locale_path = None if os.path.isdir(os.path.join('conf', 'locale')): - localedir = os.path.abspath(os.path.join('conf', 'locale')) + self.locale_paths = [os.path.abspath(os.path.join('conf', 'locale'))] + self.default_locale_path = self.locale_paths[0] self.invoked_for_django = True # Ignoring all contrib apps self.ignore_patterns += ['contrib/*'] - elif os.path.isdir('locale'): - localedir = os.path.abspath('locale') else: - raise CommandError("This script should be run from the Django Git " - "tree or your project or app tree. If you did indeed run it " - "from the Git checkout or your project or application, " - "maybe you are just missing the conf/locale (in the django " - "tree) or locale (for project and application) directory? It " - "is not created automatically, you have to create it by hand " - "if you want to enable i18n for your project or application.") + self.locale_paths.extend(list(settings.LOCALE_PATHS)) + # Allow to run makemessages inside an app dir + if os.path.isdir('locale'): + self.locale_paths.append(os.path.abspath('locale')) + if self.locale_paths: + self.default_locale_path = self.locale_paths[0] + if not os.path.exists(self.default_locale_path): + os.makedirs(self.default_locale_path) - check_programs('xgettext') - - potfile = self.build_pot_file(localedir) - - # Build po files for each selected locale + # Build locale list locales = [] if locale is not None: locales = locale elif process_all: - locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir)) + locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % self.default_locale_path)) locales = [os.path.basename(l) for l in locale_dirs] - if locales: check_programs('msguniq', 'msgmerge', 'msgattrib') + check_programs('xgettext') + try: + potfiles = self.build_potfiles() + + # Build po files for each selected locale for locale in locales: if self.verbosity > 0: self.stdout.write("processing locale %s\n" % locale) - self.write_po_file(potfile, locale) + for potfile in potfiles: + self.write_po_file(potfile, locale) finally: - if not self.keep_pot and os.path.exists(potfile): - os.unlink(potfile) + if not self.keep_pot: + self.remove_potfiles() - def build_pot_file(self, localedir): + def build_potfiles(self): + """ + Build pot files and apply msguniq to them. + """ file_list = self.find_files(".") - - potfile = os.path.join(localedir, '%s.pot' % str(self.domain)) - if os.path.exists(potfile): - # Remove a previous undeleted potfile, if any - os.unlink(potfile) - + self.remove_potfiles() for f in file_list: try: - f.process(self, potfile, self.domain, self.keep_pot) + f.process(self, self.domain) except UnicodeDecodeError: self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath)) - return potfile + + potfiles = [] + for path in self.locale_paths: + potfile = os.path.join(path, '%s.pot' % str(self.domain)) + if not os.path.exists(potfile): + continue + args = ['msguniq', '--to-code=utf-8'] + if self.wrap: + args.append(self.wrap) + if self.location: + args.append(self.location) + args.append(potfile) + msgs, errors, status = popen_wrapper(args) + if errors: + if status != STATUS_OK: + raise CommandError( + "errors happened while running msguniq\n%s" % errors) + elif self.verbosity > 0: + self.stdout.write(errors) + with open(potfile, 'w') as fp: + fp.write(msgs) + potfiles.append(potfile) + return potfiles + + def remove_potfiles(self): + for path in self.locale_paths: + pot_path = os.path.join(path, '%s.pot' % str(self.domain)) + if os.path.exists(pot_path): + os.unlink(pot_path) def find_files(self, root): """ - Helper method to get all files in the given root. + Helper method to get all files in the given root. Also check that there + is a matching locale dir for each file. """ def is_ignored(path, ignore_patterns): @@ -319,12 +353,26 @@ class Command(NoArgsCommand): dirnames.remove(dirname) if self.verbosity > 1: self.stdout.write('ignoring directory %s\n' % dirname) + elif dirname == 'locale': + dirnames.remove(dirname) + self.locale_paths.insert(0, os.path.join(os.path.abspath(dirpath), dirname)) for filename in filenames: - if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), self.ignore_patterns): + file_path = os.path.normpath(os.path.join(dirpath, filename)) + if is_ignored(file_path, self.ignore_patterns): if self.verbosity > 1: self.stdout.write('ignoring file %s in %s\n' % (filename, dirpath)) else: - all_files.append(TranslatableFile(dirpath, filename)) + locale_dir = None + for path in self.locale_paths: + if os.path.abspath(dirpath).startswith(os.path.dirname(path)): + locale_dir = path + break + if not locale_dir: + locale_dir = self.default_locale_path + if not locale_dir: + raise CommandError( + "Unable to find a locale path to store translations for file %s" % file_path) + all_files.append(TranslatableFile(dirpath, filename, locale_dir)) return sorted(all_files) def write_po_file(self, potfile, locale): @@ -332,30 +380,14 @@ class Command(NoArgsCommand): Creates or updates the PO file for self.domain and :param locale:. Uses contents of the existing :param potfile:. - Uses mguniq, msgmerge, and msgattrib GNU gettext utilities. + Uses msgmerge, and msgattrib GNU gettext utilities. """ - args = ['msguniq', '--to-code=utf-8'] - if self.wrap: - args.append(self.wrap) - if self.location: - args.append(self.location) - args.append(potfile) - msgs, errors, status = popen_wrapper(args) - if errors: - if status != STATUS_OK: - raise CommandError( - "errors happened while running msguniq\n%s" % errors) - elif self.verbosity > 0: - self.stdout.write(errors) - basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES') if not os.path.isdir(basedir): os.makedirs(basedir) pofile = os.path.join(basedir, '%s.po' % str(self.domain)) if os.path.exists(pofile): - with open(potfile, 'w') as fp: - fp.write(msgs) args = ['msgmerge', '-q'] if self.wrap: args.append(self.wrap) @@ -369,8 +401,10 @@ class Command(NoArgsCommand): "errors happened while running msgmerge\n%s" % errors) elif self.verbosity > 0: self.stdout.write(errors) - elif not self.invoked_for_django: - msgs = self.copy_plural_forms(msgs, locale) + else: + msgs = open(potfile, 'r').read() + if not self.invoked_for_django: + msgs = self.copy_plural_forms(msgs, locale) msgs = msgs.replace( "#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "") with open(pofile, 'w') as fp: diff --git a/docs/man/django-admin.1 b/docs/man/django-admin.1 index f1b568daf5..c72c3520b5 100644 --- a/docs/man/django-admin.1 +++ b/docs/man/django-admin.1 @@ -192,7 +192,8 @@ Ignore files or directories matching this glob-style pattern. Use multiple times to ignore more (makemessages command). .TP .I \-\-no\-default\-ignore -Don't ignore the common private glob-style patterns 'CVS', '.*' and '*~' (makemessages command). +Don't ignore the common private glob-style patterns 'CVS', '.*', '*~' and '*.pyc' +(makemessages command). .TP .I \-\-no\-wrap Don't break long message lines into several lines (makemessages command). diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 02c6cd5851..69555dcb5c 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -557,7 +557,7 @@ Example usage:: Use the ``--ignore`` or ``-i`` option to ignore files or directories matching the given :mod:`glob`-style pattern. Use multiple times to ignore more. -These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'`` +These patterns are used by default: ``'CVS'``, ``'.*'``, ``'*~'``, ``'*.pyc'`` Example usage:: @@ -584,7 +584,7 @@ for technically skilled translators to understand each message's context. .. versionadded:: 1.6 Use the ``--keep-pot`` option to prevent Django from deleting the temporary -.pot file it generates before creating the .po file. This is useful for +.pot files it generates before creating the .po file. This is useful for debugging errors which may prevent the final language files from being created. makemigrations [] diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 0767659540..1041edb37b 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -375,6 +375,11 @@ Internationalization in the corresponding entry in the PO file, which makes the translation process easier. +* When you run :djadmin:`makemessages` from the root directory of your project, + any extracted strings will now be automatically distributed to the proper + app or project message file. See :ref:`how-to-create-language-files` for + details. + Management Commands ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 8a484702e7..66b9a99e21 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1256,6 +1256,17 @@ is configured correctly). It creates (or updates) a message file in the directory ``locale/LANG/LC_MESSAGES``. In the ``de`` example, the file will be ``locale/de/LC_MESSAGES/django.po``. +.. versionchanged:: 1.7 + + When you run ``makemessages`` from the root directory of your project, the + extracted strings will be automatically distributed to the proper message + files. That is, a string extracted from a file of an app containing a + ``locale`` directory will go in a message file under that directory. + A string extracted from a file of an app without any ``locale`` directory + will either go in a message file under the directory listed first in + :setting:`LOCALE_PATHS` or will generate an error if :setting:`LOCALE_PATHS` + is empty. + By default :djadmin:`django-admin.py makemessages ` examines every file that has the ``.html`` or ``.txt`` file extension. In case you want to override that default, use the ``--extension`` or ``-e`` option to specify the @@ -1730,24 +1741,9 @@ All message file repositories are structured the same way. They are: * ``$PYTHONPATH/django/conf/locale//LC_MESSAGES/django.(po|mo)`` To create message files, you use the :djadmin:`django-admin.py makemessages ` -tool. You only need to be in the same directory where the ``locale/`` directory -is located. And you use :djadmin:`django-admin.py compilemessages ` +tool. And you use :djadmin:`django-admin.py compilemessages ` to produce the binary ``.mo`` files that are used by ``gettext``. You can also run :djadmin:`django-admin.py compilemessages --settings=path.to.settings ` to make the compiler process all the directories in your :setting:`LOCALE_PATHS` setting. - -Finally, you should give some thought to the structure of your translation -files. If your applications need to be delivered to other users and will be used -in other projects, you might want to use app-specific translations. But using -app-specific translations and project-specific translations could produce weird -problems with :djadmin:`makemessages`: it will traverse all directories below -the current path and so might put message IDs into a unified, common message -file for the current project that are already in application message files. - -The easiest way out is to store applications that are not part of the project -(and so carry their own translations) outside the project tree. That way, -:djadmin:`django-admin.py makemessages `, when ran on a project -level will only extract strings that are connected to your explicit project and -not strings that are distributed independently. diff --git a/tests/i18n/project_dir/__init__.py b/tests/i18n/project_dir/__init__.py new file mode 100644 index 0000000000..b32b258e37 --- /dev/null +++ b/tests/i18n/project_dir/__init__.py @@ -0,0 +1,4 @@ +# Sample project used by test_extraction.CustomLayoutExtractionTests +from django.utils.translation import ugettext as _ + +string = _("This is a project-level string") diff --git a/tests/i18n/project_dir/app_no_locale/__init__.py b/tests/i18n/project_dir/app_no_locale/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_no_locale/models.py b/tests/i18n/project_dir/app_no_locale/models.py new file mode 100644 index 0000000000..06dfbaa5d4 --- /dev/null +++ b/tests/i18n/project_dir/app_no_locale/models.py @@ -0,0 +1,3 @@ +from django.utils.translation import ugettext as _ + +string = _("This app has no locale directory") diff --git a/tests/i18n/project_dir/app_with_locale/__init__.py b/tests/i18n/project_dir/app_with_locale/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_with_locale/locale/.gitkeep b/tests/i18n/project_dir/app_with_locale/locale/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/project_dir/app_with_locale/models.py b/tests/i18n/project_dir/app_with_locale/models.py new file mode 100644 index 0000000000..ab35d5002a --- /dev/null +++ b/tests/i18n/project_dir/app_with_locale/models.py @@ -0,0 +1,3 @@ +from django.utils.translation import ugettext as _ + +string = _("This app has a locale directory") diff --git a/tests/i18n/project_dir/project_locale/.gitkeep b/tests/i18n/project_dir/project_locale/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py index d2af8b0cc8..1637e29f67 100644 --- a/tests/i18n/test_extraction.py +++ b/tests/i18n/test_extraction.py @@ -8,9 +8,11 @@ import shutil from unittest import SkipTest, skipUnless import warnings +from django.conf import settings from django.core import management from django.core.management.utils import find_command from django.test import SimpleTestCase +from django.test.utils import override_settings from django.utils.encoding import force_text from django.utils._os import upath from django.utils import six @@ -497,3 +499,47 @@ class MultipleLocaleExtractionTests(ExtractorTests): management.call_command('makemessages', locale=['pt', 'de'], verbosity=0) self.assertTrue(os.path.exists(self.PO_FILE_PT)) self.assertTrue(os.path.exists(self.PO_FILE_DE)) + + +class CustomLayoutExtractionTests(ExtractorTests): + def setUp(self): + self._cwd = os.getcwd() + self.test_dir = os.path.join(os.path.dirname(upath(__file__)), 'project_dir') + + def test_no_locale_raises(self): + os.chdir(self.test_dir) + with six.assertRaisesRegex(self, management.CommandError, + "Unable to find a locale path to store translations for file"): + management.call_command('makemessages', locale=LOCALE, verbosity=0) + + @override_settings( + LOCALE_PATHS=(os.path.join( + os.path.dirname(upath(__file__)), 'project_dir', 'project_locale'),) + ) + def test_project_locale_paths(self): + """ + Test that: + * translations for an app containing a locale folder are stored in that folder + * translations outside of that app are in LOCALE_PATHS[0] + """ + os.chdir(self.test_dir) + self.addCleanup(shutil.rmtree, + os.path.join(settings.LOCALE_PATHS[0], LOCALE), True) + self.addCleanup(shutil.rmtree, + os.path.join(self.test_dir, 'app_with_locale', 'locale', LOCALE), True) + + management.call_command('makemessages', locale=[LOCALE], verbosity=0) + project_de_locale = os.path.join( + self.test_dir, 'project_locale', 'de', 'LC_MESSAGES', 'django.po') + app_de_locale = os.path.join( + self.test_dir, 'app_with_locale', 'locale', 'de', 'LC_MESSAGES', 'django.po') + self.assertTrue(os.path.exists(project_de_locale)) + self.assertTrue(os.path.exists(app_de_locale)) + + with open(project_de_locale, 'r') as fp: + po_contents = force_text(fp.read()) + self.assertMsgId('This app has no locale directory', po_contents) + self.assertMsgId('This is a project-level string', po_contents) + with open(app_de_locale, 'r') as fp: + po_contents = force_text(fp.read()) + self.assertMsgId('This app has a locale directory', po_contents) From 6310d658b7c8a39747a8cda3e3b8716266cffb0d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 30 Nov 2013 11:30:56 +0100 Subject: [PATCH 016/158] Closed file after reading pot file Faulty line was introduced in 50a8ab7cd1. --- django/core/management/commands/makemessages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index 4994aaf6ba..a42ba96444 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -402,7 +402,8 @@ class Command(NoArgsCommand): elif self.verbosity > 0: self.stdout.write(errors) else: - msgs = open(potfile, 'r').read() + with open(potfile, 'r') as fp: + msgs = fp.read() if not self.invoked_for_django: msgs = self.copy_plural_forms(msgs, locale) msgs = msgs.replace( From ec73ce5d8a0bc3643188d97629c30d226849bddc Mon Sep 17 00:00:00 2001 From: Alex Hill Date: Sat, 30 Nov 2013 17:19:24 +0800 Subject: [PATCH 017/158] Fixed comment typos. --- django/db/models/sql/query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index 5f24e488cb..db1a630a66 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -1232,7 +1232,7 @@ class Query(object): q_object.clone(), q_object.negated) # For join promotion this case is doing an AND for the added q_object # and existing conditions. So, any existing inner join forces the join - # type to remain inner. Exsting outer joins can however be demoted. + # type to remain inner. Existing outer joins can however be demoted. # (Consider case where rel_a is LOUTER and rel_a__col=1 is added - if # rel_a doesn't produce any rows, then the whole condition must fail. # So, demotion is OK. @@ -1279,7 +1279,7 @@ class Query(object): single name in 'names' can generate multiple PathInfos (m2m for example). - 'names' is the path of names to travle, 'opts' is the model Options we + 'names' is the path of names to travel, 'opts' is the model Options we start the name resolving from, 'allow_many' is as for setup_joins(). Returns a list of PathInfo tuples. In addition returns the final field From c0a2daad78c66def2ce6592977b78dab6475dd53 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 29 Nov 2013 18:39:19 -0600 Subject: [PATCH 018/158] Documented the house style for gender neutral pronouns. --- .../contributing/writing-documentation.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/internals/contributing/writing-documentation.txt b/docs/internals/contributing/writing-documentation.txt index e565b4d54f..ba53a1a3d6 100644 --- a/docs/internals/contributing/writing-documentation.txt +++ b/docs/internals/contributing/writing-documentation.txt @@ -65,6 +65,19 @@ Primer `. After that, you'll want to read about the :ref:`Sphinx-specific markup ` that's used to manage metadata, indexing, and cross-references. +Writing style +------------- + +When using pronouns in reference to a hypothetical person, such as "a user with +a session cookie", gender neutral pronouns (they/their/them) should be used. +Instead of: + +* he or she... use they. +* him or her... use them. +* his or her... use their. +* his or hers... use theirs. +* himself or herself... use themselves. + Commonly used terms ------------------- From f3e7ab366c597571198dc8d024f09e619991bac4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 30 Nov 2013 08:37:15 -0500 Subject: [PATCH 019/158] Removed gender-based pronouns per [c0a2daad78]. --- django/contrib/auth/tests/test_forms.py | 2 +- docs/ref/clickjacking.txt | 4 ++-- docs/ref/contrib/sites.txt | 2 +- docs/ref/settings.txt | 2 +- docs/releases/1.3-beta-1.txt | 3 +-- docs/releases/1.4.6.txt | 2 +- docs/releases/1.4.txt | 2 +- docs/releases/1.5.2.txt | 2 +- docs/topics/cache.txt | 8 ++++---- docs/topics/db/aggregation.txt | 2 +- docs/topics/http/sessions.txt | 6 +++--- docs/topics/i18n/translation.txt | 4 ++-- 12 files changed, 19 insertions(+), 20 deletions(-) diff --git a/django/contrib/auth/tests/test_forms.py b/django/contrib/auth/tests/test_forms.py index bf9a002770..41cb3350b1 100644 --- a/django/contrib/auth/tests/test_forms.py +++ b/django/contrib/auth/tests/test_forms.py @@ -133,7 +133,7 @@ class AuthenticationFormTest(TestCase): [force_text(form.error_messages['inactive'])]) def test_custom_login_allowed_policy(self): - # The user is inactive, but our custom form policy allows him to log in. + # The user is inactive, but our custom form policy allows them to log in. data = { 'username': 'inactive', 'password': 'password', diff --git a/docs/ref/clickjacking.txt b/docs/ref/clickjacking.txt index cb3ac1bcd1..bb50f97f08 100644 --- a/docs/ref/clickjacking.txt +++ b/docs/ref/clickjacking.txt @@ -20,8 +20,8 @@ purchase an item. A user has chosen to stay logged into the store all the time for convenience. An attacker site might create an "I Like Ponies" button on one of their own pages, and load the store's page in a transparent iframe such that the "Buy Now" button is invisibly overlaid on the "I Like Ponies" button. If the -user visits the attacker site and clicks "I Like Ponies" he or she will inadvertently -click on the online store's "Buy Now" button and unknowingly purchase the item. +user visits the attacker's site, clicking "I Like Ponies" will cause an +inadvertent click on the "Buy Now" button and an unknowing purchase of the item. .. _clickjacking-prevention: diff --git a/docs/ref/contrib/sites.txt b/docs/ref/contrib/sites.txt index 225475803f..24b6294278 100644 --- a/docs/ref/contrib/sites.txt +++ b/docs/ref/contrib/sites.txt @@ -172,7 +172,7 @@ Getting the current domain for display LJWorld.com and Lawrence.com both have email alert functionality, which lets readers sign up to get notifications when news happens. It's pretty basic: A -reader signs up on a Web form, and he or she immediately gets an email saying, +reader signs up on a Web form and immediately gets an email saying, "Thanks for your subscription." It'd be inefficient and redundant to implement this signup-processing code diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index a1207fb0f8..d9cb07e9b3 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2468,7 +2468,7 @@ SESSION_EXPIRE_AT_BROWSER_CLOSE Default: ``False`` -Whether to expire the session when the user closes his or her browser. See +Whether to expire the session when the user closes their browser. See :ref:`browser-length-vs-persistent-sessions`. .. setting:: SESSION_FILE_PATH diff --git a/docs/releases/1.3-beta-1.txt b/docs/releases/1.3-beta-1.txt index 8719221ddd..907625a7f4 100644 --- a/docs/releases/1.3-beta-1.txt +++ b/docs/releases/1.3-beta-1.txt @@ -73,8 +73,7 @@ The Django admin has long had an undocumented "feature" allowing savvy users to manipulate the query string of changelist pages to filter the list of objects displayed. However, this also creates a security issue, as a staff user with sufficient knowledge of model structure -could use this "feature" to gain access to information he or she would -not normally have. +could use this "feature" to gain access to information not normally accessible. As a result, changelist filtering now explicitly validates all lookup arguments in the query string, and permits only fields which are diff --git a/docs/releases/1.4.6.txt b/docs/releases/1.4.6.txt index e3ccbbb344..cac640ad97 100644 --- a/docs/releases/1.4.6.txt +++ b/docs/releases/1.4.6.txt @@ -19,7 +19,7 @@ The security checks for these redirects (namely ``django.util.http.is_safe_url()``) didn't check if the scheme is ``http(s)`` and as such allowed ``javascript:...`` URLs to be entered. If a developer relied on ``is_safe_url()`` to provide safe redirect targets and put such a -URL into a link, he or she could suffer from a XSS attack. This bug doesn't affect +URL into a link, they could suffer from a XSS attack. This bug doesn't affect Django currently, since we only put this URL into the ``Location`` response header and browsers seem to ignore JavaScript there. diff --git a/docs/releases/1.4.txt b/docs/releases/1.4.txt index 1ba2c6ac7f..f8c23728b9 100644 --- a/docs/releases/1.4.txt +++ b/docs/releases/1.4.txt @@ -811,7 +811,7 @@ instance: * Consequences: The user will see an error about the form having expired and will be sent back to the first page of the wizard, losing the data - he or she has entered so far. + entered so far. * Time period: The amount of time you expect users to take filling out the affected forms. diff --git a/docs/releases/1.5.2.txt b/docs/releases/1.5.2.txt index 2a2cf9195d..6414aab509 100644 --- a/docs/releases/1.5.2.txt +++ b/docs/releases/1.5.2.txt @@ -16,7 +16,7 @@ The security checks for these redirects (namely ``django.util.http.is_safe_url()``) didn't check if the scheme is ``http(s)`` and as such allowed ``javascript:...`` URLs to be entered. If a developer relied on ``is_safe_url()`` to provide safe redirect targets and put such a -URL into a link, he or she could suffer from a XSS attack. This bug doesn't affect +URL into a link, they could suffer from a XSS attack. This bug doesn't affect Django currently, since we only put this URL into the ``Location`` response header and browsers seem to ignore JavaScript there. diff --git a/docs/topics/cache.txt b/docs/topics/cache.txt index e08ea5f960..7ef7285fb7 100644 --- a/docs/topics/cache.txt +++ b/docs/topics/cache.txt @@ -1123,10 +1123,10 @@ Controlling cache: Using other headers Other problems with caching are the privacy of data and the question of where data should be stored in a cascade of caches. -A user usually faces two kinds of caches: his or her own browser cache (a -private cache) and his or her provider's cache (a public cache). A public cache -is used by multiple users and controlled by someone else. This poses problems -with sensitive data--you don't want, say, your bank account number stored in a +A user usually faces two kinds of caches: their own browser cache (a private +cache) and their provider's cache (a public cache). A public cache is used by +multiple users and controlled by someone else. This poses problems with +sensitive data--you don't want, say, your bank account number stored in a public cache. So Web applications need a way to tell caches which data is private and which is public. diff --git a/docs/topics/db/aggregation.txt b/docs/topics/db/aggregation.txt index 1024d6b0c2..9e02b50c67 100644 --- a/docs/topics/db/aggregation.txt +++ b/docs/topics/db/aggregation.txt @@ -241,7 +241,7 @@ such alias were specified, it would be the rather long ``'book__pubdate__min'``. This doesn't apply just to foreign keys. It also works with many-to-many relations. For example, we can ask for every author, annotated with the total -number of pages considering all the books he/she has (co-)authored (note how we +number of pages considering all the books the author has (co-)authored (note how we use ``'book'`` to specify the ``Author`` -> ``Book`` reverse many-to-many hop):: >>> Author.objects.annotate(total_pages=Sum('book__pages')) diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index d39b8344c6..f7e8807945 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -166,7 +166,7 @@ and the :setting:`SECRET_KEY` setting. cookie backend might open you up to `replay attacks`_. Unlike other session backends which keep a server-side record of each session and invalidate it when a user logs out, cookie-based sessions are not invalidated when a user - logs out. Thus if an attacker steals a user's cookie, he or she can use that + logs out. Thus if an attacker steals a user's cookie, they can use that cookie to login as that user even if the user logs out. Cookies will only be detected as 'stale' if they are older than your :setting:`SESSION_COOKIE_AGE`. @@ -590,8 +590,8 @@ log in every time they open a browser. If :setting:`SESSION_EXPIRE_AT_BROWSER_CLOSE` is set to ``True``, Django will use browser-length cookies -- cookies that expire as soon as the user closes -his or her browser. Use this if you want people to have to log in every time -they open a browser. +their browser. Use this if you want people to have to log in every time they +open a browser. This setting is a global default and can be overwritten at a per-session level by explicitly calling the :meth:`~backends.base.SessionBase.set_expiry` method diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 66b9a99e21..6e56831a6b 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1579,8 +1579,8 @@ If all you want is to run Django with your native language all you need to do is set :setting:`LANGUAGE_CODE` and make sure the corresponding :term:`message files ` and their compiled versions (``.mo``) exist. -If you want to let each individual user specify which language he or she -prefers, then you also need to use use the ``LocaleMiddleware``. +If you want to let each individual user specify which language they +prefer, then you also need to use use the ``LocaleMiddleware``. ``LocaleMiddleware`` enables language selection based on data from the request. It customizes content for each user. From 2080bce695d480dd5fd851fdada221df923aa290 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 30 Nov 2013 19:07:35 +0100 Subject: [PATCH 020/158] Fixed #21358 -- Allowed runserver on non-English locales Thanks svartalf for the report. --- django/core/management/commands/runserver.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py index 402b3d342c..503cff220f 100644 --- a/django/core/management/commands/runserver.py +++ b/django/core/management/commands/runserver.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + from optparse import make_option from datetime import datetime import errno @@ -9,6 +11,7 @@ import socket from django.core.management.base import BaseCommand, CommandError from django.core.servers.basehttp import run, get_internal_wsgi_application from django.utils import autoreload +from django.utils import six naiveip_re = re.compile(r"""^(?: (?P @@ -96,13 +99,17 @@ class Command(BaseCommand): self.stdout.write("Validating models...\n\n") self.validate(display_num_errors=True) + now = datetime.now().strftime('%B %d, %Y - %X') + if six.PY2: + now = now.decode('utf-8') + self.stdout.write(( "%(started_at)s\n" "Django version %(version)s, using settings %(settings)r\n" "Starting development server at http://%(addr)s:%(port)s/\n" "Quit the server with %(quit_command)s.\n" ) % { - "started_at": datetime.now().strftime('%B %d, %Y - %X'), + "started_at": now, "version": self.get_version(), "settings": settings.SETTINGS_MODULE, "addr": '[%s]' % self.addr if self._raw_ipv6 else self.addr, From 2688462f914960b568dce4cb529db77e982a5284 Mon Sep 17 00:00:00 2001 From: Unai Zalakain Date: Thu, 28 Nov 2013 14:22:07 +0100 Subject: [PATCH 021/158] Refs #21230 -- removed direct settings manipulation from template tests --- tests/template_tests/tests.py | 71 +++++++++++++++-------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/tests/template_tests/tests.py b/tests/template_tests/tests.py index f54fc6fcba..651d46d4f1 100644 --- a/tests/template_tests/tests.py +++ b/tests/template_tests/tests.py @@ -544,7 +544,11 @@ class TemplateRegressionTests(TestCase): t.render(Context({})) -@override_settings(MEDIA_URL="/media/", STATIC_URL="/static/") +# Set ALLOWED_INCLUDE_ROOTS so that ssi works. +@override_settings(MEDIA_URL="/media/", STATIC_URL="/static/", + TEMPLATE_DEBUG=False, ALLOWED_INCLUDE_ROOTS=( + os.path.dirname(os.path.abspath(upath(__file__))),), + ) class TemplateTests(TransRealMixin, TestCase): def test_templates(self): template_tests = self.get_template_tests() @@ -565,19 +569,9 @@ class TemplateTests(TransRealMixin, TestCase): failures = [] tests = sorted(template_tests.items()) - # Turn TEMPLATE_DEBUG off, because tests assume that. - old_td, settings.TEMPLATE_DEBUG = settings.TEMPLATE_DEBUG, False - # Set TEMPLATE_STRING_IF_INVALID to a known string. - old_invalid = settings.TEMPLATE_STRING_IF_INVALID expected_invalid_str = 'INVALID' - # Set ALLOWED_INCLUDE_ROOTS so that ssi works. - old_allowed_include_roots = settings.ALLOWED_INCLUDE_ROOTS - settings.ALLOWED_INCLUDE_ROOTS = ( - os.path.dirname(os.path.abspath(upath(__file__))), - ) - # Warm the URL reversing cache. This ensures we don't pay the cost # warming the cache during one of the tests. urlresolvers.reverse('template_tests.views.client_action', @@ -613,34 +607,34 @@ class TemplateTests(TransRealMixin, TestCase): (expected_invalid_str, False, invalid_string_result), ('', True, template_debug_result) ]: - settings.TEMPLATE_STRING_IF_INVALID = invalid_str - settings.TEMPLATE_DEBUG = template_debug - for is_cached in (False, True): - try: + with override_settings(TEMPLATE_STRING_IF_INVALID=invalid_str, + TEMPLATE_DEBUG=template_debug): + for is_cached in (False, True): try: - with warnings.catch_warnings(): - # Ignore pending deprecations of the old syntax of the 'cycle' and 'firstof' tags. - warnings.filterwarnings("ignore", category=DeprecationWarning, module='django.template.base') - test_template = loader.get_template(name) - except ShouldNotExecuteException: - failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Template loading invoked method that shouldn't have been invoked." % (is_cached, invalid_str, template_debug, name)) + try: + with warnings.catch_warnings(): + # Ignore pending deprecations of the old syntax of the 'cycle' and 'firstof' tags. + warnings.filterwarnings("ignore", category=DeprecationWarning, module='django.template.base') + test_template = loader.get_template(name) + except ShouldNotExecuteException: + failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Template loading invoked method that shouldn't have been invoked." % (is_cached, invalid_str, template_debug, name)) - try: - output = self.render(test_template, vals) - except ShouldNotExecuteException: - failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Template rendering invoked method that shouldn't have been invoked." % (is_cached, invalid_str, template_debug, name)) - except ContextStackException: - failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Context stack was left imbalanced" % (is_cached, invalid_str, template_debug, name)) - continue - except Exception: - exc_type, exc_value, exc_tb = sys.exc_info() - if exc_type != result: - tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb)) - failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Got %s, exception: %s\n%s" % (is_cached, invalid_str, template_debug, name, exc_type, exc_value, tb)) - continue - if output != result: - failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Expected %r, got %r" % (is_cached, invalid_str, template_debug, name, result, output)) - cache_loader.reset() + try: + output = self.render(test_template, vals) + except ShouldNotExecuteException: + failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Template rendering invoked method that shouldn't have been invoked." % (is_cached, invalid_str, template_debug, name)) + except ContextStackException: + failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Context stack was left imbalanced" % (is_cached, invalid_str, template_debug, name)) + continue + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + if exc_type != result: + tb = '\n'.join(traceback.format_exception(exc_type, exc_value, exc_tb)) + failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Got %s, exception: %s\n%s" % (is_cached, invalid_str, template_debug, name, exc_type, exc_value, tb)) + continue + if output != result: + failures.append("Template test (Cached='%s', TEMPLATE_STRING_IF_INVALID='%s', TEMPLATE_DEBUG=%s): %s -- FAILED. Expected %r, got %r" % (is_cached, invalid_str, template_debug, name, result, output)) + cache_loader.reset() if 'LANGUAGE_CODE' in vals[1]: deactivate() @@ -651,9 +645,6 @@ class TemplateTests(TransRealMixin, TestCase): restore_template_loaders() deactivate() - settings.TEMPLATE_DEBUG = old_td - settings.TEMPLATE_STRING_IF_INVALID = old_invalid - settings.ALLOWED_INCLUDE_ROOTS = old_allowed_include_roots self.assertEqual(failures, [], "Tests failed:\n%s\n%s" % ('-' * 70, ("\n%s\n" % ('-' * 70)).join(failures))) From fddb0131d37109c809ec391e1a134ef1d9e442a7 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 29 Nov 2013 20:49:56 -0500 Subject: [PATCH 022/158] Fixed #21535 -- Fixed password hash iteration upgrade. Thanks jared_mess for the report. --- django/contrib/auth/hashers.py | 2 +- django/contrib/auth/tests/test_hashers.py | 36 +++++++++++++++++++++-- docs/releases/1.6.1.txt | 1 + 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/django/contrib/auth/hashers.py b/django/contrib/auth/hashers.py index a4e8133460..713240b502 100644 --- a/django/contrib/auth/hashers.py +++ b/django/contrib/auth/hashers.py @@ -57,7 +57,7 @@ def check_password(password, encoded, setter=None, preferred='default'): must_update = hasher.algorithm != preferred.algorithm if not must_update: - must_update = hasher.must_update(encoded) + must_update = preferred.must_update(encoded) is_correct = hasher.verify(password, encoded) if setter and is_correct and must_update: setter(password) diff --git a/django/contrib/auth/tests/test_hashers.py b/django/contrib/auth/tests/test_hashers.py index d76a81b1a3..58628cd6cd 100644 --- a/django/contrib/auth/tests/test_hashers.py +++ b/django/contrib/auth/tests/test_hashers.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import unittest from unittest import skipUnless from django.conf.global_settings import PASSWORD_HASHERS as default_hashers from django.contrib.auth.hashers import (is_password_usable, BasePasswordHasher, check_password, make_password, PBKDF2PasswordHasher, load_hashers, PBKDF2SHA1PasswordHasher, get_hasher, identify_hasher, UNUSABLE_PASSWORD_PREFIX, UNUSABLE_PASSWORD_SUFFIX_LENGTH) +from django.test import SimpleTestCase from django.utils import six @@ -22,7 +22,11 @@ except ImportError: bcrypt = None -class TestUtilsHashPass(unittest.TestCase): +class PBKDF2SingleIterationHasher(PBKDF2PasswordHasher): + iterations = 1 + + +class TestUtilsHashPass(SimpleTestCase): def setUp(self): load_hashers(password_hashers=default_hashers) @@ -279,6 +283,34 @@ class TestUtilsHashPass(unittest.TestCase): finally: hasher.iterations = old_iterations + def test_pbkdf2_upgrade_new_hasher(self): + self.assertEqual('pbkdf2_sha256', get_hasher('default').algorithm) + hasher = get_hasher('default') + self.assertNotEqual(hasher.iterations, 1) + + state = {'upgraded': False} + + def setter(password): + state['upgraded'] = True + + with self.settings(PASSWORD_HASHERS=[ + 'django.contrib.auth.tests.test_hashers.PBKDF2SingleIterationHasher']): + encoded = make_password('letmein') + algo, iterations, salt, hash = encoded.split('$', 3) + self.assertEqual(iterations, '1') + + # Check that no upgrade is triggerd + self.assertTrue(check_password('letmein', encoded, setter)) + self.assertFalse(state['upgraded']) + + # Revert to the old iteration count and check if the password would get + # updated to the new iteration count. + with self.settings(PASSWORD_HASHERS=[ + 'django.contrib.auth.hashers.PBKDF2PasswordHasher', + 'django.contrib.auth.tests.test_hashers.PBKDF2SingleIterationHasher']): + self.assertTrue(check_password('letmein', encoded, setter)) + self.assertTrue(state['upgraded']) + def test_load_library_no_algorithm(self): with self.assertRaises(ValueError) as e: BasePasswordHasher()._load_library() diff --git a/docs/releases/1.6.1.txt b/docs/releases/1.6.1.txt index f7c76afbb1..4b23737d94 100644 --- a/docs/releases/1.6.1.txt +++ b/docs/releases/1.6.1.txt @@ -40,3 +40,4 @@ Bug fixes * Fixed test client ``logout()`` method when using the cookie-based session backend (#21448). * Fixed a crash when a ``GeometryField`` uses a non-geometric widget (#21496). +* Fixed password hash upgrade when changing the iteration count (#21535). From 3b60ffa334e68cf44ac79c2997c05a2d14da15d5 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sun, 1 Dec 2013 14:21:57 -0500 Subject: [PATCH 023/158] Fixed incorrect type for max_length. --- tests/foreign_object/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/foreign_object/models.py b/tests/foreign_object/models.py index 7673d27af6..5ec6f6d28b 100644 --- a/tests/foreign_object/models.py +++ b/tests/foreign_object/models.py @@ -162,7 +162,7 @@ class NewsArticle(Article): class ArticleTranslation(models.Model): article = models.ForeignKey(Article) - lang = models.CharField(max_length='2') + lang = models.CharField(max_length=2) title = models.CharField(max_length=100) body = models.TextField() abstract = models.CharField(max_length=400, null=True) From b8be3055f1d81ee956ea1572c48cc352b63099db Mon Sep 17 00:00:00 2001 From: Krzysztof Jurewicz Date: Sun, 1 Dec 2013 20:12:48 +0100 Subject: [PATCH 024/158] Fixed #21543 -- Removed base_dir attribute in StaticFilesHandler. This code seems to be an artifact of AdminMediaHandler removed in [5c53e30607]. --- django/contrib/staticfiles/handlers.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/django/contrib/staticfiles/handlers.py b/django/contrib/staticfiles/handlers.py index 3fe0c9da26..bb54db6b7d 100644 --- a/django/contrib/staticfiles/handlers.py +++ b/django/contrib/staticfiles/handlers.py @@ -12,18 +12,11 @@ class StaticFilesHandler(WSGIHandler): WSGI middleware that intercepts calls to the static files directory, as defined by the STATIC_URL setting, and serves those files. """ - def __init__(self, application, base_dir=None): + def __init__(self, application): self.application = application - if base_dir: - self.base_dir = base_dir - else: - self.base_dir = self.get_base_dir() self.base_url = urlparse(self.get_base_url()) super(StaticFilesHandler, self).__init__() - def get_base_dir(self): - return settings.STATIC_ROOT - def get_base_url(self): utils.check_settings() return settings.STATIC_URL From 9ccde8cfafda03c961343b4523ac035428c4ff25 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Tue, 3 Dec 2013 00:12:04 +0700 Subject: [PATCH 025/158] Fixed #21545 -- autoreload kqueue events weren't cleared which caused an infinite loop. --- django/utils/autoreload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/utils/autoreload.py b/django/utils/autoreload.py index 4b0a97f12b..3c71fa2dbe 100644 --- a/django/utils/autoreload.py +++ b/django/utils/autoreload.py @@ -172,7 +172,7 @@ def kqueue_code_changed(): # Utility function to create kevents. _filter = select.KQ_FILTER_VNODE - flags = select.KQ_EV_ADD + flags = select.KQ_EV_ADD | select.KQ_EV_CLEAR fflags = select.KQ_NOTE_DELETE | select.KQ_NOTE_WRITE | select.KQ_NOTE_RENAME def make_kevent(descriptor): From c75dd664cf70a159257337c3eda978de2bec9e7a Mon Sep 17 00:00:00 2001 From: Alasdair Nicol Date: Mon, 2 Dec 2013 17:21:48 +0000 Subject: [PATCH 026/158] Fixed #21538 -- Added numpy to test/requirements/base.txt Thanks Tim Graham for the report --- docs/internals/contributing/writing-code/unit-tests.txt | 2 ++ tests/requirements/base.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index e83d0bef17..969635f2ef 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -154,6 +154,7 @@ dependencies: * bcrypt_ * docutils_ +* numpy_ * Pillow_ * PyYAML_ * pytz_ @@ -182,6 +183,7 @@ associated tests will be skipped. .. _bcrypt: https://pypi.python.org/pypi/bcrypt .. _docutils: https://pypi.python.org/pypi/docutils +.. _numpy: https://pypi.python.org/pypi/numpy .. _Pillow: https://pypi.python.org/pypi/Pillow/ .. _PyYAML: http://pyyaml.org/wiki/PyYAML .. _pytz: https://pypi.python.org/pypi/pytz/ diff --git a/tests/requirements/base.txt b/tests/requirements/base.txt index 0f439f5919..3d982bc00a 100644 --- a/tests/requirements/base.txt +++ b/tests/requirements/base.txt @@ -1,5 +1,6 @@ bcrypt docutils +numpy Pillow PyYAML pytz > dev From 12615dab78cb6fc7d8c74b7b65a4136b0feeb33f Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Mon, 2 Dec 2013 23:11:59 -0300 Subject: [PATCH 027/158] Fixed #13476 -- Added support for color in console output under Windows. Detect and use the services of the ANSICON third-party tool if it's available. --- django/core/management/color.py | 6 ++++-- docs/ref/django-admin.txt | 13 +++++++++++++ docs/releases/1.7.txt | 3 +++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/django/core/management/color.py b/django/core/management/color.py index 20e31fff93..3890a4546f 100644 --- a/django/core/management/color.py +++ b/django/core/management/color.py @@ -13,10 +13,12 @@ def supports_color(): Returns True if the running system's terminal supports color, and False otherwise. """ - unsupported_platform = (sys.platform in ('win32', 'Pocket PC')) + plat = sys.platform + supported_platform = plat != 'Pocket PC' and (plat != 'win32' or + 'ANSICON' in os.environ) # isatty is not always implemented, #6223. is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - if unsupported_platform or not is_a_tty: + if not supported_platform or not is_a_tty: return False return True diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 69555dcb5c..cac7a5bc22 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1552,6 +1552,11 @@ color-coded output if your terminal supports ANSI-colored output. It won't use the color codes if you're piping the command's output to another program. +Under Windows, the native console doesn't support ANSI escape sequences so by +default there is no color output. But you can install the `ANSICON`_ +third-party tool, the Django commands will detect its presence and will make +use of its services to color output just like on Unix-based platforms. + The colors used for syntax highlighting can be customized. Django ships with three color palettes: @@ -1636,6 +1641,14 @@ would specify the use of all the colors in the light color palette, *except* for the colors for errors and notices which would be overridden as specified. +.. versionadded:: 1.7 + +Support for color-coded output from ``django-admin.py`` / ``manage.py`` +utilities on Windows by relying on the ANSICON application was added in Django +1.7. + +.. _ANSICON: http://adoxa.hostmyway.net/ansicon/ + Bash completion --------------- diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 1041edb37b..0d37a8d99a 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -415,6 +415,9 @@ Management Commands * All HTTP requests are logged to the console, including requests for static files or ``favicon.ico`` that used to be filtered out. +* Management commands can now produce syntax colored output under Windows if + the ANSICON third-party tool is installed and active. + Models ^^^^^^ From 4d738fcc3bad4e13c3667646a86f9a17603b7d23 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Wed, 4 Dec 2013 10:11:42 +0100 Subject: [PATCH 028/158] Fixed #21546 -- Strengthened kqueue detection. Thanks Loic Bistuer. --- django/utils/autoreload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/utils/autoreload.py b/django/utils/autoreload.py index 3c71fa2dbe..f26c2c600f 100644 --- a/django/utils/autoreload.py +++ b/django/utils/autoreload.py @@ -82,7 +82,7 @@ try: import subprocess command = ["sysctl", "-n", "kern.maxfilesperproc"] NOFILES_KERN = int(subprocess.check_output(command).strip()) -except (AttributeError, OSError): +except Exception: USE_KQUEUE = False RUN_RELOADER = True From 91c38ce4b236e6eeb5f6f636250df7316fa766bd Mon Sep 17 00:00:00 2001 From: Bouke Haarsma Date: Fri, 8 Nov 2013 15:44:37 +0100 Subject: [PATCH 029/158] Fixed 21406 -- Made blocktrans 'trimmed' option preserve line numbers. Thanks Bouke Haarsma for report, fix and initial patch. --- django/utils/translation/trans_real.py | 31 +++++++----- tests/i18n/commands/templates/test.html | 1 + tests/i18n/test_extraction.py | 64 +++++++++++++++++-------- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 9dfac47381..e547800b92 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -564,6 +564,12 @@ def templatize(src, origin=None): lineno_comment_map = {} comment_lineno_cache = None + def join_tokens(tokens, trim=False): + message = ''.join(tokens) + if trim: + message = trim_whitespace(message) + return message + for t in Lexer(src, origin).tokenize(): if incomment: if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment': @@ -586,29 +592,28 @@ def templatize(src, origin=None): endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: - if trimmed: - singular = trim_whitespace(''.join(singular)) - else: - singular = ''.join(singular) - if inplural: - if trimmed: - plural = trim_whitespace(''.join(plural)) - else: - plural = ''.join(plural) if message_context: - out.write(' npgettext(%r, %r, %r,count) ' % (message_context, singular, plural)) + out.write(' npgettext(%r, %r, %r,count) ' % ( + message_context, + join_tokens(singular, trimmed), + join_tokens(plural, trimmed))) else: - out.write(' ngettext(%r, %r, count) ' % (singular, plural)) + out.write(' ngettext(%r, %r, count) ' % ( + join_tokens(singular, trimmed), + join_tokens(plural, trimmed))) for part in singular: out.write(blankout(part, 'S')) for part in plural: out.write(blankout(part, 'P')) else: if message_context: - out.write(' pgettext(%r, %r) ' % (message_context, singular)) + out.write(' pgettext(%r, %r) ' % ( + message_context, + join_tokens(singular, trimmed))) else: - out.write(' gettext(%r) ' % singular) + out.write(' gettext(%r) ' % join_tokens(singular, + trimmed)) for part in singular: out.write(blankout(part, 'S')) message_context = None diff --git a/tests/i18n/commands/templates/test.html b/tests/i18n/commands/templates/test.html index bd77728549..32920476e2 100644 --- a/tests/i18n/commands/templates/test.html +++ b/tests/i18n/commands/templates/test.html @@ -94,3 +94,4 @@ continued here.{% endcomment %} line breaks, this time should be trimmed. {% endblocktrans %} +{% trans "I'm on line 97" %} diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py index 1637e29f67..da3670ff10 100644 --- a/tests/i18n/test_extraction.py +++ b/tests/i18n/test_extraction.py @@ -66,6 +66,43 @@ class ExtractorTests(SimpleTestCase): msgid = re.escape(msgid) return self.assertTrue(not re.search('^msgid %s' % msgid, s, re.MULTILINE)) + def _assertPoLocComment(self, assert_presence, po_filename, line_number, *comment_parts): + with open(po_filename, 'r') as fp: + po_contents = force_text(fp.read()) + if os.name == 'nt': + # #: .\path\to\file.html:123 + cwd_prefix = '%s%s' % (os.curdir, os.sep) + else: + # #: path/to/file.html:123 + cwd_prefix = '' + parts = ['#: '] + parts.append(os.path.join(cwd_prefix, *comment_parts)) + if line_number is not None: + parts.append(':%d' % line_number) + needle = ''.join(parts) + if assert_presence: + return self.assertTrue(needle in po_contents, '"%s" not found in final .po file.' % needle) + else: + return self.assertFalse(needle in po_contents, '"%s" shouldn\'t be in final .po file.' % needle) + + def assertLocationCommentPresent(self, po_filename, line_number, *comment_parts): + """ + self.assertLocationCommentPresent('django.po', 42, 'dirA', 'dirB', 'foo.py') + + verifies that the django.po file has a gettext-style location comment of the form + + `#: dirA/dirB/foo.py:42` + + (or `#: .\dirA\dirB\foo.py:42` on Windows) + + None can be passed for the line_number argument to skip checking of the :42 suffix part. + """ + return self._assertPoLocComment(True, po_filename, line_number, *comment_parts) + + def assertLocationCommentNotPresent(self, po_filename, line_number, *comment_parts): + """Check the oposite of assertLocationComment()""" + return self._assertPoLocComment(False, po_filename, line_number, *comment_parts) + class BasicExtractorTests(ExtractorTests): @@ -133,6 +170,9 @@ class BasicExtractorTests(ExtractorTests): self.assertNotMsgId('Text with a few line breaks.', po_contents) # should be trimmed self.assertMsgId("Again some text with a few line breaks, this time should be trimmed.", po_contents) + # #21406 -- Should adjust for eaten line numbers + self.assertMsgId("I'm on line 97", po_contents) + self.assertLocationCommentPresent(self.PO_FILE, 97, 'templates', 'test.html') def test_force_en_us_locale(self): """Value of locale-munging option used by the command is the right one""" @@ -418,32 +458,18 @@ class LocationCommentsTests(ExtractorTests): os.chdir(self.test_dir) management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_location=True) self.assertTrue(os.path.exists(self.PO_FILE)) - with open(self.PO_FILE, 'r') as fp: - po_contents = force_text(fp.read()) - needle = os.sep.join(['#: templates', 'test.html:55']) - self.assertFalse(needle in po_contents, '"%s" shouldn\'t be in final .po file.' % needle) + self.assertLocationCommentNotPresent(self.PO_FILE, 55, 'templates', 'test.html.py') def test_no_location_disabled(self): """Behavior is correct if --no-location switch isn't specified.""" os.chdir(self.test_dir) management.call_command('makemessages', locale=[LOCALE], verbosity=0, no_location=False) self.assertTrue(os.path.exists(self.PO_FILE)) - with open(self.PO_FILE, 'r') as fp: - # Standard comment with source file relative path should be present -- #16903 - po_contents = force_text(fp.read()) - if os.name == 'nt': - # #: .\path\to\file.html:123 - cwd_prefix = '%s%s' % (os.curdir, os.sep) - else: - # #: path/to/file.html:123 - cwd_prefix = '' - needle = os.sep.join(['#: %stemplates' % cwd_prefix, 'test.html:55']) - self.assertTrue(needle in po_contents, '"%s" not found in final .po file.' % needle) + # #16903 -- Standard comment with source file relative path should be present + self.assertLocationCommentPresent(self.PO_FILE, 55, 'templates', 'test.html') - # #21208 -- Leaky paths in comments on Windows e.g. #: path\to\file.html.py:123 - bad_suffix = '.py' - bad_string = 'templates%stest.html%s' % (os.sep, bad_suffix) - self.assertFalse(bad_string in po_contents, '"%s" shouldn\'t be in final .po file.' % bad_string) + # #21208 -- Leaky paths in comments on Windows e.g. #: path\to\file.html.py:123 + self.assertLocationCommentNotPresent(self.PO_FILE, None, 'templates', 'test.html.py') class KeepPotFileExtractorTests(ExtractorTests): From abb04f1f3f19f460643177e780004ff199423757 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Wed, 4 Dec 2013 10:26:03 -0300 Subject: [PATCH 030/158] Added link to localized formatting doc from main index. --- docs/index.txt | 3 ++- docs/topics/i18n/formatting.txt | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/index.txt b/docs/index.txt index df6861aa1c..e5c92a2073 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -229,7 +229,8 @@ regions: * :doc:`Overview ` | :doc:`Internationalization ` | - :ref:`Localization ` + :ref:`Localization ` | + :doc:`Localized Web UI formatting and form input ` * :doc:`"Local flavor" ` * :doc:`Time zones ` diff --git a/docs/topics/i18n/formatting.txt b/docs/topics/i18n/formatting.txt index fc3f37de32..e306d4ab44 100644 --- a/docs/topics/i18n/formatting.txt +++ b/docs/topics/i18n/formatting.txt @@ -7,8 +7,9 @@ Format localization Overview ======== -Django's formatting system is capable to display dates, times and numbers in templates using the format specified for the current :term:`locale `. It also handles localized input in forms. +Django's formatting system is capable of displaying dates, times and numbers in +templates using the format specified for the current +:term:`locale `. It also handles localized input in forms. When it's enabled, two users accessing the same content may see dates, times and numbers formatted in different ways, depending on the formats for their current From 1d20e6df9553346c79edd92e6e8934e9c5c4aa2c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 Dec 2013 13:32:46 +0000 Subject: [PATCH 031/158] Migrate prompts if you need makemigrations, runserver prompts for migrate --- django/core/management/commands/migrate.py | 12 ++++++++++++ django/core/management/commands/runserver.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py index 191bc2c39d..3e796b655f 100644 --- a/django/core/management/commands/migrate.py +++ b/django/core/management/commands/migrate.py @@ -14,6 +14,9 @@ from django.core.management.sql import custom_sql_for_model, emit_post_migrate_s from django.db import connections, router, transaction, models, DEFAULT_DB_ALIAS from django.db.migrations.executor import MigrationExecutor from django.db.migrations.loader import MigrationLoader, AmbiguityError +from django.db.migrations.state import ProjectState +from django.db.migrations.autodetector import MigrationAutodetector +from django.db.models.loading import cache from django.utils.module_loading import module_has_submodule @@ -120,6 +123,15 @@ class Command(BaseCommand): if not plan: if self.verbosity >= 1: self.stdout.write(" No migrations needed.") + # If there's changes that aren't in migrations yet, tell them how to fix it. + autodetector = MigrationAutodetector( + executor.loader.graph.project_state(), + ProjectState.from_app_cache(cache), + ) + changes = autodetector.changes(graph=executor.loader.graph) + if changes: + self.stdout.write(self.style.NOTICE(" Your models have changes that are not yet reflected in a migration, and so won't be applied.")) + self.stdout.write(self.style.NOTICE(" Run 'manage.py makemigrations' to make new migrations, and then re-run 'manage.py migrate' to apply them.")) else: executor.migrate(targets, plan, fake=options.get("fake", False)) diff --git a/django/core/management/commands/runserver.py b/django/core/management/commands/runserver.py index 503cff220f..327832ba43 100644 --- a/django/core/management/commands/runserver.py +++ b/django/core/management/commands/runserver.py @@ -10,6 +10,8 @@ import socket from django.core.management.base import BaseCommand, CommandError from django.core.servers.basehttp import run, get_internal_wsgi_application +from django.db import connections, DEFAULT_DB_ALIAS +from django.db.migrations.executor import MigrationExecutor from django.utils import autoreload from django.utils import six @@ -99,10 +101,10 @@ class Command(BaseCommand): self.stdout.write("Validating models...\n\n") self.validate(display_num_errors=True) + self.check_migrations() now = datetime.now().strftime('%B %d, %Y - %X') if six.PY2: now = now.decode('utf-8') - self.stdout.write(( "%(started_at)s\n" "Django version %(version)s, using settings %(settings)r\n" @@ -144,6 +146,16 @@ class Command(BaseCommand): self.stdout.write(shutdown_message) sys.exit(0) + def check_migrations(self): + """ + Checks to see if the set of migrations on disk matches the + migrations in the database. Prints a warning if they don't match. + """ + executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) + plan = executor.migration_plan(executor.loader.graph.leaf_nodes()) + if plan: + self.stdout.write(self.style.NOTICE("\nYou have unapplied migrations; your app may not work properly until they are applied.")) + self.stdout.write(self.style.NOTICE("Run 'python manage.py migrate' to apply them.\n")) # Kept for backward compatibility BaseRunserverCommand = Command From ab587fa51a3be3ffc6f011189c0abab32dec6155 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 Dec 2013 13:55:20 +0000 Subject: [PATCH 032/158] Add --dry-run option to makemigrations --- .../management/commands/makemigrations.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/django/core/management/commands/makemigrations.py b/django/core/management/commands/makemigrations.py index 1079b4269c..bb07160bb6 100644 --- a/django/core/management/commands/makemigrations.py +++ b/django/core/management/commands/makemigrations.py @@ -14,17 +14,18 @@ from django.db.models.loading import cache class Command(BaseCommand): option_list = BaseCommand.option_list + ( - make_option('--empty', action='store_true', dest='empty', default=False, - help='Make a blank migration.'), + make_option('--dry-run', action='store_true', dest='dry_run', default=False, + help="Just show what migrations would be made; don't actually write them."), ) help = "Creates new migration(s) for apps." - usage_str = "Usage: ./manage.py makemigrations [--empty] [app [app ...]]" + usage_str = "Usage: ./manage.py makemigrations [--dry-run] [app [app ...]]" def handle(self, *app_labels, **options): self.verbosity = int(options.get('verbosity')) self.interactive = options.get('interactive') + self.dry_run = options.get('dry_run', False) # Make sure the app they asked for exists app_labels = set(app_labels) @@ -73,15 +74,16 @@ class Command(BaseCommand): for operation in migration.operations: self.stdout.write(" - %s\n" % operation.describe()) # Write it - migrations_directory = os.path.dirname(writer.path) - if not directory_created.get(app_label, False): - if not os.path.isdir(migrations_directory): - os.mkdir(migrations_directory) - init_path = os.path.join(migrations_directory, "__init__.py") - if not os.path.isfile(init_path): - open(init_path, "w").close() - # We just do this once per app - directory_created[app_label] = True - migration_string = writer.as_string() - with open(writer.path, "wb") as fh: - fh.write(migration_string) + if not self.dry_run: + migrations_directory = os.path.dirname(writer.path) + if not directory_created.get(app_label, False): + if not os.path.isdir(migrations_directory): + os.mkdir(migrations_directory) + init_path = os.path.join(migrations_directory, "__init__.py") + if not os.path.isfile(init_path): + open(init_path, "w").close() + # We just do this once per app + directory_created[app_label] = True + migration_string = writer.as_string() + with open(writer.path, "wb") as fh: + fh.write(migration_string) From df800b160990884246d73786e85b082d9b703b57 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 Dec 2013 13:55:45 +0000 Subject: [PATCH 033/158] Add clone() method to Field to get clean copies of it. --- django/db/models/fields/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 1fa230e1c2..6b2dc2ad21 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -240,6 +240,14 @@ class Field(object): keywords, ) + def clone(self): + """ + Uses deconstruct() to clone a new copy of this Field. + Will not preserve any class attachments/attribute names. + """ + name, path, args, kwargs = self.deconstruct() + return self.__class__(*args, **kwargs) + def __eq__(self, other): # Needed for @total_ordering if isinstance(other, Field): From ce05b8a69ee0e9f7f7e0154b2b9bf1dcb15edbcb Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 Dec 2013 13:56:22 +0000 Subject: [PATCH 034/158] Don't make a second migration if there was a force-null-default addcol. --- django/db/migrations/autodetector.py | 28 +++++++++++++------ django/db/migrations/operations/fields.py | 12 ++++++-- tests/migrations/test_operations.py | 34 ++++++++++++++++++++++- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/django/db/migrations/autodetector.py b/django/db/migrations/autodetector.py index 083351a6f1..f8ec49ab0a 100644 --- a/django/db/migrations/autodetector.py +++ b/django/db/migrations/autodetector.py @@ -188,15 +188,26 @@ class MigrationAutodetector(object): continue # You can't just add NOT NULL fields with no default if not field.null and not field.has_default(): + field = field.clone() field.default = self.questioner.ask_not_null_addition(field_name, model_name) - self.add_to_migration( - app_label, - operations.AddField( - model_name=model_name, - name=field_name, - field=field, + self.add_to_migration( + app_label, + operations.AddField( + model_name=model_name, + name=field_name, + field=field, + preserve_default=False, + ) + ) + else: + self.add_to_migration( + app_label, + operations.AddField( + model_name=model_name, + name=field_name, + field=field, + ) ) - ) # Old fields for field_name in old_field_names - new_field_names: self.add_to_migration( @@ -434,7 +445,8 @@ class InteractiveMigrationQuestioner(MigrationQuestioner): "Adding a NOT NULL field to a model" choice = self._choice_input( "You are trying to add a non-nullable field '%s' to %s without a default;\n" % (field_name, model_name) + - "this is not possible. Please select a fix:", + "we can't do that (the database needs something to populate existing rows).\n" + + "Please select a fix:", [ "Provide a one-off default now (will be set on all existing rows)", "Quit, and let me add a default in models.py", diff --git a/django/db/migrations/operations/fields.py b/django/db/migrations/operations/fields.py index b609110f85..c5f0bd1e2b 100644 --- a/django/db/migrations/operations/fields.py +++ b/django/db/migrations/operations/fields.py @@ -1,4 +1,5 @@ from django.db import router +from django.db.models.fields import NOT_PROVIDED from .base import Operation @@ -7,13 +8,20 @@ class AddField(Operation): Adds a field to a model. """ - def __init__(self, model_name, name, field): + def __init__(self, model_name, name, field, preserve_default=True): self.model_name = model_name self.name = name self.field = field + self.preserve_default = preserve_default def state_forwards(self, app_label, state): - state.models[app_label, self.model_name.lower()].fields.append((self.name, self.field)) + # If preserve default is off, don't use the default for future state + if not self.preserve_default: + field = self.field.clone() + field.default = NOT_PROVIDED + else: + field = self.field + state.models[app_label, self.model_name.lower()].fields.append((self.name, field)) def database_forwards(self, app_label, schema_editor, from_state, to_state): from_model = from_state.render().get_model(app_label, self.model_name) diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py index fb38605030..2dea14b445 100644 --- a/tests/migrations/test_operations.py +++ b/tests/migrations/test_operations.py @@ -1,4 +1,5 @@ from django.db import connection, models, migrations, router +from django.db.models.fields import NOT_PROVIDED from django.db.transaction import atomic from django.db.utils import IntegrityError from django.db.migrations.state import ProjectState @@ -130,10 +131,19 @@ class OperationTests(MigrationTestBase): """ project_state = self.set_up_test_model("test_adfl") # Test the state alteration - operation = migrations.AddField("Pony", "height", models.FloatField(null=True)) + operation = migrations.AddField( + "Pony", + "height", + models.FloatField(null=True, default=5), + ) new_state = project_state.clone() operation.state_forwards("test_adfl", new_state) self.assertEqual(len(new_state.models["test_adfl", "pony"].fields), 4) + field = [ + f for n, f in new_state.models["test_adfl", "pony"].fields + if n == "height" + ][0] + self.assertEqual(field.default, 5) # Test the database alteration self.assertColumnNotExists("test_adfl_pony", "height") with connection.schema_editor() as editor: @@ -144,6 +154,28 @@ class OperationTests(MigrationTestBase): operation.database_backwards("test_adfl", editor, new_state, project_state) self.assertColumnNotExists("test_adfl_pony", "height") + def test_add_field_preserve_default(self): + """ + Tests the AddField operation's state alteration + when preserve_default = False. + """ + project_state = self.set_up_test_model("test_adflpd") + # Test the state alteration + operation = migrations.AddField( + "Pony", + "height", + models.FloatField(null=True, default=4), + preserve_default = False, + ) + new_state = project_state.clone() + operation.state_forwards("test_adflpd", new_state) + self.assertEqual(len(new_state.models["test_adflpd", "pony"].fields), 4) + field = [ + f for n, f in new_state.models["test_adflpd", "pony"].fields + if n == "height" + ][0] + self.assertEqual(field.default, NOT_PROVIDED) + def test_add_field_m2m(self): """ Tests the AddField operation with a ManyToManyField. From cd9e85ece94214718257d972e9e5ab4bac3f0e65 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Wed, 4 Dec 2013 16:46:56 +0100 Subject: [PATCH 035/158] Fixed #21558 -- Support building CHM files. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Michał Pasternak. --- docs/_theme/djangodocs/layout.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_theme/djangodocs/layout.html b/docs/_theme/djangodocs/layout.html index fa0b497450..6bb3653084 100644 --- a/docs/_theme/djangodocs/layout.html +++ b/docs/_theme/djangodocs/layout.html @@ -17,6 +17,9 @@ {%- endmacro %} {% block extrahead %} +{# When building htmlhelp (CHM format) disable JQuery inclusion, #} +{# as it causes problems in compiled CHM files. #} +{% if builder != "htmlhelp" %} {{ super() }} +{% endif %} {% endblock %} {% block document %} From 3b8e46cbc7cdb03bb40b3b099997a5f659a2d402 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 4 Dec 2013 16:01:31 +0000 Subject: [PATCH 036/158] Migration VCS conflict detection and --merge for makemigrations --- .../management/commands/makemigrations.py | 97 +++++++++++++- django/core/management/commands/migrate.py | 10 ++ django/db/migrations/autodetector.py | 112 +--------------- django/db/migrations/loader.py | 14 ++ django/db/migrations/questioner.py | 122 ++++++++++++++++++ tests/migrations/test_autodetector.py | 5 +- tests/migrations/test_commands.py | 30 ++++- .../test_migrations_conflict/0001_initial.py | 27 ++++ .../0002_conflicting_second.py | 17 +++ .../test_migrations_conflict/0002_second.py | 24 ++++ .../test_migrations_conflict/__init__.py | 0 11 files changed, 341 insertions(+), 117 deletions(-) create mode 100644 django/db/migrations/questioner.py create mode 100644 tests/migrations/test_migrations_conflict/0001_initial.py create mode 100644 tests/migrations/test_migrations_conflict/0002_conflicting_second.py create mode 100644 tests/migrations/test_migrations_conflict/0002_second.py create mode 100644 tests/migrations/test_migrations_conflict/__init__.py diff --git a/django/core/management/commands/makemigrations.py b/django/core/management/commands/makemigrations.py index bb07160bb6..284898d8d4 100644 --- a/django/core/management/commands/makemigrations.py +++ b/django/core/management/commands/makemigrations.py @@ -1,21 +1,26 @@ import sys import os +import operator from optparse import make_option -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.core.exceptions import ImproperlyConfigured -from django.db import connections, DEFAULT_DB_ALIAS +from django.db import connections, DEFAULT_DB_ALIAS, migrations from django.db.migrations.loader import MigrationLoader -from django.db.migrations.autodetector import MigrationAutodetector, InteractiveMigrationQuestioner +from django.db.migrations.autodetector import MigrationAutodetector +from django.db.migrations.questioner import MigrationQuestioner, InteractiveMigrationQuestioner from django.db.migrations.state import ProjectState from django.db.migrations.writer import MigrationWriter from django.db.models.loading import cache +from django.utils.six.moves import reduce class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--dry-run', action='store_true', dest='dry_run', default=False, help="Just show what migrations would be made; don't actually write them."), + make_option('--merge', action='store_true', dest='merge', default=False, + help="Enable fixing of migration conflicts."), ) help = "Creates new migration(s) for apps." @@ -26,6 +31,7 @@ class Command(BaseCommand): self.verbosity = int(options.get('verbosity')) self.interactive = options.get('interactive') self.dry_run = options.get('dry_run', False) + self.merge = options.get('merge', False) # Make sure the app they asked for exists app_labels = set(app_labels) @@ -44,6 +50,26 @@ class Command(BaseCommand): # (makemigrations doesn't look at the database state). loader = MigrationLoader(connections[DEFAULT_DB_ALIAS]) + # Before anything else, see if there's conflicting apps and drop out + # hard if there are any and they don't want to merge + conflicts = loader.detect_conflicts() + if conflicts and not self.merge: + name_str = "; ".join( + "%s in %s" % (", ".join(names), app) + for app, names in conflicts.items() + ) + raise CommandError("Conflicting migrations detected (%s).\nTo fix them run 'python manage.py makemigrations --merge'" % name_str) + + # If they want to merge and there's nothing to merge, then politely exit + if self.merge and not conflicts: + self.stdout.write("No conflicts detected to merge.") + return + + # If they want to merge and there is something to merge, then + # divert into the merge code + if self.merge and conflicts: + return self.handle_merge(loader, conflicts) + # Detect changes autodetector = MigrationAutodetector( loader.graph.project_state(), @@ -87,3 +113,68 @@ class Command(BaseCommand): migration_string = writer.as_string() with open(writer.path, "wb") as fh: fh.write(migration_string) + + def handle_merge(self, loader, conflicts): + """ + Handles merging together conflicted migrations interactively, + if it's safe; otherwise, advises on how to fix it. + """ + if self.interactive: + questioner = InteractiveMigrationQuestioner() + else: + questioner = MigrationQuestioner() + for app_label, migration_names in conflicts.items(): + # Grab out the migrations in question, and work out their + # common ancestor. + merge_migrations = [] + for migration_name in migration_names: + migration = loader.get_migration(app_label, migration_name) + migration.ancestry = loader.graph.forwards_plan((app_label, migration_name)) + merge_migrations.append(migration) + common_ancestor = None + for level in zip(*[m.ancestry for m in merge_migrations]): + if reduce(operator.eq, level): + common_ancestor = level[0] + else: + break + if common_ancestor is None: + raise ValueError("Could not find common ancestor of %s" % migration_names) + # Now work out the operations along each divergent branch + for migration in merge_migrations: + migration.branch = migration.ancestry[ + (migration.ancestry.index(common_ancestor) + 1): + ] + migration.merged_operations = [] + for node_app, node_name in migration.branch: + migration.merged_operations.extend( + loader.get_migration(node_app, node_name).operations + ) + # In future, this could use some of the Optimizer code + # (can_optimize_through) to automatically see if they're + # mergeable. For now, we always just prompt the user. + if self.verbosity > 0: + self.stdout.write(self.style.MIGRATE_HEADING("Merging %s" % app_label)) + for migration in merge_migrations: + self.stdout.write(self.style.MIGRATE_LABEL(" Branch %s" % migration.name)) + for operation in migration.merged_operations: + self.stdout.write(" - %s\n" % operation.describe()) + if questioner.ask_merge(app_label): + # If they still want to merge it, then write out an empty + # file depending on the migrations needing merging. + numbers = [ + MigrationAutodetector.parse_number(migration.name) + for migration in merge_migrations + ] + try: + biggest_number = max([x for x in numbers if x is not None]) + except ValueError: + biggest_number = 1 + subclass = type("Migration", (migrations.Migration, ), { + "dependencies": [(app_label, migration.name) for migration in merge_migrations], + }) + new_migration = subclass("%04i_merge" % (biggest_number + 1), app_label) + writer = MigrationWriter(new_migration) + with open(writer.path, "wb") as fh: + fh.write(writer.as_string()) + if self.verbosity > 0: + self.stdout.write("\nCreated new merge migration %s" % writer.path) diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py index 3e796b655f..093c8a79d0 100644 --- a/django/core/management/commands/migrate.py +++ b/django/core/management/commands/migrate.py @@ -62,6 +62,16 @@ class Command(BaseCommand): # Work out which apps have migrations and which do not executor = MigrationExecutor(connection, self.migration_progress_callback) + # Before anything else, see if there's conflicting apps and drop out + # hard if there are any + conflicts = executor.loader.detect_conflicts() + if conflicts: + name_str = "; ".join( + "%s in %s" % (", ".join(names), app) + for app, names in conflicts.items() + ) + raise CommandError("Conflicting migrations detected (%s).\nTo fix them run 'python manage.py makemigrations --merge'" % name_str) + # If they supplied command line arguments, work out what they mean. run_syncdb = False target_app_labels_only = True diff --git a/django/db/migrations/autodetector.py b/django/db/migrations/autodetector.py index f8ec49ab0a..86ddd3c3d2 100644 --- a/django/db/migrations/autodetector.py +++ b/django/db/migrations/autodetector.py @@ -1,13 +1,8 @@ -import importlib -import os import re -import sys from django.db.migrations import operations from django.db.migrations.migration import Migration -from django.db.models.loading import cache -from django.utils import datetime_safe -from django.utils.six.moves import input +from django.db.migrations.questioner import MigrationQuestioner class MigrationAutodetector(object): @@ -369,108 +364,3 @@ class MigrationAutodetector(object): if re.match(r"^\d+_", name): return int(name.split("_")[0]) return None - - -class MigrationQuestioner(object): - """ - Gives the autodetector responses to questions it might have. - This base class has a built-in noninteractive mode, but the - interactive subclass is what the command-line arguments will use. - """ - - def __init__(self, defaults=None): - self.defaults = defaults or {} - - def ask_initial(self, app_label): - "Should we create an initial migration for the app?" - return self.defaults.get("ask_initial", False) - - def ask_not_null_addition(self, field_name, model_name): - "Adding a NOT NULL field to a model" - # None means quit - return None - - def ask_rename(self, model_name, old_name, new_name, field_instance): - "Was this field really renamed?" - return self.defaults.get("ask_rename", False) - - -class InteractiveMigrationQuestioner(MigrationQuestioner): - - def __init__(self, specified_apps=set()): - self.specified_apps = specified_apps - - def _boolean_input(self, question, default=None): - result = input("%s " % question) - if not result and default is not None: - return default - while len(result) < 1 or result[0].lower() not in "yn": - result = input("Please answer yes or no: ") - return result[0].lower() == "y" - - def _choice_input(self, question, choices): - print(question) - for i, choice in enumerate(choices): - print(" %s) %s" % (i + 1, choice)) - result = input("Select an option: ") - while True: - try: - value = int(result) - if 0 < value <= len(choices): - return value - except ValueError: - pass - result = input("Please select a valid option: ") - - def ask_initial(self, app_label): - "Should we create an initial migration for the app?" - # If it was specified on the command line, definitely true - if app_label in self.specified_apps: - return True - # Otherwise, we look to see if it has a migrations module - # without any Python files in it, apart from __init__.py. - # Apps from the new app template will have these; the python - # file check will ensure we skip South ones. - models_module = cache.get_app(app_label) - migrations_import_path = "%s.migrations" % models_module.__package__ - try: - migrations_module = importlib.import_module(migrations_import_path) - except ImportError: - return False - else: - filenames = os.listdir(os.path.dirname(migrations_module.__file__)) - return not any(x.endswith(".py") for x in filenames if x != "__init__.py") - - def ask_not_null_addition(self, field_name, model_name): - "Adding a NOT NULL field to a model" - choice = self._choice_input( - "You are trying to add a non-nullable field '%s' to %s without a default;\n" % (field_name, model_name) + - "we can't do that (the database needs something to populate existing rows).\n" + - "Please select a fix:", - [ - "Provide a one-off default now (will be set on all existing rows)", - "Quit, and let me add a default in models.py", - ] - ) - if choice == 2: - sys.exit(3) - else: - print("Please enter the default value now, as valid Python") - print("The datetime module is available, so you can do e.g. datetime.date.today()") - while True: - code = input(">>> ") - if not code: - print("Please enter some code, or 'exit' (with no quotes) to exit.") - elif code == "exit": - sys.exit(1) - else: - try: - return eval(code, {}, {"datetime": datetime_safe}) - except (SyntaxError, NameError) as e: - print("Invalid input: %s" % e) - else: - break - - def ask_rename(self, model_name, old_name, new_name, field_instance): - "Was this field really renamed?" - return self._boolean_input("Did you rename %s.%s to %s.%s (a %s)? [y/N]" % (model_name, old_name, model_name, new_name, field_instance.__class__.__name__), False) diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py index 598c582fa0..f2510b8368 100644 --- a/django/db/migrations/loader.py +++ b/django/db/migrations/loader.py @@ -187,6 +187,20 @@ class MigrationLoader(object): for parent in migration.dependencies: self.graph.add_dependency(key, parent) + def detect_conflicts(self): + """ + Looks through the loaded graph and detects any conflicts - apps + with more than one leaf migration. Returns a dict of the app labels + that conflict with the migration names that conflict. + """ + seen_apps = {} + conflicting_apps = set() + for app_label, migration_name in self.graph.leaf_nodes(): + if app_label in seen_apps: + conflicting_apps.add(app_label) + seen_apps.setdefault(app_label, set()).add(migration_name) + return dict((app_label, seen_apps[app_label]) for app_label in conflicting_apps) + class BadMigrationError(Exception): """ diff --git a/django/db/migrations/questioner.py b/django/db/migrations/questioner.py new file mode 100644 index 0000000000..2ae74d4ef6 --- /dev/null +++ b/django/db/migrations/questioner.py @@ -0,0 +1,122 @@ +import importlib +import os +import sys + +from django.db.models.loading import cache +from django.utils import datetime_safe +from django.utils.six.moves import input +from django.core.exceptions import ImproperlyConfigured + + +class MigrationQuestioner(object): + """ + Gives the autodetector responses to questions it might have. + This base class has a built-in noninteractive mode, but the + interactive subclass is what the command-line arguments will use. + """ + + def __init__(self, defaults=None, specified_apps=None): + self.defaults = defaults or {} + self.specified_apps = specified_apps or set() + + def ask_initial(self, app_label): + "Should we create an initial migration for the app?" + # If it was specified on the command line, definitely true + if app_label in self.specified_apps: + return True + # Otherwise, we look to see if it has a migrations module + # without any Python files in it, apart from __init__.py. + # Apps from the new app template will have these; the python + # file check will ensure we skip South ones. + try: + models_module = cache.get_app(app_label) + except ImproperlyConfigured: # It's a fake app + return self.defaults.get("ask_initial", False) + migrations_import_path = "%s.migrations" % models_module.__package__ + try: + migrations_module = importlib.import_module(migrations_import_path) + except ImportError: + return self.defaults.get("ask_initial", False) + else: + filenames = os.listdir(os.path.dirname(migrations_module.__file__)) + return not any(x.endswith(".py") for x in filenames if x != "__init__.py") + + def ask_not_null_addition(self, field_name, model_name): + "Adding a NOT NULL field to a model" + # None means quit + return None + + def ask_rename(self, model_name, old_name, new_name, field_instance): + "Was this field really renamed?" + return self.defaults.get("ask_rename", False) + + def ask_merge(self, app_label): + "Do you really want to merge these migrations?" + return self.defaults.get("ask_merge", False) + + +class InteractiveMigrationQuestioner(MigrationQuestioner): + + def _boolean_input(self, question, default=None): + result = input("%s " % question) + if not result and default is not None: + return default + while len(result) < 1 or result[0].lower() not in "yn": + result = input("Please answer yes or no: ") + return result[0].lower() == "y" + + def _choice_input(self, question, choices): + print(question) + for i, choice in enumerate(choices): + print(" %s) %s" % (i + 1, choice)) + result = input("Select an option: ") + while True: + try: + value = int(result) + if 0 < value <= len(choices): + return value + except ValueError: + pass + result = input("Please select a valid option: ") + + def ask_not_null_addition(self, field_name, model_name): + "Adding a NOT NULL field to a model" + choice = self._choice_input( + "You are trying to add a non-nullable field '%s' to %s without a default;\n" % (field_name, model_name) + + "we can't do that (the database needs something to populate existing rows).\n" + + "Please select a fix:", + [ + "Provide a one-off default now (will be set on all existing rows)", + "Quit, and let me add a default in models.py", + ] + ) + if choice == 2: + sys.exit(3) + else: + print("Please enter the default value now, as valid Python") + print("The datetime module is available, so you can do e.g. datetime.date.today()") + while True: + code = input(">>> ") + if not code: + print("Please enter some code, or 'exit' (with no quotes) to exit.") + elif code == "exit": + sys.exit(1) + else: + try: + return eval(code, {}, {"datetime": datetime_safe}) + except (SyntaxError, NameError) as e: + print("Invalid input: %s" % e) + else: + break + + def ask_rename(self, model_name, old_name, new_name, field_instance): + "Was this field really renamed?" + return self._boolean_input("Did you rename %s.%s to %s.%s (a %s)? [y/N]" % (model_name, old_name, model_name, new_name, field_instance.__class__.__name__), False) + + def ask_merge(self, app_label): + return self._boolean_input( + "\nMerging will only work if the operations printed above do not conflict\n" + + "with each other (working on different fields or models)\n" + + "Do you want to merge these migration branches? [y/N]", + False, + ) diff --git a/tests/migrations/test_autodetector.py b/tests/migrations/test_autodetector.py index a3db6e1e45..42e18c69cc 100644 --- a/tests/migrations/test_autodetector.py +++ b/tests/migrations/test_autodetector.py @@ -1,6 +1,7 @@ # encoding: utf8 from django.test import TestCase -from django.db.migrations.autodetector import MigrationAutodetector, MigrationQuestioner +from django.db.migrations.autodetector import MigrationAutodetector +from django.db.migrations.questioner import MigrationQuestioner from django.db.migrations.state import ProjectState, ModelState from django.db.migrations.graph import MigrationGraph from django.db import models @@ -63,7 +64,7 @@ class AutodetectorTests(TestCase): # Use project state to make a new migration change set before = self.make_project_state([]) after = self.make_project_state([self.author_empty, self.other_pony, self.other_stable, self.third_thing]) - autodetector = MigrationAutodetector(before, after, MigrationQuestioner({"ask_initial": True})) + autodetector = MigrationAutodetector(before, after, MigrationQuestioner(defaults={"ask_initial": True})) changes = autodetector._detect_changes() # Run through arrange_for_graph graph = MigrationGraph() diff --git a/tests/migrations/test_commands.py b/tests/migrations/test_commands.py index dbed522dd5..48fb68b03d 100644 --- a/tests/migrations/test_commands.py +++ b/tests/migrations/test_commands.py @@ -6,7 +6,7 @@ import copy import os import shutil -from django.core.management import call_command +from django.core.management import call_command, CommandError from django.db.models.loading import cache from django.test.utils import override_settings from django.utils import six @@ -72,6 +72,34 @@ class MigrateTests(MigrationTestBase): # Cleanup by unmigrating everything call_command("migrate", "migrations", "zero", verbosity=0) + @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_conflict"}) + def test_migrate_conflict_exit(self): + """ + Makes sure that migrate exits if it detects a conflict. + """ + with self.assertRaises(CommandError): + call_command("migrate", "migrations") + + @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_conflict"}) + def test_makemigrations_conflict_exit(self): + """ + Makes sure that makemigrations exits if it detects a conflict. + """ + with self.assertRaises(CommandError): + call_command("makemigrations") + + @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations_conflict"}) + def test_makemigrations_merge_basic(self): + """ + Makes sure that makemigrations doesn't error if you ask for + merge mode with a conflict present. Doesn't test writing of the merge + file, as that requires temp directories. + """ + try: + call_command("makemigrations", merge=True, verbosity=0) + except CommandError: + self.fail("Makemigrations errored in merge mode with conflicts") + @override_settings(MIGRATION_MODULES={"migrations": "migrations.test_migrations"}) def test_sqlmigrate(self): """ diff --git a/tests/migrations/test_migrations_conflict/0001_initial.py b/tests/migrations/test_migrations_conflict/0001_initial.py new file mode 100644 index 0000000000..344bebdfe3 --- /dev/null +++ b/tests/migrations/test_migrations_conflict/0001_initial.py @@ -0,0 +1,27 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + operations = [ + + migrations.CreateModel( + "Author", + [ + ("id", models.AutoField(primary_key=True)), + ("name", models.CharField(max_length=255)), + ("slug", models.SlugField(null=True)), + ("age", models.IntegerField(default=0)), + ("silly_field", models.BooleanField(default=False)), + ], + ), + + migrations.CreateModel( + "Tribble", + [ + ("id", models.AutoField(primary_key=True)), + ("fluffy", models.BooleanField(default=True)), + ], + ) + + ] diff --git a/tests/migrations/test_migrations_conflict/0002_conflicting_second.py b/tests/migrations/test_migrations_conflict/0002_conflicting_second.py new file mode 100644 index 0000000000..15ea1f063a --- /dev/null +++ b/tests/migrations/test_migrations_conflict/0002_conflicting_second.py @@ -0,0 +1,17 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("migrations", "0001_initial")] + + operations = [ + + migrations.CreateModel( + "Something", + [ + ("id", models.AutoField(primary_key=True)), + ], + ) + + ] diff --git a/tests/migrations/test_migrations_conflict/0002_second.py b/tests/migrations/test_migrations_conflict/0002_second.py new file mode 100644 index 0000000000..ace9a83347 --- /dev/null +++ b/tests/migrations/test_migrations_conflict/0002_second.py @@ -0,0 +1,24 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("migrations", "0001_initial")] + + operations = [ + + migrations.DeleteModel("Tribble"), + + migrations.RemoveField("Author", "silly_field"), + + migrations.AddField("Author", "rating", models.IntegerField(default=0)), + + migrations.CreateModel( + "Book", + [ + ("id", models.AutoField(primary_key=True)), + ("author", models.ForeignKey("migrations.Author", null=True)), + ], + ) + + ] diff --git a/tests/migrations/test_migrations_conflict/__init__.py b/tests/migrations/test_migrations_conflict/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 0daa2f1bf1dde97cc10781b4f1e6a1a987ec2330 Mon Sep 17 00:00:00 2001 From: Alasdair Nicol Date: Wed, 4 Dec 2013 21:32:41 +0000 Subject: [PATCH 037/158] List 1.6.x releases in descending order --- docs/releases/index.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 0c80cfb378..080d7a7891 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -29,8 +29,8 @@ Final releases .. toctree:: :maxdepth: 1 - 1.6 1.6.1 + 1.6 1.5 release ----------- From 164df40501d91ae8dd250700df15302c64001a69 Mon Sep 17 00:00:00 2001 From: Frank Wiles Date: Wed, 4 Dec 2013 18:03:28 -0600 Subject: [PATCH 038/158] Fixing manager documentation inaccuracy --- docs/topics/db/managers.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/db/managers.txt b/docs/topics/db/managers.txt index c97149cfd1..3377bccab6 100644 --- a/docs/topics/db/managers.txt +++ b/docs/topics/db/managers.txt @@ -219,7 +219,7 @@ custom ``QuerySet`` if you also implement them on the ``Manager``:: class PersonManager(models.Manager): def get_queryset(self): - return PersonQuerySet() + return PersonQuerySet(self.model, using=self._db) def male(self): return self.get_queryset().male() From ae734b75c1fc365be9c2e2a08c8c569166233149 Mon Sep 17 00:00:00 2001 From: maurycyp Date: Thu, 5 Dec 2013 00:43:48 -0500 Subject: [PATCH 039/158] Removed 'return' in __init__ --- django/contrib/formtools/tests/wizard/test_forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/formtools/tests/wizard/test_forms.py b/django/contrib/formtools/tests/wizard/test_forms.py index 386e448485..fedb799669 100644 --- a/django/contrib/formtools/tests/wizard/test_forms.py +++ b/django/contrib/formtools/tests/wizard/test_forms.py @@ -48,7 +48,7 @@ class CustomKwargsStep1(Step1): def __init__(self, test=None, *args, **kwargs): self.test = test - return super(CustomKwargsStep1, self).__init__(*args, **kwargs) + super(CustomKwargsStep1, self).__init__(*args, **kwargs) class TestModel(models.Model): From cbf8e8aa125d25aa0d4b3a65ef9915acca4a4e81 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Thu, 5 Dec 2013 08:08:34 -0600 Subject: [PATCH 040/158] Fixed a flake8 error --- tests/migrations/test_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/migrations/test_operations.py b/tests/migrations/test_operations.py index 2dea14b445..93f0842bcd 100644 --- a/tests/migrations/test_operations.py +++ b/tests/migrations/test_operations.py @@ -165,7 +165,7 @@ class OperationTests(MigrationTestBase): "Pony", "height", models.FloatField(null=True, default=4), - preserve_default = False, + preserve_default=False, ) new_state = project_state.clone() operation.state_forwards("test_adflpd", new_state) From 4ca39684ee1e21b7e508d58ee89f9a0c309399c4 Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 5 Dec 2013 14:10:58 +0000 Subject: [PATCH 041/158] Fix poor variable name (flake8 warning) --- django/core/management/commands/makemigrations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/core/management/commands/makemigrations.py b/django/core/management/commands/makemigrations.py index 284898d8d4..239a0a416c 100644 --- a/django/core/management/commands/makemigrations.py +++ b/django/core/management/commands/makemigrations.py @@ -89,10 +89,10 @@ class Command(BaseCommand): return directory_created = {} - for app_label, migrations in changes.items(): + for app_label, app_migrations in changes.items(): if self.verbosity >= 1: self.stdout.write(self.style.MIGRATE_HEADING("Migrations for '%s':" % app_label) + "\n") - for migration in migrations: + for migration in app_migrations: # Describe the migration writer = MigrationWriter(migration) if self.verbosity >= 1: From e9c6d0422422d7ea5ff3be8992ca73b5ba73bd0c Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Thu, 5 Dec 2013 14:19:46 +0000 Subject: [PATCH 042/158] Better error reporting when from_app_cache fails --- django/db/migrations/state.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py index f5019f65ee..0dd8d85e10 100644 --- a/django/db/migrations/state.py +++ b/django/db/migrations/state.py @@ -99,11 +99,25 @@ class ModelState(object): for field in model._meta.local_fields: name, path, args, kwargs = field.deconstruct() field_class = import_by_path(path) - fields.append((name, field_class(*args, **kwargs))) + try: + fields.append((name, field_class(*args, **kwargs))) + except TypeError as e: + raise TypeError("Couldn't reconstruct field %s on %s: %s" % ( + name, + model._meta.object_name, + e, + )) for field in model._meta.local_many_to_many: name, path, args, kwargs = field.deconstruct() field_class = import_by_path(path) - fields.append((name, field_class(*args, **kwargs))) + try: + fields.append((name, field_class(*args, **kwargs))) + except TypeError as e: + raise TypeError("Couldn't reconstruct m2m field %s on %s: %s" % ( + name, + model._meta.object_name, + e, + )) # Extract the options options = {} for name in DEFAULT_NAMES: From e36c165b140c6f86678f7d12785d1957afa1735c Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Thu, 5 Dec 2013 18:22:37 -0300 Subject: [PATCH 043/158] Corrected setting name in gis test error message. --- django/contrib/gis/geoip/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/gis/geoip/tests.py b/django/contrib/gis/geoip/tests.py index f771608e1d..8d7bdff291 100644 --- a/django/contrib/gis/geoip/tests.py +++ b/django/contrib/gis/geoip/tests.py @@ -25,7 +25,7 @@ if HAS_GEOS: @skipUnless(HAS_GEOIP and getattr(settings, "GEOIP_PATH", None), - "GeoIP is required along with the GEOIP_DATA setting.") + "GeoIP is required along with the GEOIP_PATH setting.") class GeoIPTest(unittest.TestCase): def test01_init(self): From 3396d5716b2fccf5de5322cc50efef3c6e9b1152 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Thu, 5 Dec 2013 23:07:50 +0100 Subject: [PATCH 044/158] Fixed typo in custom model field documentation. --- docs/howto/custom-model-fields.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/custom-model-fields.txt b/docs/howto/custom-model-fields.txt index 9c75f27c02..70362c93e2 100644 --- a/docs/howto/custom-model-fields.txt +++ b/docs/howto/custom-model-fields.txt @@ -251,7 +251,7 @@ you'll need to supplement the values being passed. The contract of ``deconstruct`` is simple; it returns a tuple of four items: the field's attribute name, the full import path of the field class, the -position arguments (as a list), and the keyword arguments (as a dict). +positional arguments (as a list), and the keyword arguments (as a dict). As a custom field author, you don't need to care about the first two values; the base ``Field`` class has all the code to work out the field's attribute From 38662d11c4201c9fe9ba810515c2ed0b939065c0 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Thu, 5 Dec 2013 23:11:30 +0100 Subject: [PATCH 045/158] Fixed another typo in custom model field documentation. --- docs/howto/custom-model-fields.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/custom-model-fields.txt b/docs/howto/custom-model-fields.txt index 70362c93e2..6054a83f10 100644 --- a/docs/howto/custom-model-fields.txt +++ b/docs/howto/custom-model-fields.txt @@ -304,7 +304,7 @@ Pay extra attention if you set new default values for arguments in the than disappearing if they take on the old default value. In addition, try to avoid returning values as positional arguments; where -possible, return values as keyword arguments for maximum future compatability. +possible, return values as keyword arguments for maximum future compatibility. Of course, if you change the names of things more often than their position in the constructor's argument list, you might prefer positional, but bear in mind that people will be reconstructing your field from the serialized version From aba75b0d71808ed9ebc625e31a47da158f25b29d Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 00:55:31 +0100 Subject: [PATCH 046/158] Fixed TypeError when rendering ModelState with multiple bases. --- django/db/migrations/state.py | 2 +- tests/migrations/test_state.py | 48 ++++++++++++++++++++++++++-------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py index 0dd8d85e10..53c24cb8da 100644 --- a/django/db/migrations/state.py +++ b/django/db/migrations/state.py @@ -176,7 +176,7 @@ class ModelState(object): for base in self.bases ) if None in bases: - raise InvalidBasesError("Cannot resolve one or more bases from %r" % self.bases) + raise InvalidBasesError("Cannot resolve one or more bases from %r" % (self.bases,)) # Turn fields into a dict for the body, add other bits body = dict(self.fields) body['Meta'] = meta diff --git a/tests/migrations/test_state.py b/tests/migrations/test_state.py index fe2092dad1..889367b41d 100644 --- a/tests/migrations/test_state.py +++ b/tests/migrations/test_state.py @@ -94,32 +94,58 @@ class StateTests(TestCase): self.assertEqual(new_app_cache.get_model("migrations", "Tag")._meta.get_field_by_name("name")[0].max_length, 100) self.assertEqual(new_app_cache.get_model("migrations", "Tag")._meta.get_field_by_name("hidden")[0].null, False) - def test_render_multiple_inheritance(self): - # Use a custom app cache to avoid polluting the global one. - new_app_cache = BaseAppCache() - + def test_render_model_inheritance(self): class Book(models.Model): title = models.CharField(max_length=1000) class Meta: app_label = "migrations" - app_cache = new_app_cache + app_cache = BaseAppCache() class Novel(Book): class Meta: app_label = "migrations" - app_cache = new_app_cache + app_cache = BaseAppCache() # First, test rendering individually - yet_another_app_cache = BaseAppCache() + app_cache = BaseAppCache() # We shouldn't be able to render yet - with self.assertRaises(ValueError): - ModelState.from_model(Novel).render(yet_another_app_cache) + ms = ModelState.from_model(Novel) + with self.assertRaises(InvalidBasesError): + ms.render(app_cache) # Once the parent model is in the app cache, it should be fine - ModelState.from_model(Book).render(yet_another_app_cache) - ModelState.from_model(Novel).render(yet_another_app_cache) + ModelState.from_model(Book).render(app_cache) + ModelState.from_model(Novel).render(app_cache) + + def test_render_model_with_multiple_inheritance(self): + class Foo(models.Model): + class Meta: + app_label = "migrations" + app_cache = BaseAppCache() + + class Bar(models.Model): + class Meta: + app_label = "migrations" + app_cache = BaseAppCache() + + class FooBar(Foo, Bar): + class Meta: + app_label = "migrations" + app_cache = BaseAppCache() + + app_cache = BaseAppCache() + + # We shouldn't be able to render yet + ms = ModelState.from_model(FooBar) + with self.assertRaises(InvalidBasesError): + ms.render(app_cache) + + # Once the parent models are in the app cache, it should be fine + ModelState.from_model(Foo).render(app_cache) + ModelState.from_model(Bar).render(app_cache) + ModelState.from_model(FooBar).render(app_cache) def test_render_project_dependencies(self): """ From 317fd13c7ac25db94d3dabf8ee115acbfbd3e5a7 Mon Sep 17 00:00:00 2001 From: Alasdair Nicol Date: Thu, 5 Dec 2013 13:45:49 +0000 Subject: [PATCH 047/158] Fixed #21562 -- Warned against using the same app name as a django contrib app. Thanks yourcelf for the report. --- docs/intro/reusable-apps.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/intro/reusable-apps.txt b/docs/intro/reusable-apps.txt index 99b383585f..04a8ea7417 100644 --- a/docs/intro/reusable-apps.txt +++ b/docs/intro/reusable-apps.txt @@ -128,6 +128,13 @@ this. For a small app like polls, this process isn't too difficult. This helps others looking for Django apps identify your app as Django specific. + The application names (that is, the final dotted part of the + path to the module containing ``models.py``) defined in + :setting:`INSTALLED_APPS` *must* be unique. Avoid using the + same name as any of the Django :doc:`contrib packages + `, for example ``auth``, ``admin`` or + ``messages``. + 2. Move the ``polls`` directory into the ``django-polls`` directory. 3. Create a file ``django-polls/README.rst`` with the following contents: From 4d0c72eb68fc6518ef02c757b37e5548d0007f41 Mon Sep 17 00:00:00 2001 From: maurycyp Date: Thu, 5 Dec 2013 23:42:35 -0500 Subject: [PATCH 048/158] Removed unreachable `else` in `try` block. --- django/db/migrations/questioner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django/db/migrations/questioner.py b/django/db/migrations/questioner.py index 2ae74d4ef6..7e798d3105 100644 --- a/django/db/migrations/questioner.py +++ b/django/db/migrations/questioner.py @@ -106,8 +106,6 @@ class InteractiveMigrationQuestioner(MigrationQuestioner): return eval(code, {}, {"datetime": datetime_safe}) except (SyntaxError, NameError) as e: print("Invalid input: %s" % e) - else: - break def ask_rename(self, model_name, old_name, new_name, field_instance): "Was this field really renamed?" From 482ca0cecc93a76cbcbfbf784f56db160b2e5cfc Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 6 Dec 2013 09:59:30 +0100 Subject: [PATCH 049/158] Renamed syncdb to migrate in spatialite backend --- django/contrib/gis/db/backends/spatialite/creation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django/contrib/gis/db/backends/spatialite/creation.py b/django/contrib/gis/db/backends/spatialite/creation.py index fb3606f7e9..521985259e 100644 --- a/django/contrib/gis/db/backends/spatialite/creation.py +++ b/django/contrib/gis/db/backends/spatialite/creation.py @@ -13,7 +13,7 @@ class SpatiaLiteCreation(DatabaseCreation): database already exists. Returns the name of the test database created. This method is overloaded to load up the SpatiaLite initialization - SQL prior to calling the `syncdb` command. + SQL prior to calling the `migrate` command. """ # Don't import django.core.management if it isn't needed. from django.core.management import call_command @@ -31,13 +31,13 @@ class SpatiaLiteCreation(DatabaseCreation): self.connection.close() self.connection.settings_dict["NAME"] = test_database_name - # Need to load the SpatiaLite initialization SQL before running `syncdb`. + # Need to load the SpatiaLite initialization SQL before running `migrate`. self.load_spatialite_sql() - # Report syncdb messages at one level lower than that requested. + # Report migrate messages at one level lower than that requested. # This ensures we don't get flooded with messages during testing # (unless you really ask to be flooded) - call_command('syncdb', + call_command('migrate', verbosity=max(verbosity - 1, 0), interactive=False, database=self.connection.alias, @@ -47,7 +47,7 @@ class SpatiaLiteCreation(DatabaseCreation): # custom SQL has been removed. The only test data should come from # test fixtures, or autogenerated from post_migrate triggers. # This has the side effect of loading initial data (which was - # intentionally skipped in the syncdb). + # intentionally skipped in the migrate). call_command('flush', verbosity=max(verbosity - 1, 0), interactive=False, From 9a73e7f40c851cef262dd36da444c81492632ee5 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Fri, 6 Dec 2013 08:44:43 -0300 Subject: [PATCH 050/158] Fixed #19678 -- GeoDjango test failure with spatialite >= 3.0. Thanks Julien for the report and Claude for the fix. --- django/contrib/gis/tests/test_spatialrefsys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/gis/tests/test_spatialrefsys.py b/django/contrib/gis/tests/test_spatialrefsys.py index 194578d1e0..3744546ccd 100644 --- a/django/contrib/gis/tests/test_spatialrefsys.py +++ b/django/contrib/gis/tests/test_spatialrefsys.py @@ -12,7 +12,7 @@ test_srs = ({'srid': 4326, # Only the beginning, because there are differences depending on installed libs 'srtext': 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84"', # +ellps=WGS84 has been removed in the 4326 proj string in proj-4.8 - 'proj4_re': r'\+proj=longlat (\+ellps=WGS84 )?\+datum=WGS84 \+no_defs ', + 'proj4_re': r'\+proj=longlat (\+ellps=WGS84 )?(\+datum=WGS84 |\+towgs84=0,0,0,0,0,0,0 )\+no_defs ', 'spheroid': 'WGS 84', 'name': 'WGS 84', 'geographic': True, 'projected': False, 'spatialite': True, 'ellipsoid': (6378137.0, 6356752.3, 298.257223563), # From proj's "cs2cs -le" and Wikipedia (semi-minor only) From f463789f6230c2fccf40c1e410443551bfb205a7 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 14:59:08 +0100 Subject: [PATCH 051/158] Added app_label to the error message when field reconstructing fails. --- django/db/migrations/state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py index 53c24cb8da..87cc0cb80e 100644 --- a/django/db/migrations/state.py +++ b/django/db/migrations/state.py @@ -102,8 +102,9 @@ class ModelState(object): try: fields.append((name, field_class(*args, **kwargs))) except TypeError as e: - raise TypeError("Couldn't reconstruct field %s on %s: %s" % ( + raise TypeError("Couldn't reconstruct field %s on %s.%s: %s" % ( name, + model._meta.app_label, model._meta.object_name, e, )) From 54d9e3ccf6dd16f76fc9852edc19e4007e26dd7a Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 15:01:36 +0100 Subject: [PATCH 052/158] Fixed error in ManyToManyField.deconstruct(). --- django/db/models/fields/related.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 1466270e5f..23b508ebb5 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1476,7 +1476,7 @@ class ManyToManyField(RelatedField): name, path, args, kwargs = super(ManyToManyField, self).deconstruct() # Handle the simpler arguments if self.rel.db_constraint is not True: - kwargs['db_constraint'] = self.db_constraint + kwargs['db_constraint'] = self.rel.db_constraint if "help_text" in kwargs: del kwargs['help_text'] # Rel needs more work. From 19e437497172640a1b488228fb218719f9f7f5b4 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 15:05:12 +0100 Subject: [PATCH 053/158] Fixed ModelState breaking when unique_together has unhashable elements. --- django/db/migrations/state.py | 5 +++-- django/db/models/options.py | 9 ++++++--- tests/migrations/test_state.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/django/db/migrations/state.py b/django/db/migrations/state.py index 87cc0cb80e..769b0005f8 100644 --- a/django/db/migrations/state.py +++ b/django/db/migrations/state.py @@ -1,6 +1,6 @@ from django.db import models from django.db.models.loading import BaseAppCache -from django.db.models.options import DEFAULT_NAMES +from django.db.models.options import DEFAULT_NAMES, normalize_unique_together from django.utils import six from django.utils.module_loading import import_by_path @@ -127,7 +127,8 @@ class ModelState(object): continue elif name in model._meta.original_attrs: if name == "unique_together": - options[name] = set(model._meta.original_attrs["unique_together"]) + ut = model._meta.original_attrs["unique_together"] + options[name] = set(normalize_unique_together(ut)) else: options[name] = model._meta.original_attrs[name] # Make our record diff --git a/django/db/models/options.py b/django/db/models/options.py index 5c4a9dcad7..b14e61573c 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -32,10 +32,13 @@ def normalize_unique_together(unique_together): tuple of two strings. Normalize it to a tuple of tuples, so that calling code can uniformly expect that. """ - unique_together = tuple(unique_together) - if unique_together and not isinstance(unique_together[0], (tuple, list)): + if not unique_together: + return () + first_element = next(iter(unique_together)) + if not isinstance(first_element, (tuple, list)): unique_together = (unique_together,) - return unique_together + # Normalize everything to tuples + return tuple(tuple(ut) for ut in unique_together) @python_2_unicode_compatible diff --git a/tests/migrations/test_state.py b/tests/migrations/test_state.py index 889367b41d..9cbbbf294d 100644 --- a/tests/migrations/test_state.py +++ b/tests/migrations/test_state.py @@ -55,7 +55,7 @@ class StateTests(TestCase): self.assertEqual(author_state.fields[1][1].max_length, 255) self.assertEqual(author_state.fields[2][1].null, False) self.assertEqual(author_state.fields[3][1].null, True) - self.assertEqual(author_state.options, {"unique_together": set(("name", "bio"))}) + self.assertEqual(author_state.options, {"unique_together": {("name", "bio")}}) self.assertEqual(author_state.bases, (models.Model, )) self.assertEqual(book_state.app_label, "migrations") From 72479a29579f72a7713ac3c07eb92f5bafee25bd Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 15:20:16 +0100 Subject: [PATCH 054/158] Made the migration detector use meta.local_fields instead of meta.fields. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #21010. Thanks to Loïc for the patch. --- django/db/migrations/autodetector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/migrations/autodetector.py b/django/db/migrations/autodetector.py index 86ddd3c3d2..78c9770366 100644 --- a/django/db/migrations/autodetector.py +++ b/django/db/migrations/autodetector.py @@ -67,7 +67,7 @@ class MigrationAutodetector(object): model_state = self.to_state.models[app_label, model_name] # Are there any relationships out from this model? if so, punt it to the next phase. related_fields = [] - for field in new_app_cache.get_model(app_label, model_name)._meta.fields: + for field in new_app_cache.get_model(app_label, model_name)._meta.local_fields: if field.rel: if field.rel.to: related_fields.append((field.name, field.rel.to._meta.app_label.lower(), field.rel.to._meta.object_name.lower())) From 621c25c419ae0f4d6d0f76725e5f3585d76228d0 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Fri, 6 Dec 2013 15:21:13 +0100 Subject: [PATCH 055/158] Added missing deconstruct() methods. --- django/db/models/__init__.py | 1 + django/db/models/fields/__init__.py | 2 ++ django/db/models/fields/proxy.py | 5 +++++ django/db/models/fields/related.py | 16 ++++++++++++---- tests/serializers/models.py | 5 +++++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py index e2f7348ef0..27db994b61 100644 --- a/django/db/models/__init__.py +++ b/django/db/models/__init__.py @@ -15,6 +15,7 @@ from django.db.models.fields.files import FileField, ImageField # NOQA from django.db.models.fields.related import ( # NOQA ForeignKey, ForeignObject, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel) +from django.db.models.fields.proxy import OrderWrt from django.db.models.deletion import ( # NOQA CASCADE, PROTECT, SET, SET_NULL, SET_DEFAULT, DO_NOTHING, ProtectedError) from django.db.models import signals # NOQA diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 6b2dc2ad21..826471d9ed 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -230,6 +230,8 @@ class Field(object): path = path.replace("django.db.models.fields.related", "django.db.models") if path.startswith("django.db.models.fields.files"): path = path.replace("django.db.models.fields.files", "django.db.models") + if path.startswith("django.db.models.fields.proxy"): + path = path.replace("django.db.models.fields.proxy", "django.db.models") if path.startswith("django.db.models.fields"): path = path.replace("django.db.models.fields", "django.db.models") # Return basic info - other fields should override this. diff --git a/django/db/models/fields/proxy.py b/django/db/models/fields/proxy.py index 29c782b59a..19beb89011 100644 --- a/django/db/models/fields/proxy.py +++ b/django/db/models/fields/proxy.py @@ -16,3 +16,8 @@ class OrderWrt(fields.IntegerField): kwargs['name'] = '_order' kwargs['editable'] = False super(OrderWrt, self).__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super(OrderWrt, self).deconstruct() + del kwargs['editable'] + return name, path, args, kwargs diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 23b508ebb5..be796f2af1 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1019,6 +1019,16 @@ class ForeignObject(RelatedField): super(ForeignObject, self).__init__(**kwargs) + def deconstruct(self): + name, path, args, kwargs = super(ForeignObject, self).deconstruct() + kwargs['from_fields'] = self.from_fields + kwargs['to_fields'] = self.to_fields + if isinstance(self.rel.to, six.string_types): + kwargs['to'] = self.rel.to + else: + kwargs['to'] = "%s.%s" % (self.rel.to._meta.app_label, self.rel.to._meta.object_name) + return name, path, args, kwargs + def resolve_related_fields(self): if len(self.from_fields) < 1 or len(self.from_fields) != len(self.to_fields): raise ValueError('Foreign Object from and to fields must be the same non-zero length') @@ -1243,6 +1253,8 @@ class ForeignKey(ForeignObject): def deconstruct(self): name, path, args, kwargs = super(ForeignKey, self).deconstruct() + del kwargs['to_fields'] + del kwargs['from_fields'] # Handle the simpler arguments if self.db_index: del kwargs['db_index'] @@ -1255,10 +1267,6 @@ class ForeignKey(ForeignObject): # Rel needs more work. if self.rel.field_name: kwargs['to_field'] = self.rel.field_name - if isinstance(self.rel.to, six.string_types): - kwargs['to'] = self.rel.to - else: - kwargs['to'] = "%s.%s" % (self.rel.to._meta.app_label, self.rel.to._meta.object_name) return name, path, args, kwargs @property diff --git a/tests/serializers/models.py b/tests/serializers/models.py index 680d463ad0..691d87ea29 100644 --- a/tests/serializers/models.py +++ b/tests/serializers/models.py @@ -115,6 +115,11 @@ class TeamField(six.with_metaclass(models.SubfieldBase, models.CharField)): def value_to_string(self, obj): return self._get_val_from_obj(obj).to_string() + def deconstruct(self): + name, path, args, kwargs = super(TeamField, self).deconstruct() + del kwargs['max_length'] + return name, path, args, kwargs + @python_2_unicode_compatible class Player(models.Model): From 362dd68fb20be195462af22448416c9895ce7df7 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Fri, 6 Dec 2013 08:48:05 -0300 Subject: [PATCH 056/158] Added new 'srtext' spatialite 4.x SpatialRefSys column to its model. This is for general consistency in the GeoDjango DB backends. Thanks Claude for the fix. Refs #19678. --- django/contrib/gis/db/backends/spatialite/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django/contrib/gis/db/backends/spatialite/models.py b/django/contrib/gis/db/backends/spatialite/models.py index 9860779647..551bc70ccd 100644 --- a/django/contrib/gis/db/backends/spatialite/models.py +++ b/django/contrib/gis/db/backends/spatialite/models.py @@ -1,7 +1,7 @@ """ The GeometryColumns and SpatialRefSys models for the SpatiaLite backend. """ -from django.db import models +from django.db import connection, models from django.contrib.gis.db.backends.base import SpatialRefSysMixin from django.utils.encoding import python_2_unicode_compatible @@ -53,9 +53,13 @@ class SpatialRefSys(models.Model, SpatialRefSysMixin): auth_srid = models.IntegerField() ref_sys_name = models.CharField(max_length=256) proj4text = models.CharField(max_length=2048) + if connection.ops.spatial_version[0] >= 4: + srtext = models.CharField(max_length=2048) @property def wkt(self): + if hasattr(self, 'srtext'): + return self.srtext from django.contrib.gis.gdal import SpatialReference return SpatialReference(self.proj4text).wkt From b63acdfe7123c243b28e15f29e5b7a8487d69221 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 6 Dec 2013 13:06:59 -0500 Subject: [PATCH 057/158] Removed a u'' prefix that prevented the docs from building on Python 3.2. --- docs/_ext/djangodocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index 8946eb063b..7ad058ff11 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -179,7 +179,7 @@ class SnippetWithFilename(Directive): option_spec = {'filename': directives.unchanged_required} def run(self): - code = u'\n'.join(self.content) + code = '\n'.join(self.content) literal = snippet_with_filename(code, code) if self.arguments: From 38e24d680d28b92997def9ab46a961d09bb81dce Mon Sep 17 00:00:00 2001 From: pegler Date: Thu, 5 Dec 2013 11:46:25 -0500 Subject: [PATCH 058/158] Fixed #21554 -- Incorrect SQL generated when using multiple inheritance. --- django/db/models/sql/compiler.py | 19 +++++++++++++------ tests/model_inheritance_regress/models.py | 14 ++++++++++++++ tests/model_inheritance_regress/tests.py | 9 ++++++++- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 98b953937f..7e48adb1d6 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -284,25 +284,32 @@ class SQLCompiler(object): continue alias = self.query.join_parent_model(opts, model, start_alias, seen_models) + column = field.column + for seen_model, seen_alias in seen_models.items(): + if seen_model and seen_alias == alias: + ancestor_link = seen_model._meta.get_ancestor_link(model) + if ancestor_link: + column = ancestor_link.column + break table = self.query.alias_map[alias].table_name - if table in only_load and field.column not in only_load[table]: + if table in only_load and column not in only_load[table]: continue if as_pairs: - result.append((alias, field.column)) + result.append((alias, column)) aliases.add(alias) continue - if with_aliases and field.column in col_aliases: + if with_aliases and column in col_aliases: c_alias = 'Col%d' % len(col_aliases) result.append('%s.%s AS %s' % (qn(alias), - qn2(field.column), c_alias)) + qn2(column), c_alias)) col_aliases.add(c_alias) aliases.add(c_alias) else: - r = '%s.%s' % (qn(alias), qn2(field.column)) + r = '%s.%s' % (qn(alias), qn2(column)) result.append(r) aliases.add(r) if with_aliases: - col_aliases.add(field.column) + col_aliases.add(column) return result, aliases def get_distinct(self): diff --git a/tests/model_inheritance_regress/models.py b/tests/model_inheritance_regress/models.py index 53752b6948..cb0b2fe72f 100644 --- a/tests/model_inheritance_regress/models.py +++ b/tests/model_inheritance_regress/models.py @@ -233,3 +233,17 @@ class User(models.Model): class Profile(User): profile_id = models.AutoField(primary_key=True) extra = models.CharField(max_length=30, blank=True) + + +# Check concrete + concrete -> concrete -> concrete +class Politician(models.Model): + politician_id = models.AutoField(primary_key=True) + title = models.CharField(max_length=50) + + +class Congressman(Person, Politician): + state = models.CharField(max_length=2) + + +class Senator(Congressman): + pass diff --git a/tests/model_inheritance_regress/tests.py b/tests/model_inheritance_regress/tests.py index 95c693ddd8..3664fc8c04 100644 --- a/tests/model_inheritance_regress/tests.py +++ b/tests/model_inheritance_regress/tests.py @@ -15,7 +15,7 @@ from .models import (Place, Restaurant, ItalianRestaurant, ParkingLot, SelfRefChild, ArticleWithAuthor, M2MChild, QualityControl, DerivedM, Person, BirthdayParty, BachelorParty, MessyBachelorParty, InternalCertificationAudit, BusStation, TrainStation, User, Profile, - ParkingLot4A, ParkingLot4B) + ParkingLot4A, ParkingLot4B, Senator) class ModelInheritanceTest(TestCase): @@ -455,3 +455,10 @@ class ModelInheritanceTest(TestCase): # used in the qs and top contains direct pointer to the bottom model. qs = ItalianRestaurant.objects.values_list('serves_gnocchi').filter(name='foo') self.assertEqual(str(qs.query).count('JOIN'), 1) + + def test_issue_21554(self): + senator = Senator.objects.create( + name='John Doe', title='X', state='Y' + ) + + Senator.objects.get(pk=senator.pk) From bbc73e6a12227f5ed52fd38bc37f56f434a0a72c Mon Sep 17 00:00:00 2001 From: Roger Hu Date: Fri, 6 Dec 2013 01:14:57 +0000 Subject: [PATCH 059/158] Fixed #21566 -- Fixed AttributeError when using bulk_create with ForeignObject. --- django/db/models/query.py | 2 +- tests/foreign_object/tests.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index 58180e1cf9..3226f38753 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -389,7 +389,7 @@ class QuerySet(object): return objs self._for_write = True connection = connections[self.db] - fields = self.model._meta.local_fields + fields = self.model._meta.local_concrete_fields with transaction.commit_on_success_unless_managed(using=self.db): if (connection.features.can_combine_inserts_with_and_without_auto_increment_pk and self.model._meta.has_auto_field): diff --git a/tests/foreign_object/tests.py b/tests/foreign_object/tests.py index 66f57b6f3c..f361c5c8d5 100644 --- a/tests/foreign_object/tests.py +++ b/tests/foreign_object/tests.py @@ -4,7 +4,7 @@ from operator import attrgetter from .models import ( Country, Person, Group, Membership, Friendship, Article, ArticleTranslation, ArticleTag, ArticleIdea, NewsArticle) -from django.test import TestCase +from django.test import TestCase, skipUnlessDBFeature from django.utils.translation import activate from django.core.exceptions import FieldError from django import forms @@ -380,6 +380,12 @@ class MultiColumnFKTests(TestCase): 'active_translation')[0].active_translation.title, "foo") + @skipUnlessDBFeature('has_bulk_insert') + def test_batch_create_foreign_object(self): + """ See: https://code.djangoproject.com/ticket/21566 """ + objs = [Person(name="abcd_%s" % i, person_country=self.usa) for i in range(0, 5)] + Person.objects.bulk_create(objs, 10) + class FormsTests(TestCase): # ForeignObjects should not have any form fields, currently the user needs From ffc0e0ca3760759688b497ed9f580c3ebd807f80 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Fri, 6 Dec 2013 13:22:53 -0600 Subject: [PATCH 060/158] Corrected a flake8 issue -- this line is imported for the purpose of re-exposing the name --- django/db/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/models/__init__.py b/django/db/models/__init__.py index 27db994b61..12c31f89fd 100644 --- a/django/db/models/__init__.py +++ b/django/db/models/__init__.py @@ -15,7 +15,7 @@ from django.db.models.fields.files import FileField, ImageField # NOQA from django.db.models.fields.related import ( # NOQA ForeignKey, ForeignObject, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel) -from django.db.models.fields.proxy import OrderWrt +from django.db.models.fields.proxy import OrderWrt # NOQA from django.db.models.deletion import ( # NOQA CASCADE, PROTECT, SET, SET_NULL, SET_DEFAULT, DO_NOTHING, ProtectedError) from django.db.models import signals # NOQA From a020dd0a99da13d0f024d42c46f01d8f503e9d5e Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Sat, 7 Dec 2013 03:21:58 +0100 Subject: [PATCH 061/158] Fixed #21530 -- Prevent AttributeError in default URLconf detection code. Thanks to @dmyerscoug for the report and original patch and to @alasdairnicol for the added tests. --- django/views/debug.py | 2 +- tests/view_tests/default_urls.py | 8 +++++++ tests/view_tests/regression_21530_urls.py | 5 ++++ tests/view_tests/tests/test_debug.py | 29 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/view_tests/default_urls.py create mode 100644 tests/view_tests/regression_21530_urls.py diff --git a/django/views/debug.py b/django/views/debug.py index 66453fe2c6..8f7f9dc00a 100644 --- a/django/views/debug.py +++ b/django/views/debug.py @@ -483,7 +483,7 @@ def technical_404_response(request, exception): or (request.path == '/' and len(tried) == 1 # default URLconf and len(tried[0]) == 1 - and tried[0][0].app_name == tried[0][0].namespace == 'admin')): + and getattr(tried[0][0], 'app_name', '') == getattr(tried[0][0], 'namespace', '') == 'admin')): return default_urlconf(request) urlconf = getattr(request, 'urlconf', settings.ROOT_URLCONF) diff --git a/tests/view_tests/default_urls.py b/tests/view_tests/default_urls.py new file mode 100644 index 0000000000..511689ccb1 --- /dev/null +++ b/tests/view_tests/default_urls.py @@ -0,0 +1,8 @@ +from django.conf.urls import patterns, include, url + +from django.contrib import admin + +urlpatterns = patterns('', + # This is the same as in the default project template + url(r'^admin/', include(admin.site.urls)), +) diff --git a/tests/view_tests/regression_21530_urls.py b/tests/view_tests/regression_21530_urls.py new file mode 100644 index 0000000000..14248d872b --- /dev/null +++ b/tests/view_tests/regression_21530_urls.py @@ -0,0 +1,5 @@ +from django.conf.urls import patterns, url + +urlpatterns = patterns('', + url(r'^index/$', 'view_tests.views.index_page', name='index'), +) diff --git a/tests/view_tests/tests/test_debug.py b/tests/view_tests/tests/test_debug.py index 7be5b43a96..1f84933261 100644 --- a/tests/view_tests/tests/test_debug.py +++ b/tests/view_tests/tests/test_debug.py @@ -139,6 +139,35 @@ class DebugViewTests(TestCase): """ self.assertRaises(TemplateDoesNotExist, self.client.get, '/render_no_template/') + @override_settings(ROOT_URLCONF='view_tests.default_urls') + def test_default_urlconf_template(self): + """ + Make sure that the default urlconf template is shown shown instead + of the technical 404 page, if the user has not altered their + url conf yet. + """ + response = self.client.get('/') + self.assertContains( + response, + "

    Congratulations on your first Django-powered page.

    " + ) + + @override_settings(ROOT_URLCONF='view_tests.regression_21530_urls') + def test_regression_21530(self): + """ + Regression test for bug #21530. + + If the admin app include is replaced with exactly one url + pattern, then the technical 404 template should be displayed. + + The bug here was that an AttributeError caused a 500 response. + """ + response = self.client.get('/') + self.assertContains( + response, + "Page not found (404)", + status_code=404 + ) class ExceptionReporterTests(TestCase): rf = RequestFactory() From 19b22d4f0e0eafec1aafb9489f0830ae572b5db5 Mon Sep 17 00:00:00 2001 From: Baptiste Mispelon Date: Sat, 7 Dec 2013 03:37:31 +0100 Subject: [PATCH 062/158] Added fix for #21530 to 1.6.1 release notes. --- docs/releases/1.6.1.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/releases/1.6.1.txt b/docs/releases/1.6.1.txt index 4b23737d94..5eeb446b88 100644 --- a/docs/releases/1.6.1.txt +++ b/docs/releases/1.6.1.txt @@ -41,3 +41,5 @@ Bug fixes backend (#21448). * Fixed a crash when a ``GeometryField`` uses a non-geometric widget (#21496). * Fixed password hash upgrade when changing the iteration count (#21535). +* Fixed a bug in the debug view when the urlconf only contains one element + (#21530). From a1a26690b9c060e2ba1cf6437d5fa9b9a8952017 Mon Sep 17 00:00:00 2001 From: Vajrasky Kok Date: Sat, 7 Dec 2013 16:28:22 +0800 Subject: [PATCH 063/158] Fixed #21572 -- Added unit test for django.utils.text.normalize_newlines. --- tests/utils_tests/test_text.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/utils_tests/test_text.py b/tests/utils_tests/test_text.py index 6758ee58ed..2f0e9ede6e 100644 --- a/tests/utils_tests/test_text.py +++ b/tests/utils_tests/test_text.py @@ -107,6 +107,13 @@ class TestUtilsText(SimpleTestCase): self.assertEqual(text.wrap('a %s word' % long_word, 10), 'a\n%s\nword' % long_word) + def test_normalize_newlines(self): + self.assertEqual(text.normalize_newlines("abc\ndef\rghi\r\n"), + "abc\ndef\nghi\n") + self.assertEqual(text.normalize_newlines("\n\r\r\n\r"), "\n\n\n\n") + self.assertEqual(text.normalize_newlines("abcdefghi"), "abcdefghi") + self.assertEqual(text.normalize_newlines(""), "") + def test_slugify(self): items = ( ('Hello, World!', 'hello-world'), From 8a9c8bb90736451e6bdea82723cbb23a695146fb Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 6 Dec 2013 14:40:51 +0100 Subject: [PATCH 064/158] Fixed #21568 -- Added missing ModelMultipleChoiceField to_python method Thanks dibrovsd at gmail.com for the report and Simon Charette for the review. --- django/forms/models.py | 6 ++++++ docs/releases/1.6.1.txt | 3 +++ tests/model_forms_regress/tests.py | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/django/forms/models.py b/django/forms/models.py index 5c2c77cbf2..2d068eba1e 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -1175,6 +1175,12 @@ class ModelMultipleChoiceField(ModelChoiceField): msg = _('Hold down "Control", or "Command" on a Mac, to select more than one.') self.help_text = string_concat(self.help_text, ' ', msg) + def to_python(self, value): + if not value: + return [] + to_py = super(ModelMultipleChoiceField, self).to_python + return [to_py(val) for val in value] + def clean(self, value): if self.required and not value: raise ValidationError(self.error_messages['required'], code='required') diff --git a/docs/releases/1.6.1.txt b/docs/releases/1.6.1.txt index 5eeb446b88..2ac8822c9c 100644 --- a/docs/releases/1.6.1.txt +++ b/docs/releases/1.6.1.txt @@ -22,6 +22,9 @@ Bug fixes raised an error (#21439). * Fixed a regression that prevented editable ``GenericRelation`` subclasses from working in ``ModelForms``. +* Added missing ``to_python`` method for ``ModelMultipleChoiceField`` which + is required in Django 1.6 to properly detect changes from initial values + (#21568). * Fixed ``django.contrib.humanize`` translations where the unicode sequence for the non-breaking space was returned verbatim (#21415). * Fixed :djadmin:`loaddata` error when fixture file name contained any dots diff --git a/tests/model_forms_regress/tests.py b/tests/model_forms_regress/tests.py index 963c7e552d..d6c05f3153 100644 --- a/tests/model_forms_regress/tests.py +++ b/tests/model_forms_regress/tests.py @@ -45,6 +45,29 @@ class ModelMultipleChoiceFieldTests(TestCase): f.clean([p.pk for p in Person.objects.all()[8:9]]) self.assertTrue(self._validator_run) + def test_model_multiple_choice_show_hidden_initial(self): + """ + Test support of show_hidden_initial by ModelMultipleChoiceField. + """ + class PersonForm(forms.Form): + persons = forms.ModelMultipleChoiceField(show_hidden_initial=True, + queryset=Person.objects.all()) + + person1 = Person.objects.create(name="Person 1") + person2 = Person.objects.create(name="Person 2") + + form = PersonForm(initial={'persons': [person1, person2]}, + data={'initial-persons': [str(person1.pk), str(person2.pk)], + 'persons': [str(person1.pk), str(person2.pk)]}) + self.assertTrue(form.is_valid()) + self.assertFalse(form.has_changed()) + + form = PersonForm(initial={'persons': [person1, person2]}, + data={'initial-persons': [str(person1.pk), str(person2.pk)], + 'persons': [str(person2.pk)]}) + self.assertTrue(form.is_valid()) + self.assertTrue(form.has_changed()) + class TripleForm(forms.ModelForm): class Meta: From 41ebc4838d2b09e7f3ece8889e21492902b55dc8 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 5 Dec 2013 22:55:33 +0100 Subject: [PATCH 065/158] Fixed #21551 -- Reenabled loading fixtures from subdirectory This was a regression in Django 1.6 that was only partially restored in 839940f27f. Thanks Jonas Haag for the report. --- django/core/management/commands/loaddata.py | 6 +++++- docs/releases/1.6.1.txt | 3 ++- .../fixtures_1/inner/absolute.json | 9 +++++++++ tests/fixtures_regress/tests.py | 17 ++++++++++------- 4 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures_regress/fixtures_1/inner/absolute.json diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 667fbbf493..59c1343271 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -178,11 +178,15 @@ class Command(BaseCommand): if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) - if os.path.sep in fixture_name: + if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs + if os.path.sep in fixture_name: + fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name)) + for dir_ in fixture_dirs] + fixture_name = os.path.basename(fixture_name) suffixes = ('.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) diff --git a/docs/releases/1.6.1.txt b/docs/releases/1.6.1.txt index 2ac8822c9c..ec81848d19 100644 --- a/docs/releases/1.6.1.txt +++ b/docs/releases/1.6.1.txt @@ -28,7 +28,8 @@ Bug fixes * Fixed ``django.contrib.humanize`` translations where the unicode sequence for the non-breaking space was returned verbatim (#21415). * Fixed :djadmin:`loaddata` error when fixture file name contained any dots - non related to file extensions (#21457). + non related to file extensions (#21457) or when fixture path was relative + but located in a subdirectory (#21551). * Fixed display of inline instances in formsets when parent has 0 for primary key (#21472). * Fixed a regression where custom querysets for foreign keys were overwritten diff --git a/tests/fixtures_regress/fixtures_1/inner/absolute.json b/tests/fixtures_regress/fixtures_1/inner/absolute.json new file mode 100644 index 0000000000..d62ac03fff --- /dev/null +++ b/tests/fixtures_regress/fixtures_1/inner/absolute.json @@ -0,0 +1,9 @@ +[ + { + "pk": "1", + "model": "fixtures_regress.absolute", + "fields": { + "name": "Load Absolute Path Test" + } + } +] diff --git a/tests/fixtures_regress/tests.py b/tests/fixtures_regress/tests.py index c5ef83bee0..3757dc5180 100644 --- a/tests/fixtures_regress/tests.py +++ b/tests/fixtures_regress/tests.py @@ -2,6 +2,7 @@ # Unittests for fixtures. from __future__ import unicode_literals +import json import os import re import warnings @@ -19,12 +20,13 @@ from django.utils.encoding import force_text from django.utils._os import upath from django.utils import six from django.utils.six import PY3, StringIO -import json from .models import (Animal, Stuff, Absolute, Parent, Child, Article, Widget, Store, Person, Book, NKChild, RefToNKChild, Circle1, Circle2, Circle3, ExternalDependency, Thingy) +_cur_dir = os.path.dirname(os.path.abspath(upath(__file__))) + class TestFixtures(TestCase): @@ -150,12 +152,11 @@ class TestFixtures(TestCase): ) self.assertEqual(Absolute.objects.count(), 1) - def test_relative_path(self): - directory = os.path.dirname(upath(__file__)) - relative_path = os.path.join('fixtures', 'absolute.json') + def test_relative_path(self, path=['fixtures', 'absolute.json']): + relative_path = os.path.join(*path) cwd = os.getcwd() try: - os.chdir(directory) + os.chdir(_cur_dir) management.call_command( 'loaddata', relative_path, @@ -165,6 +166,10 @@ class TestFixtures(TestCase): os.chdir(cwd) self.assertEqual(Absolute.objects.count(), 1) + @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1')]) + def test_relative_path_in_fixture_dirs(self): + self.test_relative_path(path=['inner', 'absolute.json']) + def test_path_containing_dots(self): management.call_command( 'loaddata', @@ -424,8 +429,6 @@ class TestFixtures(TestCase): verbosity=0, ) - _cur_dir = os.path.dirname(os.path.abspath(upath(__file__))) - @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'), os.path.join(_cur_dir, 'fixtures_2')]) def test_loaddata_forward_refs_split_fixtures(self): From a8f4553aaecc7bc6775e0fd54f8c615c792b3d97 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Wed, 4 Dec 2013 14:11:34 +0700 Subject: [PATCH 066/158] Fixed #21555 -- Made ValidationError pickable. Thanks trac username zanuxzan for the report and original patch. --- django/core/exceptions.py | 10 +++++++- tests/validation/test_picklable.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/validation/test_picklable.py diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 87b173b9b0..78f6d07cf3 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -4,6 +4,7 @@ Global Django exception and warning classes. from functools import reduce import operator +from django.utils import six from django.utils.encoding import force_text @@ -84,10 +85,17 @@ class ValidationError(Exception): list or dictionary can be an actual `list` or `dict` or an instance of ValidationError with its `error_list` or `error_dict` attribute set. """ + + # PY2 can't pickle naive exception: http://bugs.python.org/issue1692335. + super(ValidationError, self).__init__(message, code, params) + if isinstance(message, ValidationError): if hasattr(message, 'error_dict'): message = message.error_dict - elif not hasattr(message, 'message'): + # PY2 has a `message` property which is always there so we can't + # duck-type on it. It was introduced in Python 2.5 and already + # deprecated in Python 2.6. + elif not hasattr(message, 'message' if six.PY3 else 'code'): message = message.error_list else: message, code, params = message.message, message.code, message.params diff --git a/tests/validation/test_picklable.py b/tests/validation/test_picklable.py new file mode 100644 index 0000000000..a8d84dba20 --- /dev/null +++ b/tests/validation/test_picklable.py @@ -0,0 +1,41 @@ +import pickle +from unittest import TestCase + +from django.core.exceptions import ValidationError + + +class PickableValidationErrorTestCase(TestCase): + + def test_validationerror_is_picklable(self): + original = ValidationError('a', code='something') + unpickled = pickle.loads(pickle.dumps(original)) + self.assertIs(unpickled, unpickled.error_list[0]) + self.assertEqual(original.message, unpickled.message) + self.assertEqual(original.code, unpickled.code) + + original = ValidationError('a', code='something') + unpickled = pickle.loads(pickle.dumps(ValidationError(original))) + self.assertIs(unpickled, unpickled.error_list[0]) + self.assertEqual(original.message, unpickled.message) + self.assertEqual(original.code, unpickled.code) + + original = ValidationError(['a', 'b']) + unpickled = pickle.loads(pickle.dumps(original)) + self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message) + self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message) + + original = ValidationError(['a', 'b']) + unpickled = pickle.loads(pickle.dumps(ValidationError(original))) + self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message) + self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message) + + original = ValidationError([ValidationError('a'), ValidationError('b')]) + unpickled = pickle.loads(pickle.dumps(original)) + self.assertIs(unpickled.args[0][0], unpickled.error_list[0]) + self.assertEqual(original.error_list[0].message, unpickled.error_list[0].message) + self.assertEqual(original.error_list[1].message, unpickled.error_list[1].message) + + message_dict = {'field1': ['a', 'b'], 'field2': ['c', 'd']} + original = ValidationError(message_dict) + unpickled = pickle.loads(pickle.dumps(original)) + self.assertEqual(unpickled.message_dict, message_dict) From e7dcd40da29008070267828fc15f5dfc94493111 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 7 Dec 2013 07:06:28 -0500 Subject: [PATCH 067/158] Added extra newline for flake8. --- tests/view_tests/tests/test_debug.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/view_tests/tests/test_debug.py b/tests/view_tests/tests/test_debug.py index 1f84933261..261c534b2b 100644 --- a/tests/view_tests/tests/test_debug.py +++ b/tests/view_tests/tests/test_debug.py @@ -169,6 +169,7 @@ class DebugViewTests(TestCase): status_code=404 ) + class ExceptionReporterTests(TestCase): rf = RequestFactory() From 65faa84de3fd3f42e938301574cccdec1578657d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 7 Dec 2013 15:52:30 +0100 Subject: [PATCH 068/158] Removed unneeded string normalization in contrib.admin With Python 2.7 and up, named parameter keys are not limited to bytestrings any longer. This mainly reverts 3bd384aa626. --- django/contrib/admin/helpers.py | 25 +------------------------ django/contrib/admin/views/main.py | 9 +-------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/django/contrib/admin/helpers.py b/django/contrib/admin/helpers.py index 012a14ef64..a2a7df945a 100644 --- a/django/contrib/admin/helpers.py +++ b/django/contrib/admin/helpers.py @@ -30,7 +30,7 @@ checkbox = forms.CheckboxInput({'class': 'action-select'}, lambda value: False) class AdminForm(object): def __init__(self, form, fieldsets, prepopulated_fields, readonly_fields=None, model_admin=None): - self.form, self.fieldsets = form, normalize_fieldsets(fieldsets) + self.form, self.fieldsets = form, fieldsets self.prepopulated_fields = [{ 'field': form[field_name], 'dependencies': [form[f] for f in dependencies] @@ -334,26 +334,3 @@ class AdminErrorList(forms.utils.ErrorList): self.extend(inline_formset.non_form_errors()) for errors_in_inline_form in inline_formset.errors: self.extend(list(six.itervalues(errors_in_inline_form))) - - -def normalize_fieldsets(fieldsets): - """ - Make sure the keys in fieldset dictionaries are strings. Returns the - normalized data. - """ - result = [] - for name, options in fieldsets: - result.append((name, normalize_dictionary(options))) - return result - - -def normalize_dictionary(data_dict): - """ - Converts all the keys in "data_dict" to strings. The keys must be - convertible using str(). - """ - for key, value in data_dict.items(): - if not isinstance(key, str): - del data_dict[key] - data_dict[str(key)] = value - return data_dict diff --git a/django/contrib/admin/views/main.py b/django/contrib/admin/views/main.py index 4b0da928a2..8649b10caa 100644 --- a/django/contrib/admin/views/main.py +++ b/django/contrib/admin/views/main.py @@ -9,7 +9,7 @@ from django.db import models from django.db.models.fields import FieldDoesNotExist from django.utils import six from django.utils.deprecation import RenameMethodsBase -from django.utils.encoding import force_str, force_text +from django.utils.encoding import force_text from django.utils.translation import ugettext, ugettext_lazy from django.utils.http import urlencode @@ -142,14 +142,7 @@ class ChangeList(six.with_metaclass(RenameChangeListMethods)): lookup_params = self.get_filters_params() use_distinct = False - # Normalize the types of keys for key, value in lookup_params.items(): - if not isinstance(key, str): - # 'key' will be used as a keyword argument later, so Python - # requires it to be a string. - del lookup_params[key] - lookup_params[force_str(key)] = value - if not self.model_admin.lookup_allowed(key, value): raise DisallowedModelAdminLookup("Filtering by %s not allowed" % key) From 2e3c7d882015375c130c21884d83cb9fb7759d94 Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Sat, 7 Dec 2013 23:01:17 +0700 Subject: [PATCH 069/158] Trigger AttributeError in ValidationError.message_dict when error_dict is missing. The goal of this change is twofold; firstly, matching the behavior of Django 1.6 and secondly, an AttributeError is more informative than an obscure ValueError about mismatching sequence lengths. Refs #20867. --- django/core/exceptions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django/core/exceptions.py b/django/core/exceptions.py index 78f6d07cf3..cf4aea0f36 100644 --- a/django/core/exceptions.py +++ b/django/core/exceptions.py @@ -123,6 +123,10 @@ class ValidationError(Exception): @property def message_dict(self): + # Trigger an AttributeError if this ValidationError + # doesn't have an error_dict. + getattr(self, 'error_dict') + return dict(self) @property From a7cf48a2b7878bf98879f2963ab9a7d0e6493d9a Mon Sep 17 00:00:00 2001 From: Vajrasky Kok Date: Sat, 7 Dec 2013 16:54:50 +0800 Subject: [PATCH 070/158] Fixed #21573 -- Improved performance of utils.text.normalize_newlines. --- django/utils/text.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/utils/text.py b/django/utils/text.py index bb05afefc4..b29e125f03 100644 --- a/django/utils/text.py +++ b/django/utils/text.py @@ -24,6 +24,7 @@ capfirst = allow_lazy(capfirst, six.text_type) # Set up regular expressions re_words = re.compile(r'<.*?>|((?:\w[-\w]*|&.*?;)+)', re.U | re.S) re_tag = re.compile(r'<(/)?([^ ]+?)(?:(\s*/)| .*?)?>', re.S) +re_newlines = re.compile(r'\r\n|\r') # Used in normalize_newlines def wrap(text, width): @@ -249,7 +250,7 @@ get_text_list = allow_lazy(get_text_list, six.text_type) def normalize_newlines(text): - return force_text(re.sub(r'\r\n|\r|\n', '\n', text)) + return force_text(re_newlines.sub('\n', text)) normalize_newlines = allow_lazy(normalize_newlines, six.text_type) From 5c61b8519d8ce02e4308201f3e81fdc99ad80b58 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 7 Dec 2013 19:28:01 +0100 Subject: [PATCH 071/158] Fixed #18531 -- Deprecated Geo Sitemaps I've chosen a quick deprecation path, as Geo Sitemaps themselves are deprecated from some time now. --- django/contrib/gis/sitemaps/views.py | 6 ++++++ django/contrib/gis/tests/geoapp/test_sitemaps.py | 5 ++++- docs/internals/deprecation.txt | 4 ++++ docs/releases/1.7.txt | 6 ++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/django/contrib/gis/sitemaps/views.py b/django/contrib/gis/sitemaps/views.py index c4c08de8f5..0672b800cc 100644 --- a/django/contrib/gis/sitemaps/views.py +++ b/django/contrib/gis/sitemaps/views.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import warnings + from django.http import HttpResponse, Http404 from django.template import loader from django.contrib.sites.models import get_current_site @@ -20,6 +22,8 @@ def index(request, sitemaps): This view generates a sitemap index that uses the proper view for resolving geographic section sitemap URLs. """ + warnings.warn("Geo Sitemaps are deprecated. Use plain sitemaps from " + "django.contrib.sitemaps instead", DeprecationWarning, stacklevel=2) current_site = get_current_site(request) sites = [] protocol = request.scheme @@ -43,6 +47,8 @@ def sitemap(request, sitemaps, section=None): This view generates a sitemap with additional geographic elements defined by Google. """ + warnings.warn("Geo Sitemaps are deprecated. Use plain sitemaps from " + "django.contrib.sitemaps instead", DeprecationWarning, stacklevel=2) maps, urls = [], [] if section is not None: if section not in sitemaps: diff --git a/django/contrib/gis/tests/geoapp/test_sitemaps.py b/django/contrib/gis/tests/geoapp/test_sitemaps.py index d9ede80c57..0439a37644 100644 --- a/django/contrib/gis/tests/geoapp/test_sitemaps.py +++ b/django/contrib/gis/tests/geoapp/test_sitemaps.py @@ -11,6 +11,7 @@ from django.contrib.gis.geos import HAS_GEOS from django.contrib.gis.tests.utils import HAS_SPATIAL_DB from django.contrib.sites.models import Site from django.test import TestCase +from django.test.utils import IgnoreDeprecationWarningsMixin from django.utils._os import upath if HAS_GEOS: @@ -18,17 +19,19 @@ if HAS_GEOS: @skipUnless(HAS_GEOS and HAS_SPATIAL_DB, "Geos and spatial db are required.") -class GeoSitemapTest(TestCase): +class GeoSitemapTest(IgnoreDeprecationWarningsMixin, TestCase): urls = 'django.contrib.gis.tests.geoapp.urls' def setUp(self): + super(GeoSitemapTest, self).setUp() Site(id=settings.SITE_ID, domain="example.com", name="example.com").save() self.old_Site_meta_installed = Site._meta.installed Site._meta.installed = True def tearDown(self): Site._meta.installed = self.old_Site_meta_installed + super(GeoSitemapTest, self).tearDown() def assertChildNodes(self, elem, expected): "Taken from syndication/tests.py." diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 4c44c70501..2e53901536 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -147,6 +147,10 @@ these changes. * The session key ``django_language`` will no longer be read for backwards compatibility. +* Geographic Sitemaps will be removed + (``django.contrib.gis.sitemaps.views.index`` and + ``django.contrib.gis.sitemaps.views.sitemap``). + 1.9 --- diff --git a/docs/releases/1.7.txt b/docs/releases/1.7.txt index 0d37a8d99a..5014358ae9 100644 --- a/docs/releases/1.7.txt +++ b/docs/releases/1.7.txt @@ -854,3 +854,9 @@ The function ``memoize`` is deprecated and should be replaced by the Django ships a backport of this decorator for older Python versions and it's available at ``django.utils.lru_cache.lru_cache``. The deprecated function will be removed in Django 1.9. + +Geo Sitemaps +~~~~~~~~~~~~ + +Google has retired support for the Geo Sitemaps format. Hence Django support +for Geo Sitemaps is deprecated and will be removed in Django 1.8. From e2e2482391a4e7bd519be9b72e92ed00b2ab2f07 Mon Sep 17 00:00:00 2001 From: maurycyp Date: Sat, 7 Dec 2013 20:13:53 -0500 Subject: [PATCH 072/158] Renamed first argument of class method to cls --- django/contrib/humanize/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/humanize/tests.py b/django/contrib/humanize/tests.py index d2bb4c034e..0f31beb81f 100644 --- a/django/contrib/humanize/tests.py +++ b/django/contrib/humanize/tests.py @@ -28,7 +28,7 @@ now = datetime.datetime(2012, 3, 9, 22, 30) class MockDateTime(datetime.datetime): @classmethod - def now(self, tz=None): + def now(cls, tz=None): if tz is None or tz.utcoffset(now) is None: return now else: From 27dc7908d59385950b0f3ff1aa4bf0cddd787a2a Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Sun, 8 Dec 2013 20:16:45 +0700 Subject: [PATCH 073/158] Made flake8 ignore the .git directory. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 393d0dc094..d777622729 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ doc_files = docs extras AUTHORS INSTALL LICENSE README.rst install-script = scripts/rpm-install.sh [flake8] -exclude=./django/utils/dictconfig.py,./django/contrib/comments/*,./django/utils/unittest.py,./django/utils/lru_cache.py,./tests/comment_tests/*,./django/test/_doctest.py,./django/utils/six.py,./django/conf/app_template/* +exclude=.git,./django/utils/dictconfig.py,./django/contrib/comments/*,./django/utils/unittest.py,./django/utils/lru_cache.py,./tests/comment_tests/*,./django/test/_doctest.py,./django/utils/six.py,./django/conf/app_template/* ignore=E124,E127,E128,E501,W601 [metadata] From f876552f4bd3f4b87441856b6e93d97938c45d39 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Sun, 8 Dec 2013 11:14:18 -0300 Subject: [PATCH 074/158] (Re-)added GeoDjango instructions for building pysqlite2 correctly. This is a partial undo of 1b142ef5dd. --- docs/ref/contrib/gis/install/index.txt | 2 +- docs/ref/contrib/gis/install/spatialite.txt | 54 +++++++++++++++++++++ docs/ref/databases.txt | 5 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/docs/ref/contrib/gis/install/index.txt b/docs/ref/contrib/gis/install/index.txt index 3cf8a29822..9c46b19493 100644 --- a/docs/ref/contrib/gis/install/index.txt +++ b/docs/ref/contrib/gis/install/index.txt @@ -285,7 +285,7 @@ William Kyngesburye provides a number of `geospatial library binary packages`__ that make it simple to get GeoDjango installed on OS X without compiling them from source. However, the `Apple Developer Tools`_ are still necessary for compiling the Python database adapters :ref:`psycopg2_kyngchaos` (for PostGIS) -and ``pysqlite2`` (for SpatiaLite). +and :ref:`pysqlite2` (for SpatiaLite). .. note:: diff --git a/docs/ref/contrib/gis/install/spatialite.txt b/docs/ref/contrib/gis/install/spatialite.txt index ab40600205..b625942fa6 100644 --- a/docs/ref/contrib/gis/install/spatialite.txt +++ b/docs/ref/contrib/gis/install/spatialite.txt @@ -107,6 +107,60 @@ Finally, do the same for the SpatiaLite tools:: __ http://www.gaia-gis.it/gaia-sins/libspatialite-sources/ +.. _pysqlite2: + +pysqlite2 +^^^^^^^^^ + +If you've decided to use a :ref:`newer version of pysqlite2 +` instead of the ``sqlite3`` Python stdlib +module, then you need to make sure it can load external extensions (i.e. the +required ``enable_load_extension`` method is available so ``SpatiaLite`` can be +loaded). + +This might involve building it yourself. For this, download pysqlite2 2.6, and +untar:: + + $ wget https://pypi.python.org/packages/source/p/pysqlite/pysqlite-2.6.3.tar.gz + $ tar xzf pysqlite-2.6.3.tar.gz + $ cd pysqlite-2.6.3 + +Next, use a text editor (e.g., ``emacs`` or ``vi``) to edit the ``setup.cfg`` file +to look like the following: + +.. code-block:: ini + + [build_ext] + #define= + include_dirs=/usr/local/include + library_dirs=/usr/local/lib + libraries=sqlite3 + #define=SQLITE_OMIT_LOAD_EXTENSION + +or if you are on Mac OS X: + +.. code-block:: ini + + [build_ext] + #define= + include_dirs=/Library/Frameworks/SQLite3.framework/unix/include + library_dirs=/Library/Frameworks/SQLite3.framework/unix/lib + libraries=sqlite3 + #define=SQLITE_OMIT_LOAD_EXTENSION + +.. note:: + + The important thing here is to make sure you comment out the + ``define=SQLITE_OMIT_LOAD_EXTENSION`` flag and that the ``include_dirs`` + and ``library_dirs`` settings are uncommented and set to the appropriate + path if the SQLite header files and libraries are not in ``/usr/include`` + and ``/usr/lib``, respectively. + +After modifying ``setup.cfg`` appropriately, then run the ``setup.py`` script +to build and install:: + + $ sudo python setup.py install + .. _spatialite_macosx: Mac OS X-specific instructions diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 2b13af8ca2..0ff1861b2a 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -604,9 +604,8 @@ version of SQLite. Using newer versions of the SQLite DB-API 2.0 driver ---------------------------------------------------- -For versions of Python 2.5 or newer that include ``sqlite3`` in the standard -library Django will now use a ``pysqlite2`` interface in preference to -``sqlite3`` if it finds one is available. +Django will use a ``pysqlite2`` module in preference to ``sqlite3`` as shipped +with the Python standard library if it finds one is available. This provides the ability to upgrade both the DB-API 2.0 interface or SQLite 3 itself to versions newer than the ones included with your particular Python From 626bdf648a2c4c4f5fd25fbc4af41a1acfa18d7f Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sun, 8 Dec 2013 18:39:26 +0100 Subject: [PATCH 075/158] Updated a bunch of hyperlinks in documentation --- docs/faq/general.txt | 8 ++++---- docs/howto/deployment/index.txt | 2 +- docs/howto/deployment/wsgi/gunicorn.txt | 2 +- docs/howto/deployment/wsgi/uwsgi.txt | 6 +++--- docs/howto/jython.txt | 2 +- docs/index.txt | 2 +- docs/internals/committers.txt | 6 +++--- docs/internals/contributing/localizing.txt | 4 ++-- .../contributing/writing-code/coding-style.txt | 2 +- .../internals/contributing/writing-code/unit-tests.txt | 6 +++--- docs/internals/mailing-lists.txt | 10 +++++----- docs/internals/security.txt | 2 +- docs/intro/index.txt | 4 ++-- docs/intro/reusable-apps.txt | 10 +++++----- docs/ref/contrib/gis/admin.txt | 2 +- docs/ref/contrib/gis/feeds.txt | 2 +- docs/ref/contrib/gis/forms-api.txt | 2 +- docs/ref/contrib/gis/install/index.txt | 4 ++-- docs/ref/contrib/gis/install/postgis.txt | 2 +- docs/ref/contrib/gis/measure.txt | 2 +- docs/ref/contrib/gis/sitemaps.txt | 2 +- docs/ref/contrib/gis/tutorial.txt | 2 +- docs/ref/contrib/sitemaps.txt | 2 +- docs/ref/databases.txt | 2 +- docs/releases/1.2.txt | 2 +- docs/topics/install.txt | 2 +- docs/topics/logging.txt | 2 +- docs/topics/performance.txt | 2 +- docs/topics/python3.txt | 6 +++--- docs/topics/templates.txt | 2 +- docs/topics/testing/advanced.txt | 2 +- docs/topics/testing/overview.txt | 4 ++-- 32 files changed, 55 insertions(+), 55 deletions(-) diff --git a/docs/faq/general.txt b/docs/faq/general.txt index 5db3141f82..801cb334ca 100644 --- a/docs/faq/general.txt +++ b/docs/faq/general.txt @@ -159,7 +159,7 @@ Django has special conveniences for building "CMS-y" apps, that doesn't mean it's not just as appropriate for building "non-CMS-y" apps (whatever that means!). -.. _Drupal: http://drupal.org/ +.. _Drupal: https://drupal.org/ How can I download the Django documentation to read it offline? --------------------------------------------------------------- @@ -183,7 +183,7 @@ Where can I find Django developers for hire? Consult our `developers for hire page`_ for a list of Django developers who would be happy to help you. -You might also be interested in posting a job to http://djangogigs.com/ . +You might also be interested in posting a job to https://djangogigs.com/ . If you want to find Django-capable people in your local area, try https://people.djangoproject.com/ . @@ -198,7 +198,7 @@ software are still a matter of some debate. For example, `APA style`_, would dictate something like:: - Django (Version 1.5) [Computer Software]. (2013). Retrieved from http://djangoproject.com. + Django (Version 1.5) [Computer Software]. (2013). Retrieved from https://djangoproject.com. However, the only true guide is what your publisher will accept, so get a copy of those guidelines and fill in the gaps as best you can. @@ -208,7 +208,7 @@ Foundation". If you need a publishing location, use "Lawrence, Kansas". -If you need a web address, use http://djangoproject.com. +If you need a web address, use https://djangoproject.com. If you need a name, just use "Django", without any tagline. diff --git a/docs/howto/deployment/index.txt b/docs/howto/deployment/index.txt index 688bae2397..1a5d137596 100644 --- a/docs/howto/deployment/index.txt +++ b/docs/howto/deployment/index.txt @@ -31,4 +31,4 @@ the easiest, fastest, and most stable deployment choice. ``mod_python`` was first deprecated, then completely removed in Django 1.5. -.. _chapter 12 of the django book (second edition): http://djangobook.com/en/2.0/chapter12/ +.. _chapter 12 of the django book (second edition): http://djangobook.com/en/2.0/chapter12.html diff --git a/docs/howto/deployment/wsgi/gunicorn.txt b/docs/howto/deployment/wsgi/gunicorn.txt index 14c80af0a0..95051b690b 100644 --- a/docs/howto/deployment/wsgi/gunicorn.txt +++ b/docs/howto/deployment/wsgi/gunicorn.txt @@ -13,7 +13,7 @@ There are two ways to use Gunicorn with Django. One is to have Gunicorn treat Django as just another WSGI application. The second is to use Gunicorn's special `integration with Django`_. -.. _integration with Django: http://gunicorn.org/run.html#django-manage-py +.. _integration with Django: http://docs.gunicorn.org/en/latest/run.html#django-manage-py Installing Gunicorn =================== diff --git a/docs/howto/deployment/wsgi/uwsgi.txt b/docs/howto/deployment/wsgi/uwsgi.txt index 59dddda418..47f1e7a61d 100644 --- a/docs/howto/deployment/wsgi/uwsgi.txt +++ b/docs/howto/deployment/wsgi/uwsgi.txt @@ -32,7 +32,7 @@ command. For example: # Or install LTS (long term support). $ sudo pip install http://projects.unbit.it/downloads/uwsgi-lts.tar.gz -.. _installation procedures: http://projects.unbit.it/uwsgi/wiki/Install +.. _installation procedures: http://uwsgi-docs.readthedocs.org/en/latest/Install.html .. warning:: @@ -58,7 +58,7 @@ Configuring and starting the uWSGI server for Django uWSGI supports multiple ways to configure the process. See uWSGI's `configuration documentation`_ and `examples`_ -.. _configuration documentation: http://projects.unbit.it/uwsgi/wiki/Doc +.. _configuration documentation: https://uwsgi.readthedocs.org/en/latest/Configuration.html .. _examples: http://projects.unbit.it/uwsgi/wiki/Example Here's an example command to start a uWSGI server:: @@ -111,4 +111,4 @@ Example ini configuration file usage:: See the uWSGI docs on `managing the uWSGI process`_ for information on starting, stopping and reloading the uWSGI workers. -.. _managing the uWSGI process: http://projects.unbit.it/uwsgi/wiki/Management +.. _managing the uWSGI process: http://uwsgi-docs.readthedocs.org/en/latest/Management.html diff --git a/docs/howto/jython.txt b/docs/howto/jython.txt index 96a3d64251..2d2c22872a 100644 --- a/docs/howto/jython.txt +++ b/docs/howto/jython.txt @@ -30,7 +30,7 @@ such as `Apache Tomcat`_. Full JavaEE applications servers such as `GlassFish`_ or `JBoss`_ are also OK, if you need the extra features they include. .. _`Apache Tomcat`: http://tomcat.apache.org/ -.. _GlassFish: http://glassfish.java.net/ +.. _GlassFish: https://glassfish.java.net/ .. _JBoss: http://www.jboss.org/ Installing Django diff --git a/docs/index.txt b/docs/index.txt index e5c92a2073..bd3baf8a9c 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -26,7 +26,7 @@ Having trouble? We'd like to help! * Report bugs with Django in our `ticket tracker`_. .. _archives: http://groups.google.com/group/django-users/ -.. _post a question: http://groups.google.com/group/django-users/ +.. _post a question: https://groups.google.com/d/forum/django-users .. _#django IRC channel: irc://irc.freenode.net/django .. _IRC logs: http://django-irc-logs.com/ .. _ticket tracker: https://code.djangoproject.com/ diff --git a/docs/internals/committers.txt b/docs/internals/committers.txt index 3b37a2ca58..0da26a31eb 100644 --- a/docs/internals/committers.txt +++ b/docs/internals/committers.txt @@ -223,7 +223,7 @@ Karen Tracey Jannis lives in Berlin, Germany. - .. _Jannis Leidel: http://jezdez.com/ + .. _Jannis Leidel: https://jezdez.com/ .. _Bauhaus-University Weimar: http://www.uni-weimar.de/ .. _virtualenv: http://www.virtualenv.org/ .. _pip: http://www.pip-installer.org/ @@ -396,7 +396,7 @@ Tim Graham He works in a `management consulting company`_ in Paris, France. - .. _Aymeric Augustin: http://myks.org/ + .. _Aymeric Augustin: https://myks.org/ .. _management consulting company: http://www.polyconseil.fr/ `Claude Paroz`_ @@ -410,7 +410,7 @@ Tim Graham Django-based `l10n.gnome.org`_. .. _Claude Paroz: http://www.2xlibre.net - .. _l10n.gnome.org: http://l10n.gnome.org + .. _l10n.gnome.org: https://l10n.gnome.org Anssi Kääriäinen Anssi works as a developer at Finnish National Institute for Health and diff --git a/docs/internals/contributing/localizing.txt b/docs/internals/contributing/localizing.txt index 01b88d8d9a..2069ea264c 100644 --- a/docs/internals/contributing/localizing.txt +++ b/docs/internals/contributing/localizing.txt @@ -61,6 +61,6 @@ Django source tree, as for any code change: ``Translations``, and attach the patch to it. .. _Transifex: https://www.transifex.com/ -.. _Django i18n mailing list: http://groups.google.com/group/django-i18n/ +.. _Django i18n mailing list: /https://groups.google.com/d/forum/django-i18n .. _Django project page: https://www.transifex.com/projects/p/django-core/ -.. _Transifex User Guide: http://help.transifex.com/ +.. _Transifex User Guide: http://support.transifex.com/ diff --git a/docs/internals/contributing/writing-code/coding-style.txt b/docs/internals/contributing/writing-code/coding-style.txt index afcf9c072c..c3d1b7c758 100644 --- a/docs/internals/contributing/writing-code/coding-style.txt +++ b/docs/internals/contributing/writing-code/coding-style.txt @@ -213,4 +213,4 @@ Miscellaneous change to the ``AUTHORS`` file in your patch if you make more than a single trivial change. -.. _flake8: http://pypi.python.org/pypi/flake8 +.. _flake8: https://pypi.python.org/pypi/flake8 diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 969635f2ef..2bbd9dcf0e 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -187,11 +187,11 @@ associated tests will be skipped. .. _Pillow: https://pypi.python.org/pypi/Pillow/ .. _PyYAML: http://pyyaml.org/wiki/PyYAML .. _pytz: https://pypi.python.org/pypi/pytz/ -.. _setuptools: http://pypi.python.org/pypi/setuptools/ +.. _setuptools: https://pypi.python.org/pypi/setuptools/ .. _memcached: http://memcached.org/ .. _gettext: http://www.gnu.org/software/gettext/manual/gettext.html -.. _selenium: http://pypi.python.org/pypi/selenium -.. _pip requirements files: http://www.pip-installer.org/en/latest/requirements.html +.. _selenium: https://pypi.python.org/pypi/selenium +.. _pip requirements files: http://www.pip-installer.org/en/latest/cookbook.html#requirements-files Code coverage ~~~~~~~~~~~~~ diff --git a/docs/internals/mailing-lists.txt b/docs/internals/mailing-lists.txt index e5e6ea9a3d..68b8035758 100644 --- a/docs/internals/mailing-lists.txt +++ b/docs/internals/mailing-lists.txt @@ -31,7 +31,7 @@ installation, usage, or debugging of Django. * `django-users subscription email address`_ * `django-users posting email`_ -.. _django-users mailing archive: http://groups.google.com/group/django-users/ +.. _django-users mailing archive: https://groups.google.com/d/forum/django-users .. _django-users subscription email address: mailto:django-users+subscribe@googlegroups.com .. _django-users posting email: mailto:django-users@googlegroups.com @@ -48,7 +48,7 @@ Django development. * `django-core-mentorship subscription email address`_ * `django-core-mentorship posting email`_ -.. _django-core-mentorship mailing archive: http://groups.google.com/group/django-core-mentorship/ +.. _django-core-mentorship mailing archive: https://groups.google.com/d/forum/django-core-mentorship .. _django-core-mentorship subscription email address: mailto:django-core-mentorship+subscribe@googlegroups.com .. _django-core-mentorship posting email: mailto:django-core-mentorship@googlegroups.com @@ -69,7 +69,7 @@ The discussion about the development of Django itself takes place here. * `django-developers subscription email address`_ * `django-developers posting email`_ -.. _django-developers mailing archive: http://groups.google.com/group/django-developers/ +.. _django-developers mailing archive: https://groups.google.com/d/forum/django-developers .. _django-developers subscription email address: mailto:django-developers+subscribe@googlegroups.com .. _django-developers posting email: mailto:django-developers@googlegroups.com @@ -85,7 +85,7 @@ bugfixes. * `django-announce subscription email address`_ * `django-announce posting email`_ -.. _django-announce mailing archive: http://groups.google.com/group/django-announce/ +.. _django-announce mailing archive: https://groups.google.com/d/forum/django-announce .. _django-announce subscription email address: mailto:django-announce+subscribe@googlegroups.com .. _django-announce posting email: mailto:django-announce@googlegroups.com @@ -101,6 +101,6 @@ by developers and interested community members. * `django-updates subscription email address`_ * `django-updates posting email`_ -.. _django-updates mailing archive: http://groups.google.com/group/django-updates/ +.. _django-updates mailing archive: https://groups.google.com/d/forum/django-updates .. _django-updates subscription email address: mailto:django-updates+subscribe@googlegroups.com .. _django-updates posting email: mailto:django-updates@googlegroups.com diff --git a/docs/internals/security.txt b/docs/internals/security.txt index 7c0a9242d5..d9aab377f5 100644 --- a/docs/internals/security.txt +++ b/docs/internals/security.txt @@ -113,7 +113,7 @@ On the day of disclosure, we will take the following steps: 4. Post a notice to the |django-announce| mailing list that links to the blog post. -.. _the Python Package Index: http://pypi.python.org/pypi +.. _the Python Package Index: https://pypi.python.org/pypi .. _the official Django development blog: https://www.djangoproject.com/weblog/ If a reported issue is believed to be particularly time-sensitive -- diff --git a/docs/intro/index.txt b/docs/intro/index.txt index 9e88402f6d..a7872017b9 100644 --- a/docs/intro/index.txt +++ b/docs/intro/index.txt @@ -35,8 +35,8 @@ place: read this material to quickly get up and running. a few other `books about Python`_. .. _python: http://python.org/ - .. _list of Python resources for non-programmers: http://wiki.python.org/moin/BeginnersGuide/NonProgrammers + .. _list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Python 2: http://diveintopython.net/ .. _Python 3: http://diveintopython3.net/ .. _dead-tree version: http://www.amazon.com/exec/obidos/ASIN/1590593561/ref=nosim/jacobian20 - .. _books about Python: http://wiki.python.org/moin/PythonBooks + .. _books about Python: https://wiki.python.org/moin/PythonBooks diff --git a/docs/intro/reusable-apps.txt b/docs/intro/reusable-apps.txt index 04a8ea7417..8959aae06a 100644 --- a/docs/intro/reusable-apps.txt +++ b/docs/intro/reusable-apps.txt @@ -19,7 +19,7 @@ could save some of this repeated work? Reusability is the way of life in Python. `The Python Package Index (PyPI) `_ has a vast range of packages you can use in your own Python programs. Check out `Django -Packages `_ for existing reusable apps you could +Packages `_ for existing reusable apps you could incorporate in your project. Django itself is also just a Python package. This means that you can take existing Python packages or Django apps and compose them into your own web project. You only need to write the parts that make @@ -108,7 +108,7 @@ Django with pip`. You can install ``setuptools`` the same way. .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _pip: http://pypi.python.org/pypi/pip +.. _pip: https://pypi.python.org/pypi/pip Packaging your app ================== @@ -182,7 +182,7 @@ this. For a small app like polls, this process isn't too difficult. 5. Next we'll create a ``setup.py`` file which provides details about how to build and install the app. A full explanation of this file is beyond the scope of this tutorial, but the `setuptools docs - `_ have a good + `_ have a good explanation. Create a file ``django-polls/setup.py`` with the following contents: @@ -246,7 +246,7 @@ this. For a small app like polls, this process isn't too difficult. Note that the ``docs`` directory won't be included in your package unless you add some files to it. Many Django apps also provide their documentation - online through sites like `readthedocs.org `_. + online through sites like `readthedocs.org `_. 8. Try building your package with ``python setup.py sdist`` (run from inside ``django-polls``). This creates a directory called ``dist`` and builds your @@ -285,7 +285,7 @@ working. We'll now fix this by installing our new ``django-polls`` package. pip uninstall django-polls -.. _pip: http://pypi.python.org/pypi/pip +.. _pip: https://pypi.python.org/pypi/pip Publishing your app =================== diff --git a/docs/ref/contrib/gis/admin.txt b/docs/ref/contrib/gis/admin.txt index d1a9fc1dcb..d3bc1d26a1 100644 --- a/docs/ref/contrib/gis/admin.txt +++ b/docs/ref/contrib/gis/admin.txt @@ -67,6 +67,6 @@ GeoDjango's admin site .. class:: OSMGeoAdmin A subclass of :class:`GeoModelAdmin` that uses a spherical mercator projection - with `OpenStreetMap `_ street data tiles. + with `OpenStreetMap `_ street data tiles. See the :ref:`OSMGeoAdmin introduction ` in the tutorial for a usage example. diff --git a/docs/ref/contrib/gis/feeds.txt b/docs/ref/contrib/gis/feeds.txt index 7b1b6ebccf..72c51b57ab 100644 --- a/docs/ref/contrib/gis/feeds.txt +++ b/docs/ref/contrib/gis/feeds.txt @@ -13,7 +13,7 @@ Django's, please consult :doc:`Django's syndication documentation .. _W3C Geo: http://www.w3.org/2003/01/geo/ -__ http://georss.org/1.0#simple +__ http://georss.org/simple.html Example ======= diff --git a/docs/ref/contrib/gis/forms-api.txt b/docs/ref/contrib/gis/forms-api.txt index da4c96792e..435bcfc8b1 100644 --- a/docs/ref/contrib/gis/forms-api.txt +++ b/docs/ref/contrib/gis/forms-api.txt @@ -14,7 +14,7 @@ display and edit geolocalized data on a map. By default, they use `OpenLayers`_-powered maps, with a base WMS layer provided by `Metacarta`_. .. _OpenLayers: http://openlayers.org/ -.. _Metacarta: http://metacarta.com/ +.. _Metacarta: http://www.metacarta.com/ Field arguments =============== diff --git a/docs/ref/contrib/gis/install/index.txt b/docs/ref/contrib/gis/install/index.txt index 9c46b19493..4bd0471572 100644 --- a/docs/ref/contrib/gis/install/index.txt +++ b/docs/ref/contrib/gis/install/index.txt @@ -140,7 +140,7 @@ community! You can: sure to provide a complete description of the problem, versions used, and specify the component as "GIS". -__ http://groups.google.com/group/geodjango +__ https://groups.google.com/d/forum/geodjango __ https://code.djangoproject.com/newticket .. _libsettings: @@ -273,7 +273,7 @@ Summary:: $ brew install gdal $ brew install libgeoip -__ http://mxcl.github.com/homebrew/ +__ http://brew.sh/ .. _Apple Developer Tools: https://developer.apple.com/technologies/tools/ .. _kyngchaos: diff --git a/docs/ref/contrib/gis/install/postgis.txt b/docs/ref/contrib/gis/install/postgis.txt index c651fe8fca..75597fbb38 100644 --- a/docs/ref/contrib/gis/install/postgis.txt +++ b/docs/ref/contrib/gis/install/postgis.txt @@ -15,7 +15,7 @@ might also need additional libraries, see `PostGIS requirements`_. when using GeoDjango with PostGIS. .. _psycopg2: http://initd.org/psycopg/ -.. _PostGIS requirements: http://www.postgis.org/documentation/manual-2.0/postgis_installation.html#id2711662 +.. _PostGIS requirements: http://www.postgis.org/documentation/manual-2.0/postgis_installation.html#id554707 On Debian/Ubuntu, you are advised to install the following packages: postgresql-x.x, postgresql-x.x-postgis, postgresql-server-dev-x.x, diff --git a/docs/ref/contrib/gis/measure.txt b/docs/ref/contrib/gis/measure.txt index 2a05b42600..197f584729 100644 --- a/docs/ref/contrib/gis/measure.txt +++ b/docs/ref/contrib/gis/measure.txt @@ -175,6 +175,6 @@ Measurement API Alias for :class:`Area` class. .. rubric:: Footnotes -.. [#] `Robert Coup `_ is the initial author of the measure objects, +.. [#] `Robert Coup `_ is the initial author of the measure objects, and was inspired by Brian Beck's work in `geopy `_ and Geoff Biggs' PhD work on dimensioned units for robotics. diff --git a/docs/ref/contrib/gis/sitemaps.txt b/docs/ref/contrib/gis/sitemaps.txt index 0ab8f75825..60df2b87b2 100644 --- a/docs/ref/contrib/gis/sitemaps.txt +++ b/docs/ref/contrib/gis/sitemaps.txt @@ -23,4 +23,4 @@ Reference ----------------- .. rubric:: Footnotes -.. [#] Google, Inc., `What is a Geo Sitemap? `_. +.. [#] Google, Inc., `What is a Geo Sitemap? `_. diff --git a/docs/ref/contrib/gis/tutorial.txt b/docs/ref/contrib/gis/tutorial.txt index 17ce1edbf0..183b2b9875 100644 --- a/docs/ref/contrib/gis/tutorial.txt +++ b/docs/ref/contrib/gis/tutorial.txt @@ -747,7 +747,7 @@ entries -- the borders may be edited by clicking on a polygon and dragging the vertexes to the desired position. .. _OpenLayers: http://openlayers.org/ -.. _Open Street Map: http://openstreetmap.org/ +.. _Open Street Map: http://www.openstreetmap.org/ .. _Vector Map Level 0: http://earth-info.nga.mil/publications/vmap0.html .. _OSGeo: http://www.osgeo.org diff --git a/docs/ref/contrib/sitemaps.txt b/docs/ref/contrib/sitemaps.txt index 7b53fc8c12..fefe90a420 100644 --- a/docs/ref/contrib/sitemaps.txt +++ b/docs/ref/contrib/sitemaps.txt @@ -454,7 +454,7 @@ generate a Google News compatible sitemap: {% endspaceless %} -.. _`Google news sitemaps`: http://support.google.com/webmasters/bin/answer.py?hl=en&answer=74288 +.. _`Google news sitemaps`: https://support.google.com/webmasters/answer/74288?hl=en Pinging Google ============== diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 0ff1861b2a..3429531d76 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -866,5 +866,5 @@ the support channels provided by each 3rd party project. .. _IBM DB2: http://code.google.com/p/ibm-db/ .. _Microsoft SQL Server 2005: http://code.google.com/p/django-mssql/ .. _Firebird: http://code.google.com/p/django-firebird/ -.. _ODBC: https://github.com/aurorasoftware/django-pyodbc/ +.. _ODBC: https://github.com/lionheart/django-pyodbc/ .. _ADSDB: http://code.google.com/p/adsdb-django/ diff --git a/docs/releases/1.2.txt b/docs/releases/1.2.txt index ac355a1d87..387f2508ef 100644 --- a/docs/releases/1.2.txt +++ b/docs/releases/1.2.txt @@ -50,7 +50,7 @@ be found below`_. `Django Advent`_ covered the release of Django 1.2 with a series of articles and tutorials that cover some of the new features in depth. -.. _django advent: http://djangoadvent.com/ +.. _django advent: https://github.com/djangoadvent/djangoadvent-articles Wherever possible these features have been introduced in a backwards-compatible manner per :doc:`our API stability policy ` policy. diff --git a/docs/topics/install.txt b/docs/topics/install.txt index 923ae44270..1868c73a6b 100644 --- a/docs/topics/install.txt +++ b/docs/topics/install.txt @@ -221,7 +221,7 @@ This is the recommended way to install Django. .. _pip: http://www.pip-installer.org/ .. _virtualenv: http://www.virtualenv.org/ -.. _virtualenvwrapper: http://www.doughellmann.com/docs/virtualenvwrapper/ +.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.org/en/latest/ .. _standalone pip installer: http://www.pip-installer.org/en/latest/installing.html#using-the-installer Installing an official release manually diff --git a/docs/topics/logging.txt b/docs/topics/logging.txt index f4c1043eb4..15c1aa9dd8 100644 --- a/docs/topics/logging.txt +++ b/docs/topics/logging.txt @@ -516,7 +516,7 @@ Python logging module. By default, an instance of the email backend specified in :setting:`EMAIL_BACKEND` will be used. -.. _Sentry: http://pypi.python.org/pypi/sentry +.. _Sentry: https://pypi.python.org/pypi/sentry Filters diff --git a/docs/topics/performance.txt b/docs/topics/performance.txt index 1c1ed86762..1e3bf7b9d4 100644 --- a/docs/topics/performance.txt +++ b/docs/topics/performance.txt @@ -417,7 +417,7 @@ With these caveats in mind, you should be aware of: performance gains, typically for heavyweight applications. A key aim of the PyPy project is `compatibility -`_ with existing Python APIs and libraries. +`_ with existing Python APIs and libraries. Django is compatible, but you will need to check the compatibility of other libraries you rely on. diff --git a/docs/topics/python3.txt b/docs/topics/python3.txt index d059bc042d..15c07ccbf0 100644 --- a/docs/topics/python3.txt +++ b/docs/topics/python3.txt @@ -6,7 +6,7 @@ Django 1.5 is the first version of Django to support Python 3. The same code runs both on Python 2 (≥ 2.6.5) and Python 3 (≥ 3.2), thanks to the six_ compatibility layer. -.. _six: http://packages.python.org/six/ +.. _six: http://pythonhosted.org/six/ This document is primarily targeted at authors of pluggable application who want to support both Python 2 and 3. It also describes guidelines that @@ -42,7 +42,7 @@ developers are used to dealing with such constraints. Porting tools provided by Django are inspired by this philosophy, and it's reflected throughout this guide. -.. _Python's official porting guide: http://docs.python.org/py3k/howto/pyporting.html +.. _Python's official porting guide: http://docs.python.org/3/howto/pyporting.html .. _Pragmatic Unicode: http://nedbatchelder.com/text/unipain.html Porting tips @@ -246,7 +246,7 @@ consequence, the following pattern is sometimes necessary:: Be cautious if you have to `index bytestrings`_. -.. _index bytestrings: http://docs.python.org/py3k/howto/pyporting.html#bytes-literals +.. _index bytestrings: http://docs.python.org/3/howto/pyporting.html#bytes-literals Exceptions ~~~~~~~~~~ diff --git a/docs/topics/templates.txt b/docs/topics/templates.txt index b1389b85f6..53ee292e37 100644 --- a/docs/topics/templates.txt +++ b/docs/topics/templates.txt @@ -30,7 +30,7 @@ or CheetahTemplate_, you should feel right at home with Django's templates. ` to the template language as needed). .. _`The Django template language: For Python programmers`: ../templates_python/ -.. _Smarty: http://smarty.php.net/ +.. _Smarty: http://www.smarty.net/ .. _CheetahTemplate: http://www.cheetahtemplate.org/ Templates diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt index 90111d05ff..936b90b747 100644 --- a/docs/topics/testing/advanced.txt +++ b/docs/topics/testing/advanced.txt @@ -549,4 +549,4 @@ For more options like annotated HTML listings detailing missed lines, see the `coverage.py`_ docs. .. _coverage.py: http://nedbatchelder.com/code/coverage/ -.. _install coverage.py: http://pypi.python.org/pypi/coverage +.. _install coverage.py: https://pypi.python.org/pypi/coverage diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index a0bed1b2ee..f8464638c2 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -31,7 +31,7 @@ module defines tests using a class-based approach. Since Django no longer supports Python versions older than 2.7, ``django.utils.unittest`` is deprecated. Simply use ``unittest``. -.. _unittest2: http://pypi.python.org/pypi/unittest2 +.. _unittest2: https://pypi.python.org/pypi/unittest2 Here is an example which subclasses from :class:`django.test.TestCase`, which is a subclass of :class:`unittest.TestCase` that runs each test inside a @@ -1053,7 +1053,7 @@ example above is just a tiny fraction of what the Selenium client can do; check out the `full reference`_ for more details. .. _Selenium: http://seleniumhq.org/ -.. _selenium package: http://pypi.python.org/pypi/selenium +.. _selenium package: https://pypi.python.org/pypi/selenium .. _full reference: http://selenium-python.readthedocs.org/en/latest/api.html .. _Firefox: http://www.mozilla.com/firefox/ From c047dda057542b3b8722c6433028e3b800c38af9 Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Sun, 8 Dec 2013 13:18:53 -0500 Subject: [PATCH 076/158] Removed an erroneous leading slash introduced by a626bdf648a. --- docs/internals/contributing/localizing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals/contributing/localizing.txt b/docs/internals/contributing/localizing.txt index 2069ea264c..1b23a2296c 100644 --- a/docs/internals/contributing/localizing.txt +++ b/docs/internals/contributing/localizing.txt @@ -61,6 +61,6 @@ Django source tree, as for any code change: ``Translations``, and attach the patch to it. .. _Transifex: https://www.transifex.com/ -.. _Django i18n mailing list: /https://groups.google.com/d/forum/django-i18n +.. _Django i18n mailing list: https://groups.google.com/d/forum/django-i18n .. _Django project page: https://www.transifex.com/projects/p/django-core/ .. _Transifex User Guide: http://support.transifex.com/ From 744aac6dace325752e3b1c7c8af64a7bc655186f Mon Sep 17 00:00:00 2001 From: Christian Schmitt Date: Mon, 9 Dec 2013 22:54:11 +0100 Subject: [PATCH 077/158] Fixed #21560 -- missing 'slug' field in example code I updated the documentation, that the modelform now includes the 'slug' field. --- docs/topics/forms/modelforms.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/forms/modelforms.txt b/docs/topics/forms/modelforms.txt index d8d1c106f8..394cf1f319 100644 --- a/docs/topics/forms/modelforms.txt +++ b/docs/topics/forms/modelforms.txt @@ -528,7 +528,7 @@ field, you could do the following:: class Meta: model = Article - fields = ['pub_date', 'headline', 'content', 'reporter'] + fields = ['pub_date', 'headline', 'content', 'reporter', 'slug'] If you want to specify a field's validators, you can do so by defining From a2814846ca006b4fbf3a893ec52cd9d444b3a3eb Mon Sep 17 00:00:00 2001 From: Loic Bistuer Date: Mon, 9 Dec 2013 00:20:06 +0700 Subject: [PATCH 078/158] Fixed E124 pep8 warnings. --- django/contrib/admin/helpers.py | 3 +- django/contrib/admin/options.py | 6 +- django/contrib/admin/sites.py | 3 +- .../contrib/admin/templatetags/admin_list.py | 3 +- django/contrib/admindocs/tests/test_fields.py | 3 +- django/contrib/admindocs/urls.py | 27 +-- django/contrib/auth/admin.py | 4 +- django/contrib/auth/tests/test_basic.py | 15 +- django/contrib/auth/tests/test_management.py | 15 +- django/contrib/contenttypes/generic.py | 7 +- django/contrib/gis/db/models/query.py | 25 +-- django/contrib/gis/forms/widgets.py | 3 +- .../contrib/gis/tests/geoapp/test_regress.py | 9 +- django/core/management/commands/testserver.py | 3 +- django/core/management/validation.py | 12 +- django/db/backends/__init__.py | 3 +- django/db/backends/mysql/base.py | 8 +- .../db/backends/postgresql_psycopg2/base.py | 3 +- django/db/migrations/optimizer.py | 114 +++++++----- django/db/models/fields/__init__.py | 3 +- django/db/models/fields/related.py | 3 +- django/forms/utils.py | 18 +- django/forms/widgets.py | 2 +- django/test/__init__.py | 3 +- django/test/simple.py | 3 +- django/utils/text.py | 3 +- setup.cfg | 2 +- tests/createsuperuser/tests.py | 3 +- tests/custom_columns/tests.py | 3 +- tests/custom_pk/tests.py | 3 +- tests/extra_regress/tests.py | 29 +-- tests/get_object_or_404/tests.py | 21 ++- tests/get_or_create/tests.py | 9 +- tests/inline_formsets/tests.py | 9 +- tests/m2m_regress/tests.py | 3 +- tests/many_to_one/tests.py | 5 +- tests/middleware_exceptions/tests.py | 4 +- tests/model_validation/tests.py | 14 +- tests/modeladmin/tests.py | 174 ++++++++++++------ tests/proxy_models/tests.py | 18 +- tests/raw_query/tests.py | 4 +- tests/select_related/tests.py | 13 +- tests/staticfiles_tests/tests.py | 6 +- tests/template_tests/test_parser.py | 8 +- tests/test_utils/tests.py | 4 +- 45 files changed, 371 insertions(+), 262 deletions(-) diff --git a/django/contrib/admin/helpers.py b/django/contrib/admin/helpers.py index a2a7df945a..318c393c09 100644 --- a/django/contrib/admin/helpers.py +++ b/django/contrib/admin/helpers.py @@ -42,7 +42,8 @@ class AdminForm(object): def __iter__(self): for name, options in self.fieldsets: - yield Fieldset(self.form, name, + yield Fieldset( + self.form, name, readonly_fields=self.readonly_fields, model_admin=self.model_admin, **options diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py index 51e8a1b2e6..4c5adaf0f8 100644 --- a/django/contrib/admin/options.py +++ b/django/contrib/admin/options.py @@ -1520,7 +1520,8 @@ class ModelAdmin(BaseModelAdmin): selection_note_all = ungettext('%(total_count)s selected', 'All %(total_count)s selected', cl.result_count) - context = dict(self.admin_site.each_context(), + context = dict( + self.admin_site.each_context(), module_name=force_text(opts.verbose_name_plural), selection_note=_('0 of %(cnt)s selected') % {'cnt': len(cl.result_list)}, selection_note_all=selection_note_all % {'total_count': cl.result_count}, @@ -1587,7 +1588,8 @@ class ModelAdmin(BaseModelAdmin): else: title = _("Are you sure?") - context = dict(self.admin_site.each_context(), + context = dict( + self.admin_site.each_context(), title=title, object_name=object_name, object=obj, diff --git a/django/contrib/admin/sites.py b/django/contrib/admin/sites.py index e620154312..7b633bef89 100644 --- a/django/contrib/admin/sites.py +++ b/django/contrib/admin/sites.py @@ -398,7 +398,8 @@ class AdminSite(object): for app in app_list: app['models'].sort(key=lambda x: x['name']) - context = dict(self.each_context(), + context = dict( + self.each_context(), title=self.index_title, app_list=app_list, ) diff --git a/django/contrib/admin/templatetags/admin_list.py b/django/contrib/admin/templatetags/admin_list.py index 777bfbac81..0181f665ad 100644 --- a/django/contrib/admin/templatetags/admin_list.py +++ b/django/contrib/admin/templatetags/admin_list.py @@ -95,7 +95,8 @@ def result_headers(cl): """ ordering_field_columns = cl.get_ordering_field_columns() for i, field_name in enumerate(cl.list_display): - text, attr = label_for_field(field_name, cl.model, + text, attr = label_for_field( + field_name, cl.model, model_admin=cl.model_admin, return_attr=True ) diff --git a/django/contrib/admindocs/tests/test_fields.py b/django/contrib/admindocs/tests/test_fields.py index dd465111a1..3f4efedb9c 100644 --- a/django/contrib/admindocs/tests/test_fields.py +++ b/django/contrib/admindocs/tests/test_fields.py @@ -21,7 +21,8 @@ class TestFieldType(unittest.TestCase): pass def test_field_name(self): - self.assertRaises(AttributeError, + self.assertRaises( + AttributeError, views.get_readable_field_data_type, "NotAField" ) diff --git a/django/contrib/admindocs/urls.py b/django/contrib/admindocs/urls.py index 8aa4dcf946..3dc55657df 100644 --- a/django/contrib/admindocs/urls.py +++ b/django/contrib/admindocs/urls.py @@ -4,38 +4,29 @@ from django.contrib.admindocs import views urlpatterns = patterns('', url('^$', views.BaseAdminDocsView.as_view(template_name='admin_doc/index.html'), - name='django-admindocs-docroot' - ), + name='django-admindocs-docroot'), url('^bookmarklets/$', views.BookmarkletsView.as_view(), - name='django-admindocs-bookmarklets' - ), + name='django-admindocs-bookmarklets'), url('^tags/$', views.TemplateTagIndexView.as_view(), - name='django-admindocs-tags' - ), + name='django-admindocs-tags'), url('^filters/$', views.TemplateFilterIndexView.as_view(), - name='django-admindocs-filters' - ), + name='django-admindocs-filters'), url('^views/$', views.ViewIndexView.as_view(), - name='django-admindocs-views-index' - ), + name='django-admindocs-views-index'), url('^views/(?P[^/]+)/$', views.ViewDetailView.as_view(), - name='django-admindocs-views-detail' - ), + name='django-admindocs-views-detail'), url('^models/$', views.ModelIndexView.as_view(), - name='django-admindocs-models-index' - ), + name='django-admindocs-models-index'), url('^models/(?P[^\.]+)\.(?P[^/]+)/$', views.ModelDetailView.as_view(), - name='django-admindocs-models-detail' - ), + name='django-admindocs-models-detail'), url('^templates/(?P