From 9ed6e08ff99c18710c0e4875f827235f04c89d76 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 31 Mar 2012 13:55:03 +0000 Subject: [PATCH] Removed deprecated URLField.verify_exists. The deprecation schedule was slightly accelerated because of possible security ramifications. git-svn-id: http://code.djangoproject.com/svn/django/trunk@17847 bcc190cf-cafb-0310-a4f2-bffc1f526a37 --- django/conf/global_settings.py | 5 -- django/core/validators.py | 66 +------------------ django/db/models/fields/__init__.py | 8 +-- django/forms/fields.py | 9 +-- docs/ref/forms/fields.txt | 16 +---- docs/ref/models/fields.txt | 26 ++------ docs/ref/settings.txt | 21 +++--- docs/ref/validators.txt | 22 +------ docs/topics/forms/modelforms.txt | 3 +- tests/modeltests/validation/models.py | 3 +- tests/modeltests/validation/tests.py | 32 --------- .../forms/tests/error_messages.py | 7 +- tests/regressiontests/forms/tests/fields.py | 61 +---------------- .../generic_inline_admin/models.py | 2 +- .../model_forms_regress/models.py | 2 +- 15 files changed, 32 insertions(+), 251 deletions(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index bd85c121b8..a88ea409f6 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -403,11 +403,6 @@ THOUSAND_SEPARATOR = ',' # Hint: you really don't! TRANSACTIONS_MANAGED = False -# The User-Agent string to use when checking for URL validity through the -# isExistingURL validator. -from django import get_version -URL_VALIDATOR_USER_AGENT = "Django/%s (https://www.djangoproject.com)" % get_version() - # The tablespaces to use for each model when not specified otherwise. DEFAULT_TABLESPACE = '' DEFAULT_INDEX_TABLESPACE = '' diff --git a/django/core/validators.py b/django/core/validators.py index 9f42476ab0..7d7f473b6d 100644 --- a/django/core/validators.py +++ b/django/core/validators.py @@ -1,6 +1,4 @@ import re -import urllib -import urllib2 import urlparse from django.core.exceptions import ValidationError @@ -11,13 +9,6 @@ from django.utils.ipv6 import is_valid_ipv6_address # These values, if given to validate(), will trigger the self.required check. EMPTY_VALUES = (None, '', [], (), {}) -try: - from django.conf import settings - URL_VALIDATOR_USER_AGENT = settings.URL_VALIDATOR_USER_AGENT -except ImportError: - # It's OK if Django settings aren't configured. - URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)' - class RegexValidator(object): regex = '' message = _(u'Enter a valid value.') @@ -51,11 +42,8 @@ class URLValidator(RegexValidator): r'(?::\d+)?' # optional port r'(?:/?|[/?]\S+)$', re.IGNORECASE) - def __init__(self, verify_exists=False, - validator_user_agent=URL_VALIDATOR_USER_AGENT): + def __init__(self): super(URLValidator, self).__init__() - self.verify_exists = verify_exists - self.user_agent = validator_user_agent def __call__(self, value): try: @@ -76,58 +64,6 @@ class URLValidator(RegexValidator): else: url = value - if self.verify_exists: - import warnings - warnings.warn( - "The URLField verify_exists argument has intractable security " - "and performance issues. Accordingly, it has been deprecated.", - DeprecationWarning - ) - - headers = { - "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5", - "Accept-Language": "en-us,en;q=0.5", - "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", - "Connection": "close", - "User-Agent": self.user_agent, - } - url = url.encode('utf-8') - # Quote characters from the unreserved set, refs #16812 - url = urllib.quote(url, "!*'();:@&=+$,/?#[]") - broken_error = ValidationError( - _(u'This URL appears to be a broken link.'), code='invalid_link') - try: - req = urllib2.Request(url, None, headers) - req.get_method = lambda: 'HEAD' - #Create an opener that does not support local file access - opener = urllib2.OpenerDirector() - - #Don't follow redirects, but don't treat them as errors either - error_nop = lambda *args, **kwargs: True - http_error_processor = urllib2.HTTPErrorProcessor() - http_error_processor.http_error_301 = error_nop - http_error_processor.http_error_302 = error_nop - http_error_processor.http_error_307 = error_nop - - handlers = [urllib2.UnknownHandler(), - urllib2.HTTPHandler(), - urllib2.HTTPDefaultErrorHandler(), - urllib2.FTPHandler(), - http_error_processor] - try: - import ssl - except ImportError: - # Python isn't compiled with SSL support - pass - else: - handlers.append(urllib2.HTTPSHandler()) - map(opener.add_handler, handlers) - opener.open(req, timeout=10) - except ValueError: - raise ValidationError(_(u'Enter a valid URL.'), code='invalid') - except: # urllib2.URLError, httplib.InvalidURL, etc. - raise broken_error - def validate_integer(value): try: diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 23be549224..121797a581 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -890,7 +890,7 @@ class EmailField(CharField): description = _("E-mail address") def __init__(self, *args, **kwargs): - # max_length should be overridden to 254 characters to be fully + # max_length should be overridden to 254 characters to be fully # compliant with RFCs 3696 and 5321 kwargs['max_length'] = kwargs.get('max_length', 75) @@ -1251,12 +1251,10 @@ class TimeField(Field): class URLField(CharField): description = _("URL") - def __init__(self, verbose_name=None, name=None, verify_exists=False, - **kwargs): + def __init__(self, verbose_name=None, name=None, **kwargs): kwargs['max_length'] = kwargs.get('max_length', 200) CharField.__init__(self, verbose_name, name, **kwargs) - self.validators.append( - validators.URLValidator(verify_exists=verify_exists)) + self.validators.append(validators.URLValidator()) def formfield(self, **kwargs): # As with CharField, this will cause URL validation to be performed diff --git a/django/forms/fields.py b/django/forms/fields.py index 1fa8d57dff..216bb4c489 100644 --- a/django/forms/fields.py +++ b/django/forms/fields.py @@ -598,14 +598,11 @@ class ImageField(FileField): class URLField(CharField): default_error_messages = { 'invalid': _(u'Enter a valid URL.'), - 'invalid_link': _(u'This URL appears to be a broken link.'), } - def __init__(self, max_length=None, min_length=None, verify_exists=False, - validator_user_agent=validators.URL_VALIDATOR_USER_AGENT, *args, **kwargs): - super(URLField, self).__init__(max_length, min_length, *args, - **kwargs) - self.validators.append(validators.URLValidator(verify_exists=verify_exists, validator_user_agent=validator_user_agent)) + def __init__(self, max_length=None, min_length=None, *args, **kwargs): + super(URLField, self).__init__(max_length, min_length, *args, **kwargs) + self.validators.append(validators.URLValidator()) def to_python(self, value): diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index fc8665b252..4956a71b44 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -789,7 +789,7 @@ For each field, we describe the default widget used if you don't specify * Empty value: ``''`` (an empty string) * Normalizes to: A Unicode object. * Validates that the given value is a valid URL. - * Error message keys: ``required``, ``invalid``, ``invalid_link`` + * Error message keys: ``required``, ``invalid`` Takes the following optional arguments: @@ -798,20 +798,6 @@ For each field, we describe the default widget used if you don't specify These are the same as ``CharField.max_length`` and ``CharField.min_length``. - .. attribute:: verify_exists - - If ``True``, the validator will attempt to load the given URL, raising - ``ValidationError`` if the page gives a 404. Defaults to ``False``. - - .. deprecated:: 1.4 - ``verify_exists`` was deprecated for security reasons and will be removed in - Django 1.5. This deprecation also removes ``validator_user_agent``. - - .. attribute:: validator_user_agent - - String used as the user-agent used when checking for a URL's existence. - Defaults to the value of the :setting:`URL_VALIDATOR_USER_AGENT` setting. - .. versionchanged:: 1.2 The URLField previously did not recognize URLs as valid that contained an IDN (Internationalized Domain Name; a domain name containing unicode characters) diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 8c6a2c9d35..5521a0ed24 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -484,10 +484,10 @@ A :class:`CharField` that checks that the value is a valid email address. .. admonition:: Incompliance to RFCs The default 75 character ``max_length`` is not capable of storing all - possible RFC3696/5321-compliant email addresses. In order to store all - possible valid email addresses, a ``max_length`` of 254 is required. - The default ``max_length`` of 75 exists for historical reasons. The - default has not been changed in order to maintain backwards + possible RFC3696/5321-compliant email addresses. In order to store all + possible valid email addresses, a ``max_length`` of 254 is required. + The default ``max_length`` of 75 exists for historical reasons. The + default has not been changed in order to maintain backwards compatibility with existing uses of :class:`EmailField`. ``FileField`` @@ -887,23 +887,9 @@ shortcuts. ``URLField`` ------------ -.. class:: URLField([verify_exists=False, max_length=200, **options]) +.. class:: URLField([max_length=200, **options]) -A :class:`CharField` for a URL. Has one extra optional argument: - -.. deprecated:: 1.4 - ``verify_exists`` is deprecated for security reasons as of 1.4 and will be - removed in Django 1.5. Prior to 1.3.1, the default value was ``True``. - -.. attribute:: URLField.verify_exists - - If ``True``, the URL given will be checked for existence (i.e., - the URL actually loads and doesn't give a 404 response) using a - ``HEAD`` request. Redirects are allowed, but will not be followed. - - Note that when you're using the single-threaded development server, - validating a URL being served by the same server will hang. This should not - be a problem for multithreaded servers. +A :class:`CharField` for a URL. The admin represents this as an ```` (a single-line input). diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index c06ef1ad3f..a1a0c76470 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2148,18 +2148,6 @@ to ensure your processes are running in the correct environment. .. _pytz: http://pytz.sourceforge.net/ -.. setting:: URL_VALIDATOR_USER_AGENT - -URL_VALIDATOR_USER_AGENT ------------------------- - -Default: ``Django/ (https://www.djangoproject.com/)`` - -The string to use as the ``User-Agent`` header when checking to see if -URLs exist (see the ``verify_exists`` option on -:class:`~django.db.models.URLField`). This setting was deprecated in -1.3.1 along with ``verify_exists`` and will be removed in 1.4. - .. setting:: USE_ETAGS USE_ETAGS @@ -2344,3 +2332,12 @@ IGNORABLE_404_STARTS .. deprecated:: 1.4 This setting has been superseded by :setting:`IGNORABLE_404_URLS`. + +.. setting:: URL_VALIDATOR_USER_AGENT + +URL_VALIDATOR_USER_AGENT +------------------------ + +.. deprecated:: 1.5 + This value was used as the ``User-Agent`` header when checking if a URL + exists, a feature that was removed due to security and performance issues. diff --git a/docs/ref/validators.txt b/docs/ref/validators.txt index e58e53700d..5b83d26d87 100644 --- a/docs/ref/validators.txt +++ b/docs/ref/validators.txt @@ -88,26 +88,10 @@ to, or in lieu of custom ``field.clean()`` methods. ``URLValidator`` ---------------- -.. class:: URLValidator([verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT]) +.. class:: URLValidator() - A :class:`RegexValidator` that ensures a value looks like a URL and - optionally verifies that the URL actually exists (i.e., doesn't return a - 404 status code). Raises an error code of ``'invalid'`` if it doesn't look - like a URL, and a code of ``'invalid_link'`` if it doesn't exist. - - :param verify_exists: Sets :attr:`verify_exists`. Defaults to ``False``. - :param validator_user_agent: Sets :attr:`validator_user_agent`. Defaults to - :setting:`URL_VALIDATOR_USER_AGENT` or, if that setting is set to a - null value, ``"Django (https://www.djangoproject.com/)"``. - - .. attribute:: verify_exists - - If set to ``True``, this validator checks that the URL actually exists. - - .. attribute:: validator_user_agent - - If :attr:`verify_exists` is ``True``, Django uses this value as the - "User-agent" for the request. + A :class:`RegexValidator` that ensures a value looks like a URL, and raises + an error code of ``'invalid'`` if it doesn't. ``validate_email`` ------------------ diff --git a/docs/topics/forms/modelforms.txt b/docs/topics/forms/modelforms.txt index d2aa8863fc..6eadf217cc 100644 --- a/docs/topics/forms/modelforms.txt +++ b/docs/topics/forms/modelforms.txt @@ -106,8 +106,7 @@ Model field Form field ``TimeField`` ``TimeField`` -``URLField`` ``URLField`` with ``verify_exists`` set - to the model field's ``verify_exists`` +``URLField`` ``URLField`` =============================== ======================================== .. versionadded:: 1.2 diff --git a/tests/modeltests/validation/models.py b/tests/modeltests/validation/models.py index 8010701823..e402162b9a 100644 --- a/tests/modeltests/validation/models.py +++ b/tests/modeltests/validation/models.py @@ -15,7 +15,6 @@ class ModelToValidate(models.Model): parent = models.ForeignKey('self', blank=True, null=True, limit_choices_to={'number': 10}) email = models.EmailField(blank=True) url = models.URLField(blank=True) - url_verify = models.URLField(blank=True, verify_exists=True) f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe]) def clean(self): @@ -102,4 +101,4 @@ try: auto2 = models.AutoField(primary_key=True) except AssertionError, assertion_error: pass # Fail silently -assert str(assertion_error) == u"A model can't have more than one AutoField." \ No newline at end of file +assert str(assertion_error) == u"A model can't have more than one AutoField." diff --git a/tests/modeltests/validation/tests.py b/tests/modeltests/validation/tests.py index 5c94f439d1..f255498306 100644 --- a/tests/modeltests/validation/tests.py +++ b/tests/modeltests/validation/tests.py @@ -1,14 +1,9 @@ from __future__ import absolute_import -import warnings - from django import forms from django.core.exceptions import NON_FIELD_ERRORS from django.test import TestCase -# Import the verify_exists_urls from the 'forms' test app -from regressiontests.forms.tests.fields import verify_exists_urls - from . import ValidationTestCase from .models import (Author, Article, ModelToValidate, GenericIPAddressTestModel, GenericIPAddrUnpackUniqueTest) @@ -22,14 +17,6 @@ from .validators import TestModelsWithValidators class BaseModelValidationTests(ValidationTestCase): - def setUp(self): - self.save_warnings_state() - warnings.filterwarnings('ignore', category=DeprecationWarning, - module='django.core.validators') - - def tearDown(self): - self.restore_warnings_state() - def test_missing_required_field_raises_error(self): mtv = ModelToValidate(f_with_custom_validator=42) self.assertFailsValidation(mtv.full_clean, ['name', 'number']) @@ -70,25 +57,6 @@ class BaseModelValidationTests(ValidationTestCase): mtv = ModelToValidate(number=10, name='Some Name', url='not a url') self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'Enter a valid value.']) - #The tests below which use url_verify are deprecated - def test_correct_url_but_nonexisting_gives_404(self): - mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=404') - self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.']) - - @verify_exists_urls(existing_urls=('http://www.google.com/',)) - def test_correct_url_value_passes(self): - mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://www.google.com/') - self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection - - @verify_exists_urls(existing_urls=('http://qa-dev.w3.org/link-testsuite/http.php?code=301',)) - def test_correct_url_with_redirect(self): - mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=301') #example.com is a redirect to iana.org now - self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection - - def test_correct_https_url_but_nonexisting(self): - mtv = ModelToValidate(number=10, name='Some Name', url_verify='https://www.example.com/') - self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.']) - def test_text_greater_that_charfields_max_length_raises_erros(self): mtv = ModelToValidate(number=10, name='Some Name'*100) self.assertFailsValidation(mtv.full_clean, ['name',]) diff --git a/tests/regressiontests/forms/tests/error_messages.py b/tests/regressiontests/forms/tests/error_messages.py index b7b4e986b3..973bf22ab1 100644 --- a/tests/regressiontests/forms/tests/error_messages.py +++ b/tests/regressiontests/forms/tests/error_messages.py @@ -7,8 +7,6 @@ from django.forms import * from django.test import TestCase from django.utils.safestring import mark_safe -from .fields import verify_exists_urls - class AssertFormErrorsMixin(object): def assertFormErrors(self, expected, the_callable, *args, **kwargs): @@ -143,17 +141,14 @@ class FormsErrorMessagesTestCase(TestCase, AssertFormErrorsMixin): self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', None)) self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', '')) - @verify_exists_urls() def test_urlfield(self): e = { 'required': 'REQUIRED', 'invalid': 'INVALID', - 'invalid_link': 'INVALID LINK', } - f = URLField(verify_exists=True, error_messages=e) + f = URLField(error_messages=e) self.assertFormErrors([u'REQUIRED'], f.clean, '') self.assertFormErrors([u'INVALID'], f.clean, 'abc.c') - self.assertFormErrors([u'INVALID LINK'], f.clean, 'http://www.broken.djangoproject.com') def test_booleanfield(self): e = { diff --git a/tests/regressiontests/forms/tests/fields.py b/tests/regressiontests/forms/tests/fields.py index 03e0ff65a3..3820bb9858 100644 --- a/tests/regressiontests/forms/tests/fields.py +++ b/tests/regressiontests/forms/tests/fields.py @@ -27,10 +27,8 @@ __init__(). For example, CharField has a max_length option. import datetime import re import os -import urllib2 import warnings from decimal import Decimal -from functools import wraps from django.core.files.uploadedfile import SimpleUploadedFile from django.forms import * @@ -48,32 +46,6 @@ def fix_os_paths(x): return x -def verify_exists_urls(existing_urls=()): - """ - Patches urllib to simulate the availability of some urls even when there - is no Internet connection. This hack should be removed alongside with - `URLField.verify_exists` in Django 1.5. - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - from django.core import validators - # patch urllib2.OpenerDirector - original_open = validators.urllib2.OpenerDirector.open - def custom_open(self, req, data=None, timeout=None): - if req.get_full_url() in existing_urls: - return True - raise Exception() - try: - urllib2.OpenerDirector.open = custom_open - func(*args, **kwargs) - finally: - # unpatch urllib2.OpenerDirector - validators.urllib2.OpenerDirector.open = original_open - return wrapper - return decorator - - class FieldsTests(SimpleTestCase): def setUp(self): @@ -651,30 +623,6 @@ class FieldsTests(SimpleTestCase): self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example.') self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://.com') - @verify_exists_urls(('http://www.google.com/',)) - def test_urlfield_3(self): - f = URLField(verify_exists=True) - self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com')) - self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example') - self.assertRaises(ValidationError, f.clean, 'http://www.broken.djangoproject.com') # bad domain - self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=405') # Method not allowed - try: - f.clean('http://www.broken.djangoproject.com') # bad domain - except ValidationError, e: - self.assertEqual("[u'This URL appears to be a broken link.']", str(e)) - self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=400') # good domain, bad page - try: - f.clean('http://google.com/we-love-microsoft.html') # good domain, bad page - except ValidationError, e: - self.assertEqual("[u'This URL appears to be a broken link.']", str(e)) - - @verify_exists_urls(('http://www.google.com/',)) - def test_urlfield_4(self): - f = URLField(verify_exists=True, required=False) - self.assertEqual(u'', f.clean('')) - self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com')) - - @verify_exists_urls(('http://example.com/',)) def test_urlfield_5(self): f = URLField(min_length=15, max_length=20) self.assertRaisesMessage(ValidationError, "[u'Ensure this value has at least 15 characters (it has 13).']", f.clean, 'http://f.com') @@ -698,7 +646,7 @@ class FieldsTests(SimpleTestCase): self.assertEqual(u'http://example.com/?some_param=some_value', f.clean('http://example.com?some_param=some_value')) def test_urlfield_9(self): - f = URLField(verify_exists=False) + f = URLField() urls = ( u'http://עברית.idn.icann.org/', u'http://sãopaulo.com/', @@ -722,13 +670,6 @@ class FieldsTests(SimpleTestCase): except ValidationError, e: self.assertEqual("[u'This URL appears to be a broken link.']", str(e)) - @verify_exists_urls((u'http://xn--hxargifdar.idn.icann.org/%CE%91%CF%81%CF%87%CE%B9%CE%BA%CE%AE_%CF%83%CE%B5%CE%BB%CE%AF%CE%B4%CE%B1',)) - def test_urlfield_10(self): - # UTF-8 in the domain. - f = URLField(verify_exists=True) - url = u'http://\u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac.idn.icann.org/\u0391\u03c1\u03c7\u03b9\u03ba\u03ae_\u03c3\u03b5\u03bb\u03af\u03b4\u03b1' - self.assertEqual(url, f.clean(url)) - def test_urlfield_not_string(self): f = URLField(required=False) self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 23) diff --git a/tests/regressiontests/generic_inline_admin/models.py b/tests/regressiontests/generic_inline_admin/models.py index b9426b4f2a..a1555666b3 100644 --- a/tests/regressiontests/generic_inline_admin/models.py +++ b/tests/regressiontests/generic_inline_admin/models.py @@ -16,7 +16,7 @@ class Media(models.Model): content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey() - url = models.URLField(verify_exists=False) + url = models.URLField() description = models.CharField(max_length=100, blank=True) keywords = models.CharField(max_length=100, blank=True) diff --git a/tests/regressiontests/model_forms_regress/models.py b/tests/regressiontests/model_forms_regress/models.py index cdf324164d..d070a9be75 100644 --- a/tests/regressiontests/model_forms_regress/models.py +++ b/tests/regressiontests/model_forms_regress/models.py @@ -57,7 +57,7 @@ class Author1(models.Model): full_name = models.CharField(max_length=255) class Homepage(models.Model): - url = models.URLField(verify_exists=False) + url = models.URLField() class Document(models.Model): myfile = models.FileField(upload_to='unused', blank=True)