test0908
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Removed deprecated URLField.verify_exists. 

The deprecation schedule was slightly accelerated because of possible security ramifications.



git-svn-id: http://code.djangoproject.com/svn/django/trunk@17847 bcc190cf-cafb-0310-a4f2-bffc1f526a37
This commit is contained in:
Aymeric Augustin 2012-03-31 13:55:03 +00:00
parent 800e3941c5
commit 9ed6e08ff9
15 changed files with 32 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
django/conf/global_settings.py
View File

@ -403,11 +403,6 @@ THOUSAND_SEPARATOR = ','
# Hint: you really don't!
TRANSACTIONS_MANAGED = False
# The User-Agent string to use when checking for URL validity through the
# isExistingURL validator.
from django import get_version
URL_VALIDATOR_USER_AGENT = "Django/%s (https://www.djangoproject.com)" % get_version()
# The tablespaces to use for each model when not specified otherwise.
DEFAULT_TABLESPACE = ''
DEFAULT_INDEX_TABLESPACE = ''

66
django/core/validators.py
View File

@ -1,6 +1,4 @@
import re
import urllib
import urllib2
import urlparse
from django.core.exceptions import ValidationError
@ -11,13 +9,6 @@ from django.utils.ipv6 import is_valid_ipv6_address
# These values, if given to validate(), will trigger the self.required check.
EMPTY_VALUES = (None, '', [], (), {})
try:
from django.conf import settings
URL_VALIDATOR_USER_AGENT = settings.URL_VALIDATOR_USER_AGENT
except ImportError:
# It's OK if Django settings aren't configured.
URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)'
class RegexValidator(object):
regex = ''
message = _(u'Enter a valid value.')
@ -51,11 +42,8 @@ class URLValidator(RegexValidator):
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
def __init__(self, verify_exists=False,
validator_user_agent=URL_VALIDATOR_USER_AGENT):
def __init__(self):
super(URLValidator, self).__init__()
self.verify_exists = verify_exists
self.user_agent = validator_user_agent
def __call__(self, value):
try:
@ -76,58 +64,6 @@ class URLValidator(RegexValidator):
else:
url = value
if self.verify_exists:
import warnings
warnings.warn(
"The URLField verify_exists argument has intractable security "
"and performance issues. Accordingly, it has been deprecated.",
DeprecationWarning
)
headers = {
"Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5",
"Accept-Language": "en-us,en;q=0.5",
"Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Connection": "close",
"User-Agent": self.user_agent,
}
url = url.encode('utf-8')
# Quote characters from the unreserved set, refs #16812
url = urllib.quote(url, "!*'();:@&=+$,/?#[]")
broken_error = ValidationError(
_(u'This URL appears to be a broken link.'), code='invalid_link')
try:
req = urllib2.Request(url, None, headers)
req.get_method = lambda: 'HEAD'
#Create an opener that does not support local file access
opener = urllib2.OpenerDirector()
#Don't follow redirects, but don't treat them as errors either
error_nop = lambda *args, **kwargs: True
http_error_processor = urllib2.HTTPErrorProcessor()
http_error_processor.http_error_301 = error_nop
http_error_processor.http_error_302 = error_nop
http_error_processor.http_error_307 = error_nop
handlers = [urllib2.UnknownHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPDefaultErrorHandler(),
urllib2.FTPHandler(),
http_error_processor]
try:
import ssl
except ImportError:
# Python isn't compiled with SSL support
pass
else:
handlers.append(urllib2.HTTPSHandler())
map(opener.add_handler, handlers)
opener.open(req, timeout=10)
except ValueError:
raise ValidationError(_(u'Enter a valid URL.'), code='invalid')
except: # urllib2.URLError, httplib.InvalidURL, etc.
raise broken_error
def validate_integer(value):
try:

8
django/db/models/fields/__init__.py
View File

@ -890,7 +890,7 @@ class EmailField(CharField):
description = _("E-mail address")
def __init__(self, *args, **kwargs):
# max_length should be overridden to 254 characters to be fully
# max_length should be overridden to 254 characters to be fully
# compliant with RFCs 3696 and 5321
kwargs['max_length'] = kwargs.get('max_length', 75)
@ -1251,12 +1251,10 @@ class TimeField(Field):
class URLField(CharField):
description = _("URL")
def __init__(self, verbose_name=None, name=None, verify_exists=False,
**kwargs):
def __init__(self, verbose_name=None, name=None, **kwargs):
kwargs['max_length'] = kwargs.get('max_length', 200)
CharField.__init__(self, verbose_name, name, **kwargs)
self.validators.append(
validators.URLValidator(verify_exists=verify_exists))
self.validators.append(validators.URLValidator())
def formfield(self, **kwargs):
# As with CharField, this will cause URL validation to be performed

9
django/forms/fields.py
View File

@ -598,14 +598,11 @@ class ImageField(FileField):
class URLField(CharField):
default_error_messages = {
'invalid': _(u'Enter a valid URL.'),
'invalid_link': _(u'This URL appears to be a broken link.'),
}
def __init__(self, max_length=None, min_length=None, verify_exists=False,
validator_user_agent=validators.URL_VALIDATOR_USER_AGENT, *args, **kwargs):
super(URLField, self).__init__(max_length, min_length, *args,
**kwargs)
self.validators.append(validators.URLValidator(verify_exists=verify_exists, validator_user_agent=validator_user_agent))
def __init__(self, max_length=None, min_length=None, *args, **kwargs):
super(URLField, self).__init__(max_length, min_length, *args, **kwargs)
self.validators.append(validators.URLValidator())
def to_python(self, value):

16
docs/ref/forms/fields.txt
View File

@ -789,7 +789,7 @@ For each field, we describe the default widget used if you don't specify
* Empty value: ``''`` (an empty string)
* Normalizes to: A Unicode object.
* Validates that the given value is a valid URL.
* Error message keys: ``required``, ``invalid``, ``invalid_link``
* Error message keys: ``required``, ``invalid``
Takes the following optional arguments:
@ -798,20 +798,6 @@ For each field, we describe the default widget used if you don't specify
These are the same as ``CharField.max_length`` and ``CharField.min_length``.
.. attribute:: verify_exists
If ``True``, the validator will attempt to load the given URL, raising
``ValidationError`` if the page gives a 404. Defaults to ``False``.
.. deprecated:: 1.4
``verify_exists`` was deprecated for security reasons and will be removed in
Django 1.5. This deprecation also removes ``validator_user_agent``.
.. attribute:: validator_user_agent
String used as the user-agent used when checking for a URL's existence.
Defaults to the value of the :setting:`URL_VALIDATOR_USER_AGENT` setting.
.. versionchanged:: 1.2
The URLField previously did not recognize URLs as valid that contained an IDN
(Internationalized Domain Name; a domain name containing unicode characters)

26
docs/ref/models/fields.txt
View File

@ -484,10 +484,10 @@ A :class:`CharField` that checks that the value is a valid email address.
.. admonition:: Incompliance to RFCs
The default 75 character ``max_length`` is not capable of storing all
possible RFC3696/5321-compliant email addresses. In order to store all
possible valid email addresses, a ``max_length`` of 254 is required.
The default ``max_length`` of 75 exists for historical reasons. The
default has not been changed in order to maintain backwards
possible RFC3696/5321-compliant email addresses. In order to store all
possible valid email addresses, a ``max_length`` of 254 is required.
The default ``max_length`` of 75 exists for historical reasons. The
default has not been changed in order to maintain backwards
compatibility with existing uses of :class:`EmailField`.
``FileField``
@ -887,23 +887,9 @@ shortcuts.
``URLField``
------------
.. class:: URLField([verify_exists=False, max_length=200, **options])
.. class:: URLField([max_length=200, **options])
A :class:`CharField` for a URL. Has one extra optional argument:
.. deprecated:: 1.4
``verify_exists`` is deprecated for security reasons as of 1.4 and will be
removed in Django 1.5. Prior to 1.3.1, the default value was ``True``.
.. attribute:: URLField.verify_exists
If ``True``, the URL given will be checked for existence (i.e.,
the URL actually loads and doesn't give a 404 response) using a
``HEAD`` request. Redirects are allowed, but will not be followed.
Note that when you're using the single-threaded development server,
validating a URL being served by the same server will hang. This should not
be a problem for multithreaded servers.
A :class:`CharField` for a URL.
The admin represents this as an ``<input type="text">`` (a single-line input).

21
docs/ref/settings.txt
View File

@ -2148,18 +2148,6 @@ to ensure your processes are running in the correct environment.
.. _pytz: http://pytz.sourceforge.net/
.. setting:: URL_VALIDATOR_USER_AGENT
URL_VALIDATOR_USER_AGENT
------------------------
Default: ``Django/<version> (https://www.djangoproject.com/)``
The string to use as the ``User-Agent`` header when checking to see if
URLs exist (see the ``verify_exists`` option on
:class:`~django.db.models.URLField`). This setting was deprecated in
1.3.1 along with ``verify_exists`` and will be removed in 1.4.
.. setting:: USE_ETAGS
USE_ETAGS
@ -2344,3 +2332,12 @@ IGNORABLE_404_STARTS
.. deprecated:: 1.4
This setting has been superseded by :setting:`IGNORABLE_404_URLS`.
.. setting:: URL_VALIDATOR_USER_AGENT
URL_VALIDATOR_USER_AGENT
------------------------
.. deprecated:: 1.5
This value was used as the ``User-Agent`` header when checking if a URL
exists, a feature that was removed due to security and performance issues.

22
docs/ref/validators.txt
View File

@ -88,26 +88,10 @@ to, or in lieu of custom ``field.clean()`` methods.
``URLValidator``
----------------
.. class:: URLValidator([verify_exists=False, validator_user_agent=URL_VALIDATOR_USER_AGENT])
.. class:: URLValidator()
A :class:`RegexValidator` that ensures a value looks like a URL and
optionally verifies that the URL actually exists (i.e., doesn't return a
404 status code). Raises an error code of ``'invalid'`` if it doesn't look
like a URL, and a code of ``'invalid_link'`` if it doesn't exist.
:param verify_exists: Sets :attr:`verify_exists`. Defaults to ``False``.
:param validator_user_agent: Sets :attr:`validator_user_agent`. Defaults to
:setting:`URL_VALIDATOR_USER_AGENT` or, if that setting is set to a
null value, ``"Django (https://www.djangoproject.com/)"``.
.. attribute:: verify_exists
If set to ``True``, this validator checks that the URL actually exists.
.. attribute:: validator_user_agent
If :attr:`verify_exists` is ``True``, Django uses this value as the
"User-agent" for the request.
A :class:`RegexValidator` that ensures a value looks like a URL, and raises
an error code of ``'invalid'`` if it doesn't.
``validate_email``
------------------

3
docs/topics/forms/modelforms.txt
View File

@ -106,8 +106,7 @@ Model field Form field
``TimeField`` ``TimeField``
``URLField`` ``URLField`` with ``verify_exists`` set
to the model field's ``verify_exists``
``URLField`` ``URLField``
=============================== ========================================
.. versionadded:: 1.2

3
tests/modeltests/validation/models.py
View File

@ -15,7 +15,6 @@ class ModelToValidate(models.Model):
parent = models.ForeignKey('self', blank=True, null=True, limit_choices_to={'number': 10})
email = models.EmailField(blank=True)
url = models.URLField(blank=True)
url_verify = models.URLField(blank=True, verify_exists=True)
f_with_custom_validator = models.IntegerField(blank=True, null=True, validators=[validate_answer_to_universe])
def clean(self):
@ -102,4 +101,4 @@ try:
auto2 = models.AutoField(primary_key=True)
except AssertionError, assertion_error:
pass # Fail silently
assert str(assertion_error) == u"A model can't have more than one AutoField."
assert str(assertion_error) == u"A model can't have more than one AutoField."

32
tests/modeltests/validation/tests.py
View File

@ -1,14 +1,9 @@
from __future__ import absolute_import
import warnings
from django import forms
from django.core.exceptions import NON_FIELD_ERRORS
from django.test import TestCase
# Import the verify_exists_urls from the 'forms' test app
from regressiontests.forms.tests.fields import verify_exists_urls
from . import ValidationTestCase
from .models import (Author, Article, ModelToValidate,
GenericIPAddressTestModel, GenericIPAddrUnpackUniqueTest)
@ -22,14 +17,6 @@ from .validators import TestModelsWithValidators
class BaseModelValidationTests(ValidationTestCase):
def setUp(self):
self.save_warnings_state()
warnings.filterwarnings('ignore', category=DeprecationWarning,
module='django.core.validators')
def tearDown(self):
self.restore_warnings_state()
def test_missing_required_field_raises_error(self):
mtv = ModelToValidate(f_with_custom_validator=42)
self.assertFailsValidation(mtv.full_clean, ['name', 'number'])
@ -70,25 +57,6 @@ class BaseModelValidationTests(ValidationTestCase):
mtv = ModelToValidate(number=10, name='Some Name', url='not a url')
self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url', [u'Enter a valid value.'])
#The tests below which use url_verify are deprecated
def test_correct_url_but_nonexisting_gives_404(self):
mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=404')
self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
@verify_exists_urls(existing_urls=('http://www.google.com/',))
def test_correct_url_value_passes(self):
mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://www.google.com/')
self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
@verify_exists_urls(existing_urls=('http://qa-dev.w3.org/link-testsuite/http.php?code=301',))
def test_correct_url_with_redirect(self):
mtv = ModelToValidate(number=10, name='Some Name', url_verify='http://qa-dev.w3.org/link-testsuite/http.php?code=301') #example.com is a redirect to iana.org now
self.assertEqual(None, mtv.full_clean()) # This will fail if there's no Internet connection
def test_correct_https_url_but_nonexisting(self):
mtv = ModelToValidate(number=10, name='Some Name', url_verify='https://www.example.com/')
self.assertFieldFailsValidationWithMessage(mtv.full_clean, 'url_verify', [u'This URL appears to be a broken link.'])
def test_text_greater_that_charfields_max_length_raises_erros(self):
mtv = ModelToValidate(number=10, name='Some Name'*100)
self.assertFailsValidation(mtv.full_clean, ['name',])

7
tests/regressiontests/forms/tests/error_messages.py
View File

@ -7,8 +7,6 @@ from django.forms import *
from django.test import TestCase
from django.utils.safestring import mark_safe
from .fields import verify_exists_urls
class AssertFormErrorsMixin(object):
def assertFormErrors(self, expected, the_callable, *args, **kwargs):
@ -143,17 +141,14 @@ class FormsErrorMessagesTestCase(TestCase, AssertFormErrorsMixin):
self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', None))
self.assertFormErrors([u'EMPTY FILE'], f.clean, SimpleUploadedFile('name', ''))
@verify_exists_urls()
def test_urlfield(self):
e = {
'required': 'REQUIRED',
'invalid': 'INVALID',
'invalid_link': 'INVALID LINK',
}
f = URLField(verify_exists=True, error_messages=e)
f = URLField(error_messages=e)
self.assertFormErrors([u'REQUIRED'], f.clean, '')
self.assertFormErrors([u'INVALID'], f.clean, 'abc.c')
self.assertFormErrors([u'INVALID LINK'], f.clean, 'http://www.broken.djangoproject.com')
def test_booleanfield(self):
e = {

61
tests/regressiontests/forms/tests/fields.py
View File

@ -27,10 +27,8 @@ __init__(). For example, CharField has a max_length option.
import datetime
import re
import os
import urllib2
import warnings
from decimal import Decimal
from functools import wraps
from django.core.files.uploadedfile import SimpleUploadedFile
from django.forms import *
@ -48,32 +46,6 @@ def fix_os_paths(x):
return x
def verify_exists_urls(existing_urls=()):
"""
Patches urllib to simulate the availability of some urls even when there
is no Internet connection. This hack should be removed alongside with
`URLField.verify_exists` in Django 1.5.
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
from django.core import validators
# patch urllib2.OpenerDirector
original_open = validators.urllib2.OpenerDirector.open
def custom_open(self, req, data=None, timeout=None):
if req.get_full_url() in existing_urls:
return True
raise Exception()
try:
urllib2.OpenerDirector.open = custom_open
func(*args, **kwargs)
finally:
# unpatch urllib2.OpenerDirector
validators.urllib2.OpenerDirector.open = original_open
return wrapper
return decorator
class FieldsTests(SimpleTestCase):
def setUp(self):
@ -651,30 +623,6 @@ class FieldsTests(SimpleTestCase):
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example.')
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://.com')
@verify_exists_urls(('http://www.google.com/',))
def test_urlfield_3(self):
f = URLField(verify_exists=True)
self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://example')
self.assertRaises(ValidationError, f.clean, 'http://www.broken.djangoproject.com') # bad domain
self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=405') # Method not allowed
try:
f.clean('http://www.broken.djangoproject.com') # bad domain
except ValidationError, e:
self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
self.assertRaises(ValidationError, f.clean, 'http://qa-dev.w3.org/link-testsuite/http.php?code=400') # good domain, bad page
try:
f.clean('http://google.com/we-love-microsoft.html') # good domain, bad page
except ValidationError, e:
self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
@verify_exists_urls(('http://www.google.com/',))
def test_urlfield_4(self):
f = URLField(verify_exists=True, required=False)
self.assertEqual(u'', f.clean(''))
self.assertEqual(u'http://www.google.com/', f.clean('http://www.google.com'))
@verify_exists_urls(('http://example.com/',))
def test_urlfield_5(self):
f = URLField(min_length=15, max_length=20)
self.assertRaisesMessage(ValidationError, "[u'Ensure this value has at least 15 characters (it has 13).']", f.clean, 'http://f.com')
@ -698,7 +646,7 @@ class FieldsTests(SimpleTestCase):
self.assertEqual(u'http://example.com/?some_param=some_value', f.clean('http://example.com?some_param=some_value'))
def test_urlfield_9(self):
f = URLField(verify_exists=False)
f = URLField()
urls = (
u'http://עברית.idn.icann.org/',
u'http://sãopaulo.com/',
@ -722,13 +670,6 @@ class FieldsTests(SimpleTestCase):
except ValidationError, e:
self.assertEqual("[u'This URL appears to be a broken link.']", str(e))
@verify_exists_urls((u'http://xn--hxargifdar.idn.icann.org/%CE%91%CF%81%CF%87%CE%B9%CE%BA%CE%AE_%CF%83%CE%B5%CE%BB%CE%AF%CE%B4%CE%B1',))
def test_urlfield_10(self):
# UTF-8 in the domain.
f = URLField(verify_exists=True)
url = u'http://\u03b5\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac.idn.icann.org/\u0391\u03c1\u03c7\u03b9\u03ba\u03ae_\u03c3\u03b5\u03bb\u03af\u03b4\u03b1'
self.assertEqual(url, f.clean(url))
def test_urlfield_not_string(self):
f = URLField(required=False)
self.assertRaisesMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 23)

2
tests/regressiontests/generic_inline_admin/models.py
View File

@ -16,7 +16,7 @@ class Media(models.Model):
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
content_object = generic.GenericForeignKey()
url = models.URLField(verify_exists=False)
url = models.URLField()
description = models.CharField(max_length=100, blank=True)
keywords = models.CharField(max_length=100, blank=True)

2
tests/regressiontests/model_forms_regress/models.py
View File

@ -57,7 +57,7 @@ class Author1(models.Model):
full_name = models.CharField(max_length=255)
class Homepage(models.Model):
url = models.URLField(verify_exists=False)
url = models.URLField()
class Document(models.Model):
myfile = models.FileField(upload_to='unused', blank=True)